// Copyright 2026 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

enum Status {
  // The media analytics process is waiting to be launched.
  "UNINITIALIZED",

  // The analytics process is running and the media processing pipeline is
  // started, but it is not yet receiving image frames. This is a
  // transitional state between <code>SUSPENDED</code> and
  // <code>RUNNING</code> for the time it takes to warm up the media
  // processing pipeline, which can take anywhere from a few seconds to a
  // minute.
  // Note: <code>STARTED</code> is the initial reply to SetState
  // <code>RUNNING</code>.
  "STARTED",

  // The analytics process is running and the media processing pipeling is
  // injesting image frames. At this point, MediaPerception signals should
  // be coming over D-Bus.
  "RUNNING",

  // Analytics process is running and the media processing pipeline is ready
  // to be set to state <code>RUNNING</code>. The D-Bus communications
  // are enabled but the media processing pipeline is suspended.
  "SUSPENDED",

  // Enum for restarting the media analytics process using Upstart.
  // Calling setState <code>RESTARTING</code> will restart the media process
  // to the <code>SUSPENDED</code> state. The app has to set the state to
  // <code>RUNNING</code> in order to start receiving media perception
  // information again.
  "RESTARTING",

  // Stops the media analytics process via Upstart.
  "STOPPED",

  // Indicates that a ServiceError has occurred.
  "SERVICE_ERROR"
};

enum ServiceError {
  // The media analytics process could not be reached. This is likely due to
  // a faulty comms configuration or that the process crashed.
  "SERVICE_UNREACHABLE",

  // The media analytics process is not running. The MPP API knows that the
  // process has not been started yet.
  "SERVICE_NOT_RUNNING",

  // The media analytics process is busy launching. Wait for setState
  // <code>RUNNING</code> or setState <code>RESTARTING</code> callback.
  "SERVICE_BUSY_LAUNCHING",

  // The component is not installed properly.
  "SERVICE_NOT_INSTALLED",

  // Failed to establish a Mojo connection to the service.
  "MOJO_CONNECTION_FAILURE"
};

enum Feature {
  "AUTOZOOM",
  "HOTWORD_DETECTION",
  "OCCUPANCY_DETECTION",
  "EDGE_EMBEDDINGS",
  "SOFTWARE_CROPPING"
};

dictionary NamedTemplateArgument {
  DOMString name;
  (DOMString or double)value;
};

enum ComponentType {
  // The smaller component with limited functionality (smaller size and
  // limited models).
  "LIGHT",
  // The fully-featured component with more functionality (larger size and
  // more models).
  "FULL"
};

// The status of the media analytics process component on the device.
enum ComponentStatus {
  "UNKNOWN",
  // The component is successfully installed and the image is mounted.
  "INSTALLED",
  // The component failed to download, install or load.
  "FAILED_TO_INSTALL"
};

// Error code associated with a failure to install the media analytics
// component.
enum ComponentInstallationError {
  // Component requested does not exist.
  "UNKNOWN_COMPONENT",

  // The update engine fails to install component.
  "INSTALL_FAILURE",

  // Component can not be mounted.
  "MOUNT_FAILURE",

  // The component is not compatible with the device.
  "COMPATIBILITY_CHECK_FAILED",

  // The component was not found - reported for load requests with kSkip
  // update policy.
  "NOT_FOUND"
};

dictionary Component {
  required ComponentType type;
};

// The state of the media analytics downloadable component.
dictionary ComponentState {
  required ComponentStatus status;

  // The version string for the current component.
  DOMString version;

  // If the component installation failed, the encountered installation
  // error. Not set if the component installation succeeded.
  ComponentInstallationError installationErrorCode;
};

// ------------------- Start of process management definitions. ------------
// New interface for managing the process state of the media perception
// service with the intention of eventually phasing out the setState() call.
enum ProcessStatus {
  // The component process state is unknown, for example, if the process is
  // waiting to be launched. This is the initial state before
  // $(ref:setComponentProcessState) is first called.
  "UNKNOWN",

  // The component process has been started.
  // This value can only be passed to $(ref:setComponentProcessState) if the
  // process is currently in state <code>STOPPED</code> or
  // <code>UNKNOWN</code>.
  "STARTED",

  // The component process has been stopped.
  // This value can only be passed to $(ref:setComponentProcessState) if the
  // process is currently in state <code>STARTED</code>.
  // Note: the process is automatically stopped when the Chrome process
  // is closed.
  "STOPPED",

  // Indicates that a ServiceError has occurred.
  "SERVICE_ERROR"
};

dictionary ProcessState {
  ProcessStatus status;

  // Return parameter for $(ref:setComponentProcessState) that
  // specifies the error type for failure cases.
  ServiceError serviceError;
};
// ------------------- End of process management definitions. --------------

// The parameters for processing a particular video stream.
dictionary VideoStreamParam {
  // Identifies the video stream described by these parameters.
  DOMString id;

  // Frame width in pixels.
  long width;

  // Frame height in pixels.
  long height;

  // The frame rate at which this video stream would be processed.
  long frameRate;
};

dictionary Point {
  // The horizontal distance from the top left corner of the image.
  double x;

  // The vertical distance from the top left corner of the image.
  double y;
};

// The parameters for a whiteboard in the image frame. Corners are given in
// pixel coordinates normalized to the size of the image frame (i.e. in the
// range [(0.0, 0.0), (1.0, 1.0)]. The aspectRatio is the physical aspect
// ratio of the whiteboard (e.g. for a 1m high and 2m wide whiteboard, the
// aspect ratio would be 2).
dictionary Whiteboard {
  // The top left corner of the whiteboard in the image frame.
  Point topLeft;

  // The top right corner of the whiteboard in the image frame.
  Point topRight;

  // The bottom left corner of the whiteboard in the image frame.
  Point bottomLeft;

  // The bottom right corner of the whiteboard in the image frame.
  Point bottomRight;

  // The physical aspect ratio of the whiteboard.
  double aspectRatio;
};

// The system and configuration state of the analytics process.
dictionary State {
  required Status status;

  // Optional $(ref:setState) parameter. Specifies the video device the media
  // analytics process should open while the media processing pipeline is
  // starting. To set this parameter, status has to be <code>RUNNING</code>.
  DOMString deviceContext;

  // Return parameter for $(ref:setState) or $(ref:getState) that
  // specifies the error type for failure cases.
  ServiceError serviceError;

  // A list of video streams processed by the analytics process. To set this
  // parameter, status has to be <code>RUNNING</code>.
  sequence<VideoStreamParam> videoStreamParam;

  // Media analytics configuration. It can only be used when setting state to
  // RUNNING.
  DOMString configuration;

  // Corners and aspect ratio of the whiteboard in the image frame. Should
  // only be set when setting state to <code>RUNNING</code> and configuration
  // to whiteboard.
  Whiteboard whiteboard;

  // A list of enabled media perception features.
  sequence<Feature> features;

  // A list of named parameters to be substituted at start-up. Will
  // only have effect when setting state to <code>RUNNING</code>.
  sequence<NamedTemplateArgument> namedTemplateArguments;
};

dictionary BoundingBox {
  // Specifies whether the points are normalized to the size of the image.
  boolean normalized;

  // The two points that define the corners of a bounding box.
  Point topLeft;
  Point bottomRight;
};

enum DistanceUnits { "UNSPECIFIED", "METERS", "PIXELS" };

// Generic dictionary to encapsulate a distance magnitude and units.
dictionary Distance {
  // This field provides flexibility to report depths or distances of
  // different entity types with different units.
  DistanceUnits units;

  double magnitude;
};

enum EntityType {
  "UNSPECIFIED",
  "FACE",
  "PERSON",
  "MOTION_REGION",
  "LABELED_REGION"
};

enum FramePerceptionType {
  "UNKNOWN_TYPE",
  "FACE_DETECTION",
  "PERSON_DETECTION",
  "MOTION_DETECTION"
};

dictionary Entity {
  // A unique id associated with the detected entity, which can be used to
  // track the entity over time.
  long id;

  EntityType type;

  // Label for this entity.
  DOMString entityLabel;

  // Minimum box which captures entire detected entity.
  BoundingBox boundingBox;

  // A value for the quality of this detection.
  double confidence;

  // The estimated depth of the entity from the camera.
  Distance depth;
};

dictionary PacketLatency {
  // Label for this packet.
  DOMString packetLabel;

  // Packet processing latency in microseconds.
  long latencyUsec;
};

// Type of lighting conditions.
enum LightCondition {
  "UNSPECIFIED",

  // No noticeable change occurred.
  "NO_CHANGE",

  // Light was switched on in the room.
  "TURNED_ON",

  // Light was switched off in the room.
  "TURNED_OFF",

  // Light gradually got dimmer (for example, due to a sunset).
  "DIMMER",

  // Light gradually got brighter (for example, due to a sunrise).
  "BRIGHTER",

  // Black frame was detected - the current frame contains only noise.
  "BLACK_FRAME"
};

// Detection of human presence close to the camera.
dictionary VideoHumanPresenceDetection {
  // Indicates a probability in [0, 1] interval that a human is present in
  // the video frame.
  double humanPresenceLikelihood;

  // Indicates a probability in [0, 1] that motion has been detected in the
  // video frame.
  double motionDetectedLikelihood;

  // Indicates lighting condition in the video frame.
  LightCondition lightCondition;

  // Indicates a probablity in [0, 1] interval that
  // <code>lightCondition</code> value is correct.
  double lightConditionLikelihood;
};

// The set of computer vision metadata for an image frame.
dictionary FramePerception {
  long frameId;

  long frameWidthInPx;
  long frameHeightInPx;

  // The timestamp associated with the frame (when its recieved by the
  // analytics process).
  double timestamp;

  // The list of entities detected in this frame.
  sequence<Entity> entities;

  // Processing latency for a list of packets.
  sequence<PacketLatency> packetLatency;

  // Human presence detection results for a video frame.
  VideoHumanPresenceDetection videoHumanPresenceDetection;

  // Indicates what types of frame perception were run.
  sequence<FramePerceptionType> framePerceptionTypes;
};

// An estimate of the direction that the sound is coming from.
dictionary AudioLocalization {
  // An angle in radians in the horizontal plane. It roughly points to the
  // peak in the probability distribution of azimuth defined below.
  double azimuthRadians;

  // A probability distribution for the current snapshot in time that shows
  // the likelihood of a sound source being at a particular azimuth. For
  // example, <code>azimuthScores = [0.1, 0.2, 0.3, 0.4]</code> means that
  // the probability that the sound is coming from an azimuth of 0, pi/2, pi,
  // 3*pi/2 is 0.1, 0.2, 0.3 and 0.4, respectively.
  sequence<double> azimuthScores;
};

// Spectrogram of an audio frame.
dictionary AudioSpectrogram {
  sequence<double> values;
};

// Detection of human presence close to the microphone.
dictionary AudioHumanPresenceDetection {
  // Indicates a probability in [0, 1] interval that a human has caused a
  // sound close to the microphone.
  double humanPresenceLikelihood;

  // Estimate of the noise spectrogram.
  AudioSpectrogram noiseSpectrogram;

  // Spectrogram of an audio frame.
  AudioSpectrogram frameSpectrogram;
};

enum HotwordType { "UNKNOWN_TYPE", "OK_GOOGLE" };

// A hotword detected in the audio stream.
dictionary Hotword {
  // Unique identifier for the hotword instance. Note that a single hotword
  // instance can span more than one audio frame. In that case a single
  // hotword instance can be reported in multiple Hotword or HotwordDetection
  // results. Hotword results associated with the same hotword instance will
  // have the same <code>id</code>.
  long id;

  // Indicates the type of this hotword.
  HotwordType type;

  // Id of the audio frame in which the hotword was detected.
  long frameId;

  // Indicates the start time of this hotword in the audio frame.
  long startTimestampMs;

  // Indicates the end time of this hotword in the audio frame.
  long endTimestampMs;

  // Indicates a probability in [0, 1] interval that this hotword is present
  // in the audio frame.
  double confidence;
};

// Detection of hotword in the audio stream.
dictionary HotwordDetection {
  sequence<Hotword> hotwords;
};

// Audio perception results for an audio frame.
dictionary AudioPerception {
  // A timestamp in microseconds attached when this message was generated.
  double timestampUs;

  // Audio localization results for an audio frame.
  AudioLocalization audioLocalization;

  // Audio human presence detection results for an audio frame.
  AudioHumanPresenceDetection audioHumanPresenceDetection;

  // Hotword detection results.
  HotwordDetection hotwordDetection;
};

// Detection of human presence based on both audio and video inputs.
dictionary AudioVisualHumanPresenceDetection {
  // Indicates a probability in [0, 1] interval that a human is present.
  double humanPresenceLikelihood;
};

// Perception results based on both audio and video inputs.
dictionary AudioVisualPerception {
  // A timestamp in microseconds attached when this message was generated.
  double timestampUs;

  // Human presence detection results.
  AudioVisualHumanPresenceDetection audioVisualHumanPresenceDetection;
};

// Stores metadata such as version of media perception features.
dictionary Metadata {
  DOMString visualExperienceControllerVersion;
};

dictionary MediaPerception {
  // The time the media perception data was emitted by the media processing
  // pipeline. This value will be greater than the timestamp stored within
  // the FramePerception dictionary and the difference between them can be
  // viewed as the processing time for a single frame.
  double timestamp;

  // An array of framePerceptions.
  sequence<FramePerception> framePerceptions;

  // An array of audio perceptions.
  sequence<AudioPerception> audioPerceptions;

  // An array of audio-visual perceptions.
  sequence<AudioVisualPerception> audioVisualPerceptions;

  // Stores metadata such as version of media perception features.
  Metadata metadata;
};

enum ImageFormat {
  // Image represented by RGB data channels.
  "RAW",
  "PNG",
  "JPEG"
};

dictionary ImageFrame {
  long width;
  long height;

  ImageFormat format;

  long dataLength;

  // The bytes of the image frame.
  ArrayBuffer frame;
};

dictionary PerceptionSample {
  // The video analytics FramePerception for the associated image frame
  // data.
  FramePerception framePerception;

  // The image frame data for the associated FramePerception object.
  ImageFrame imageFrame;

  // The audio perception results for an audio frame.
  AudioPerception audioPerception;

  // Perception results based on both audio and video inputs.
  AudioVisualPerception audioVisualPerception;

  // Stores metadata such as version of media perception features.
  Metadata metadata;
};

dictionary Diagnostics {
  // Return parameter for $(ref:getDiagnostics) that specifies the error
  // type for failure cases.
  ServiceError serviceError;

  // A buffer of image frames and the associated video analytics information
  // that can be used to diagnose a malfunction.
  sequence<PerceptionSample> perceptionSamples;
};

// Listener callback for the onMediaPerception event.
// |mediaPerception| : The dictionary which contains a dump of everything
// the analytics process has detected or determined from the incoming media
// streams.
callback OnMediaPerceptionListener = undefined(MediaPerception mediaPerception);

interface OnMediaPerceptionEvent : ExtensionEvent {
  static undefined addListener(OnMediaPerceptionListener listener);
  static undefined removeListener(OnMediaPerceptionListener listener);
  static boolean hasListener(OnMediaPerceptionListener listener);
};

// Private API for receiving real-time media perception information.
[platforms = ("chromeos")] interface MediaPerceptionPrivate {
  // Gets the status of the media perception process.
  // |Returns| : The current state of the system.
  // |PromiseValue| : state
  static Promise<State> getState();

  // Sets the desired state of the system.
  // |state| : A dictionary with the desired new state. The only settable
  // states are <code>RUNNING</code>, <code>SUSPENDED</code>, and
  // <code>RESTARTING</code>.
  // |Returns| : Invoked with the State of the system after setting it. Can
  // be used to verify the state was set as desired.
  // |PromiseValue| : state
  static Promise<State> setState(State state);

  // Get a diagnostics buffer out of the video analytics process.
  // |Returns| : Returns a Diagnostics dictionary object.
  // |PromiseValue| : diagnostics
  static Promise<Diagnostics> getDiagnostics();

  // Attempts to download and load the media analytics component. This
  // function should be called every time a client starts using this API. If
  // the component is already loaded, the callback will simply return that
  // information. The process must be <code>STOPPED</code> for this function
  // to succeed.
  // Note: If a different component type is desired, this function can
  // be called with the new desired type and the new component will be
  // downloaded and installed.
  // |component| : The desired component to install and load.
  // |Returns| : Returns the state of the component.
  // |PromiseValue| : componentState
  static Promise<ComponentState> setAnalyticsComponent(Component component);

  // Manages the lifetime of the component process. This function should
  // only be used if the component is installed. It will fail if the
  // component is not installed.
  // |processState| : The desired state for the component process.
  // |Returns| : Reports the new state of the process, which is expected to
  // be the same as the desired state, unless something goes wrong.
  // |PromiseValue| : processState
  static Promise<ProcessState> setComponentProcessState(
      ProcessState processState);

  // Fired when media perception information is received from the media
  // analytics process.
  static attribute OnMediaPerceptionEvent onMediaPerception;
};

partial interface Browser {
  static attribute MediaPerceptionPrivate mediaPerceptionPrivate;
};