// Copyright 2024 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. module ash.recorder_app.mojom; import "chromeos/services/machine_learning/public/mojom/soda.mojom"; import "components/metrics/structured/mojom/event.mojom"; import "mojo/public/mojom/base/string16.mojom"; import "mojo/public/mojom/base/uuid.mojom"; import "services/on_device_model/public/mojom/on_device_model.mojom"; import "services/on_device_model/public/mojom/on_device_model_service.mojom"; // The state of the SODA or on-device model installation. enum ModelStateType { // The model isn't available on this platform. kUnavailable, // The model is available but aren't fully installed yet (Can be either // language pack / library not installed for SODA, or base model / LoRA layer // not installed for on-device models). kNotInstalled, // The model is currently downloading and installing. kInstalling, // The model is installed and ready to be used. kInstalled, // There is an unspecified error while trying to install the model. kError, // The model requires an OS reboot. kNeedsReboot, }; // The state of the model installation, including download progress. struct ModelState { // Current state of the model installation. ModelStateType type; // The install progress. Only available when the state is `kInstalling`. // Progress is the weighted average of the combined download percentage, // in a range of [0, 100]. uint8? progress; }; // Interface for listening to SODA or on-device model installation state. interface ModelStateMonitor { // Update when state of the model is changed. // The initial state are returned by the AddSodaMonitor/AddModelMonitor call // when the `ModelStateMonitor` is added, and this method is only called on // subsequent state updates. Update(ModelState state); }; // Information of an on-device model. struct ModelInfo { // Associated DownLoadable Content (DLC) id. Id is used for requesting DLC // service to download corresponding system feature on-demand at run-time. mojo_base.mojom.Uuid model_id; // The maximum input token size. uint32 input_token_limit; // Whether the model is a large model. bool is_large_model; }; // Retrieved information for a microphone. struct MicrophoneInfo { // Whether the microphone is the system default microphone. bool is_default; // Whether the microphone is an internal microphone. bool is_internal; }; // Interface for listening to system quiet mode state. interface QuietModeMonitor { // Update when system quiet mode is changed. // The initial state is returned by the `AddQuietModeMonitor` call when the // `QuietModeMonitor` is added, and this method is only called on subsequent // state updates. Update(bool in_quiet_mode); }; // Information for a SODA language pack. struct LangPackInfo { // Localized language in BCP-47 format e.g. "en-US". string language_code; // Language name in the application locale. mojo_base.mojom.String16 display_name; // Whether the language is supported by summarization and title suggestion. bool is_gen_ai_supported; // Whether the language is supported by speaker labeling. False if large SODA // model, which supports speaker label, is unavailable on the device. bool is_speaker_label_supported; }; // Primary interface for the chrome://recorder-app WebUI. interface PageHandler { // Returns model information of the given format feature type. // `feature` should be either `kAudioTitle` or `kAudioSummary`. GetModelInfo(on_device_model.mojom.FormatFeature feature) => (ModelInfo model_info); // Binds a new OnDeviceModel interface if possible using model from DLC // specified by `model_id`. // The list of possible IDs are listed in core/platform_handler.ts, and // corresponds to the DLC ID on platform side. LoadModel(mojo_base.mojom.Uuid model_id, pending_receiver model) => (on_device_model.mojom.LoadModelResult result); // Formats the input fields with the model uuid and feature combination. // The `fields` will represent the corresponding input key value pairs for // different model uuid and feature combinations. // The `result` can be nullopt if the model uuid and feature combination is // invalid or not supported. // // For example, a feature may need "name" & "something" & "action" to complete // the following string: "I'm {name}, I have {something}, please {action} for // me.". And different feature may require different sets of key value pairs. FormatModelInput(mojo_base.mojom.Uuid uuid, on_device_model.mojom.FormatFeature feature, map fields) => (string? result); // Validates the `text` is safe or not with the given `safety_feature` and // `safety_info`. The `safety_info` should be the result returned by // `ClassifyTextSafety()` from `on_device_model.mojom.OnDeviceModel`. Returns // true if the `text` is safe. ValidateSafetyResult(on_device_model.mojom.SafetyFeature safety_feature, string text, on_device_model.mojom.SafetyInfo safety_info) => (bool is_safe); // Adds a new monitor for model state change. // Returns the current state. AddModelMonitor(mojo_base.mojom.Uuid model_id, pending_remote monitor) => (ModelState state); // Returns available language information. GetAvailableLangPacks() => (array lang_packs); // Returns default transcription language based on system locale. GetDefaultLanguage() => (string language_code); // Adds a new monitor for SODA state change. `language` is a localized // language in BCP-47 format e.g. "en-US". // Returns the current state. AddSodaMonitor(string language, pending_remote monitor) => (ModelState state); // Requests installation for SODA library and the language pack. `language` is // a localized language in BCP-47 format e.g. "en-US". InstallSoda(string language) => (); // Loads speech recognizer with the given SodaClient and SodaRecognizer // interface. `language` is a localized language in BCP-47 format e.g. // "en-US". Returns false when there's error loading the speech recognizer. // // SODA should be installed before calling this, otherwise false will be // returned. // // The return is a boolean instead of // chromeos.machine_learning.mojom.LoadModelResult, to avoid exposing // implementation details to JS binding. // TODO(pihsun): Move the enum to a separate mojom file and return the full // enum, or have our own enum if different error needs to be treated // differently in frontend. LoadSpeechRecognizer( string language, pending_remote soda_client, pending_receiver soda_recognizer) => (bool result); // Opens the chrome feedback dialog for AI and show `description_template` in // the description field. OpenAiFeedbackDialog(string description_template); // Returns the info of the microphone with the deviceId `source_id`, which is // the ID returned from `enumerateDevices()` in JavaScript. GetMicrophoneInfo(string source_id) => (MicrophoneInfo? info); // Adds a new monitor for system quiet mode change. Returns the current // state. AddQuietModeMonitor(pending_remote monitor) => (bool in_quiet_mode); // Sets the system quiet mode. SetQuietMode(bool quiet_mode); // Gets whether speaker label can be used. // Note that this method specifically only query if the current profile has // the capabilities needed to use speaker label, and the device supports SODA // language pack that supports speaker label, but the caller still need to // ensure that SODA is available and installed to use speaker label. CanUseSpeakerLabel() => (bool supported); // Records the consent of speaker label from user. // Since the frontend doesn't have info about resource ID, the given // consent_description_names and consent_confirmation_name corresponds to the // name (camelCase, without the IDS_RECORDER prefix) in the resources.h, and // will be converted back to the corresponding numbered resource IDs. RecordSpeakerLabelConsent(bool consent_given, array consent_description_names, string consent_confirmation_name); // Gets whether the system audio will be captured by audio loopback. // If returns true, users can use UI switch to include/exclude system audio // via `getDisplayMedia`. Otherwise, microphone echo cancellation should be // turned off to capture the system audio. CanCaptureSystemAudioWithLoopback() => (bool supported); // Tries to record all |events|. Events sent over will be validated to ensure // they are registered in //tools/metrics/structured/sync/structured.xml. Any // event that is not valid will be ignored. RecordStructuredMetrics(array events); };