// Copyright 2024 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #ifndef CHROME_BROWSER_UI_LENS_LENS_OVERLAY_CONTROLLER_H_ #define CHROME_BROWSER_UI_LENS_LENS_OVERLAY_CONTROLLER_H_ #include #include #include #include #include "base/callback_list.h" #include "base/memory/raw_ptr.h" #include "base/memory/weak_ptr.h" #include "base/scoped_observation.h" #include "base/time/time.h" #include "base/timer/timer.h" #include "chrome/browser/content_extraction/inner_html.h" #include "chrome/browser/lens/core/mojom/geometry.mojom.h" #include "chrome/browser/lens/core/mojom/lens.mojom.h" #include "chrome/browser/lens/core/mojom/lens_side_panel.mojom.h" #include "chrome/browser/lens/core/mojom/overlay_object.mojom.h" #include "chrome/browser/lens/core/mojom/page_content_type.mojom.h" #include "chrome/browser/lens/core/mojom/text.mojom.h" #include "chrome/browser/themes/theme_service.h" #include "base/callback_list.h" #include "chrome/browser/ui/lens/lens_overlay_blur_layer_delegate.h" #include "chrome/browser/ui/lens/lens_overlay_colors.h" #include "chrome/browser/ui/lens/lens_overlay_gen204_controller.h" #include "chrome/browser/ui/lens/lens_overlay_languages_controller.h" #include "chrome/browser/ui/lens/lens_overlay_translate_options.h" #include "chrome/browser/ui/lens/lens_query_flow_router.h" #include "chrome/browser/ui/lens/overlay_base_controller.h" #include "chrome/browser/ui/omnibox/omnibox_tab_helper.h" #include "chrome/browser/ui/webui/searchbox/lens_searchbox_client.h" #include "components/content_extraction/content/browser/inner_text.h" #include "components/find_in_page/find_result_observer.h" #include "components/lens/lens_overlay_dismissal_source.h" #include "components/lens/lens_overlay_first_interaction_type.h" #include "components/lens/lens_overlay_invocation_source.h" #include "components/lens/lens_overlay_metrics.h" #include "components/lens/lens_overlay_mime_type.h" #include "components/lens/lens_overlay_side_panel_result.h" #include "components/omnibox/browser/autocomplete_match_type.h" #include "components/sessions/core/session_id.h" #include "components/tabs/public/tab_interface.h" #include "components/url_matcher/regex_set_matcher.h" #include "components/url_matcher/url_matcher.h" #include "components/url_matcher/url_util.h" #include "components/viz/common/frame_timing_details.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" #include "pdf/buildflags.h" #include "third_party/skia/include/core/SkBitmap.h" #include "ui/base/interaction/element_identifier.h" #include "ui/base/mojom/window_open_disposition.mojom.h" #include "ui/views/view_observer.h" #include "ui/views/widget/widget_observer.h" #if BUILDFLAG(ENABLE_PDF) #include "pdf/mojom/pdf.mojom.h" #endif // BUILDFLAG(ENABLE_PDF) namespace content { class WebUI; } // namespace content namespace lens { class LensSessionMetricsLogger; class LensOverlayQueryController; class LensOverlaySidePanelCoordinator; class LensPermissionBubbleController; class LensResultsPanelRouter; class LensSearchboxController; class LensSearchContextualizationController; struct SearchQuery; class SidePanelInUse; namespace proto { class LensOverlaySuggestInputs; class LensOverlayUrlResponse; } // namespace proto } // namespace lens namespace ui { class TrackedElement; } // namespace ui class LensSearchController; class PrefService; enum class SidePanelEntryHideReason; // Manages all state associated with the lens overlay. // This class is not thread safe. It should only be used from the browser // thread. class LensOverlayController : public OverlayBaseController, public lens::mojom::LensPageHandler, public OmniboxTabHelper::Observer, public find_in_page::FindResultObserver { public: LensOverlayController(tabs::TabInterface* tab, LensSearchController* lens_search_controller, PrefService* pref_service); ~LensOverlayController() override; DECLARE_CLASS_ELEMENT_IDENTIFIER_VALUE(kOverlaySidePanelWebViewId); // A simple utility that gets the the LensOverlayController TabFeature set by // the embedding tab of a lens WebUI hosted in `webui_web_contents`. // May return nullptr if no LensOverlayController TabFeature is associated // with `webui_web_contents`. static LensOverlayController* FromWebUIWebContents( content::WebContents* webui_web_contents); // A simple utility that gets the the LensOverlayController TabFeature set by // the instances of WebContents associated with a tab. // May return nullptr if no LensOverlayController TabFeature is associated // with `tab_web_contents`. static LensOverlayController* FromTabWebContents( content::WebContents* tab_web_contents); // This method is used to set up communication between this instance and the // overlay WebUI. This is called by the WebUIController when the WebUI is // executing javascript and ready to bind. virtual void BindOverlay( mojo::PendingReceiver receiver, mojo::PendingRemote page); // Returns the screenshot initially displayed on this overlay. If no // screenshot is showing, will return nullptr. const SkBitmap& initial_screenshot() { return initialization_data_->initial_screenshot_; } // Returns the screenshot of the live page which may have been updated after // the overlay is hidden and the live page is shown. If no screenshot is // showing, will return nullptr. const SkBitmap& updated_screenshot() { return initialization_data_->updated_screenshot_; } // Returns the dynamic color palette identifier based on the screenshot. // Returns whether visual searches should be fulfilled by AIM rather than // load immediately in the results panel. bool use_aim_for_visual_search() { return use_aim_for_visual_search_; } // Returns invocation time since epoch. Used to set up html source for metric // logging. uint64_t GetInvocationTimeSinceEpoch(); // Send text data to the WebUI, or stores it to be sent when the WebUI is // ready. void SendText(lens::mojom::TextPtr text); // Send region text data to the WebUI and indicates whether the text is from // an injected image. If the WebUI is not ready, this is a no-op. void SendRegionText(lens::mojom::TextPtr text, bool is_injected_image); // Creates theme with data obtained from `palette_id` to be sent to the WebUI. // Send overlay object data to the WebUI, or stores it to be sent when the // WebUI is ready. void SendObjects(std::vector objects); // Send message to overlay notifying that the results side panel opened. virtual void NotifyResultsPanelOpened(); // Send message to overlay to copy the currently selection if any. void TriggerCopy(); // Returns true if the overlay has a region selection. bool HasRegionSelection() const; // Returns true if the results side panel is currently showing, or is opening. bool IsResultsSidePanelShowingOrWillOpen(); // Pass a result frame URL to load in the side panel. void LoadURLInResultsFrame(const GURL& url); // Queues a tutorial IPH to be shown if the given URL is eligible. Cancels any // queued IPH. void MaybeShowDelayedTutorialIPH(const GURL& url); // Updates the metrics related to navigations for the current page. void UpdateNavigationMetrics(); // Clears any selections currently made in the overlay. void ClearAllSelections(); // Handles a new region thumbnail being created. void HandleRegionBitmapCreated(const SkBitmap& region_bitmap); // TODO(crbug.com/404941800): All the Handle*Response methods should not exist // in this class. They currently exist to unblock development. They will be // removed once the migration is complete. Handles the response to the Lens // start query request. virtual void HandleStartQueryResponse( std::vector objects, lens::mojom::TextPtr text, bool is_error); // Testing function to issue a Lens region selection request. void IssueLensRegionRequestForTesting(lens::mojom::CenterRotatedBoxPtr region, bool is_click); // Testing function to issue a text request. void IssueTextSelectionRequestForTesting(const std::string& text_query, int selection_start_index, int selection_end_index, bool is_translate = false); // Testing function to issue a task completion event for a user action. void RecordUkmAndTaskCompletionForLensOverlayInteractionForTesting( lens::mojom::UserAction user_action); // Testing function to issue a semantic event. void RecordSemanticEventForTesting(lens::mojom::SemanticEvent event); // Testing function to issue a translate request. void IssueTranslateSelectionRequestForTesting( const std::string& text_query, const std::string& content_language, int selection_start_index, int selection_end_index); // Testing function to issue a math request. void IssueMathSelectionRequestForTesting(const std::string& query, const std::string& formula, int selection_start_index, int selection_end_index); // Testing function to issue a full page translate request. void IssueTranslateFullPageRequestForTesting( const std::string& source_language, const std::string& target_language); // Testing function to end translate mode. void IssueEndTranslateModeRequestForTesting(); // Testing function to issue a searchbox request. void IssueSearchBoxRequestForTesting( base::Time query_start_time, const std::string& search_box_text, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, std::map additional_query_params); // Gets the invocation source enum. lens::LensOverlayInvocationSource invocation_source() const { return invocation_source_; } // Returns true if the CoBrowse panel with Lens overlay is enabled. bool CoBrowsePanelWithLensOverlayEnabled() const; // Returns true if the non-blocking privacy notice should be hidden. Note that // non-blocking permission flows may still apply if this is true; the notice // is just not displayed. bool ShouldHideNonBlockingPrivacyNotice() const; // Gets string for invocation source enum, used for logging metrics. std::string GetInvocationSourceString(); // Gets the WebContents housed in the side panel for testing. content::WebContents* GetSidePanelWebContentsForTesting(); // Returns the current page URL for testing. const GURL& GetPageURLForTesting(); // Returns the current tab ID for testing. SessionID GetTabIdForTesting(); // Returns the current searchbox page classification for testing. metrics::OmniboxEventProto::PageClassification GetPageClassificationForTesting(); // Returns the current thumbnail URI for testing. const std::string& GetThumbnailForTesting(); // Handles the event where text was modified in the searchbox for testing. void OnTextModifiedForTesting(); // Handles the event where the thumbnail was removed from the searchbox for // testing. void OnThumbnailRemovedForTesting(); // Handles the event where searchbox was focused for testing. void OnFocusChangedForTesting(bool focused); // Handles the event where zero suggest was shown for testing. void OnZeroSuggestShownForTesting(); // Opens the side panel for testing. If the side panel is already open, this // does nothing. void OpenSidePanelForTesting(); // Sets the invocation time for WebUI binding. void SetInvocationTimeForWebUIBinding(base::TimeTicks time); // Clears the selected region. void ClearRegionSelection(); // Returns the lens suggest inputs stored in this controller for testing. lens::proto::LensOverlaySuggestInputs GetLensSuggestInputsForTesting(); // Returns true if tutorial IPH is eligible to be shown for the given URL for // testing. bool IsUrlEligibleForTutorialIPHForTesting(const GURL& url); const lens::mojom::CenterRotatedBoxPtr& get_selected_region_for_testing() { return initialization_data_->selected_region_; } LensSearchController* get_lens_search_controller_for_testing() { return lens_search_controller_; } const std::optional> get_selected_text_for_region() { return initialization_data_->selected_text_; } const std::vector& GetSignificantRegionBoxesForTesting() { return initialization_data_->significant_region_boxes_; } protected: friend class LensSearchController; friend class lens::LensSearchboxController; friend class lens::LensOverlaySidePanelCoordinator; // This is entry point for showing the overlay UI. This has no effect if state // is not kOff. This has no effect if the tab is not in the foreground. If the // overlay is successfully invoked, then the value of `invocation_source` will // be recorded in the relevant metrics. void ShowUI(lens::LensOverlayInvocationSource invocation_source); // Issues a text search request for Lens to fulfill, which may or may not be // contextualized. // No-op if the Lens Overlay is off or closing. If the Lens Overlay is in the // process of opening, the request will be queued until the overlay is fully // opened. void IssueTextSearchRequest( std::string query_text, std::map additional_query_parameters, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, lens::LensOverlayInvocationSource invocation_source); // Sets a region to search after the overlay loads, then calls ShowUI(). // All units are in device pixels. region_bitmap contains the high definition // image bytes to use for the search instead of cropping the region from the // viewport. void ShowUIWithPendingRegion( lens::LensOverlayInvocationSource invocation_source, lens::mojom::CenterRotatedBoxPtr region, const SkBitmap& region_bitmap); // Closes the overlay UI and sets state to kOff. This method is the final // cleanup of closing the overlay UI. This resets all state internal to the // LensOverlayController. // Anyone called trying to close the UI should go through CloseUIAsync or // CloseUISync. Those methods also reset state external to // LensOverlayController. void CloseUI() override; // Returns the vsrid to use for the new tab URL. std::string GetVsridForNewTab(); // Sets the overlay translate mode. If `translate_options` is nullopt, it will // disable translate mode. void SetTranslateMode( std::optional translate_options); // Sets the text selection on the overlay. void SetTextSelection(int32_t selection_start_index, int32_t selection_end_index); // Sets the post region selection on the overlay. void SetPostRegionSelection(lens::mojom::CenterRotatedBoxPtr); // Stores the additional query parameters to pass to the query controller for // generating urls, set by the search box. void SetAdditionalSearchQueryParams( std::map additional_search_query_params); // Clears the selected text from the overlay if there is any. void ClearTextSelection(); // Called by the searchbox controller when the focus on the searchbox changes. void OnSearchboxFocusChanged(bool focused); // Called by the searchbox controller when zero suggest is shown. void OnZeroSuggestShown(); // Makes a Lens request and updates all state related to the Lens request. If // region_bitmap is provided, it will use those bytes to send to the Lens // server instead of cropping the region from the full page screenshot. void IssueLensRequest(base::Time query_start_time, lens::mojom::CenterRotatedBoxPtr region, lens::LensOverlaySelectionType selection_type, std::optional region_bitmap); // Issues a multimodal request to the query controller. void IssueMultimodalRequest(base::Time query_start_time, lens::mojom::CenterRotatedBoxPtr region, const std::string& text_query, lens::LensOverlaySelectionType selection_type, std::optional region_bitmap); // Tries to update the page content and then issues a searchbox request. void IssueSearchBoxRequest( base::Time query_start_time, const std::string& search_box_text, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, std::map additional_query_params, std::optional invocation_source); // Shows the mobile promo if the user is eligible. void MaybeShowMobilePromo(); // Issues a contextual text request to the query controller. void IssueContextualTextRequest( base::Time query_start_time, const std::string& text_query, lens::LensOverlaySelectionType selection_type); // Returns a search query struct containing the current state of the overlay. void AddOverlayStateToSearchQuery(lens::SearchQuery& search_query); // Handles the URL response to the Lens interaction request. void HandleInteractionURLResponse( lens::proto::LensOverlayUrlResponse response); // Handles the text response to the Lens interaction request. void HandleInteractionResponse(lens::mojom::TextPtr text); // Handles the progress of the page content upload. Notifies the side panel // to update the progress bar. void HandlePageContentUploadProgress(uint64_t position, uint64_t total); // Should only be called when the overlay is in kHidden state. This will // reshow the overlay using the current viewport screenshot and page context // on the live page. void ReshowOverlay() override; // OverlayBaseController overrides allowed for testing: bool IsResultsSidePanelShowing() override; bool ShouldWaitForSidePanelReflow() override; private: // Data class for constructing overlay and storing overlay state for // kSuspended state. struct OverlayInitializationData { public: // This is data used to initialize the overlay after the WebUI has been // bound to the overlay controller. The only required fields are the // screenshot, data URI, and the page information if the data is allowed // to be shared. The rest of the fields are optional because the overlay // does not require any server response data for use. rgb_screenshot passes // ownership of the Bitmap to OverlayInitializationData. OverlayInitializationData(const SkBitmap& screenshot, SkBitmap rgb_screenshot, GURL page_url, std::optional page_title); ~OverlayInitializationData(); // Whether there is any full image response data present. bool has_full_image_response() const { return !text_.is_null() || !objects_.empty(); } // The screenshot that is initially rendered by the WebUI. // initial_screenshot_ is in native format and is needed to encode JPEGs to // send to the server. initial_rgb_screenshot_ is in RGBA color type and // used to display in the WebUI. initial_rgb_screenshot_ cannot be used to // encode JPEGs because the JPEG encoder expects the native color format. SkBitmap initial_screenshot_; SkBitmap initial_rgb_screenshot_; // Screenshot of the live page which may be updated after the overlay is // hidden and the live page is shown. Initially equal to // initial_screenshot_. SkBitmap updated_screenshot_; // The dynamic color palette identifier based on the screenshot. // The page url. Empty if it is not allowed to be shared. GURL page_url_; // The page title, if it is allowed to be shared. std::optional page_title_; // The data of the content the user is viewing. There can be multiple // content types for a single page, so we store them all in this struct. std::vector page_contents_; // The primary type of the data stored in page_contents_. This is the value // used to determine request params and what content to look at when // determining if the page_contents_ needs to be present. lens::MimeType primary_content_type_ = lens::MimeType::kUnknown; // The page count of the PDF document if page_content_type_ is kPdf. std::optional pdf_page_count_; // The partial representation of a PDF document. The element at a given // index holds the text of the PDF page at the same index. std::vector pdf_pages_text_; // The most visible page of the PDF document when the viewport was last // updated, if page_content_type_ is kPdf. std::optional last_retrieved_most_visible_page_; // Bounding boxes for significant regions identified in the screenshot. std::vector significant_region_boxes_; // The selected region. Stored so that it can be used for multiple // requests, such as if the user changes the text query without changing // the region. Cleared if the user makes a text-only or object selection // query. lens::mojom::CenterRotatedBoxPtr selected_region_; // The selected region bitmap. This should only be set if the user opened // the overlay with a pending region bitmap. Stored so that it can be used // for multiple requests, such as if the user changes the text query without // changing the region. Cleared if the user makes a text-only or object // selection query. SkBitmap selected_region_bitmap_; // A pair representing the start and end selection indexes for the currently // selected text. This needs to be an optional since std::pair will // initialize with default values. std::optional> selected_text_; // Text returned from the full image response. lens::mojom::TextPtr text_; // Overlay objects returned from the full image response. std::vector objects_; // The additional query parameters to pass to the query controller for // generating urls, set by the search box. std::map additional_search_query_params_; // The translate options currently enabled in the overlay. std::optional translate_options_; }; // Implementation of IssueTextSearchRequest() for passing query_start_time. void IssueTextSearchRequestInner( base::Time query_start_time, std::string query_text, std::map additional_query_parameters, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, lens::LensOverlayInvocationSource invocation_source); // Process the bitmap and creates all necessary data to initialize the // overlay. Happens on a separate thread to prevent main thread from hanging. void CreateInitializationData(const SkBitmap& screenshot, const std::vector& all_bounds, std::optional pdf_current_page); // Called after creating the RGB bitmap and we are back on the main thread. void ContinueCreateInitializationData( const SkBitmap& screenshot, const std::vector& all_bounds, std::optional pdf_current_page, base::TimeTicks screenshot_bitmap_start_time, SkBitmap rgb_screenshot); // Stores the page content and continues the initialization process. Also // records the page count for PDF. void StorePageContentAndContinueInitialization( std::unique_ptr initialization_data, std::optional page_context_start_time, std::vector page_contents, lens::MimeType primary_content_type, std::optional page_count); // Creates the mojo bounding boxes for the significant regions. std::vector ConvertSignificantRegionBoxes( const std::vector& all_bounds); // Updates state of the ghost loader. |suppress_ghost_loader| is true when // the page bytes can't be uploaded. void SuppressGhostLoader(); // Requests to open the side panel if this class has not already done so. // Must be called before issuing results to the side panel. void MaybeOpenSidePanel(); // Initializes all parts of our UI and starts the query flow. // Runs once the overlay WebUI and initialization data are both ready. // Once initialization_data is ready, it should be passed to this method to be // cached until all parts of the flow are ready. Parts of the initialization // flow (like creating WebUI) that do not touch initialization_data should // pass initialization_data as nullptr. void InitializeOverlay( std::unique_ptr initialization_data); // Initializes the overlay UI after it has been created with data fetched // before its creation. void InitializeOverlayUI(const OverlayInitializationData& init_data); // Returns true if the searchbox is a CONTEXTUAL_SEARCHBOX. bool IsContextualSearchbox(); // Returns true if the current query flow is in region-only mode. bool IsSelectedRegionOnlyMode(); // OverlayBaseController overrides: void RequestSyncClose(DismissalSource source) override; GURL GetInitialURL() override; void NotifyIsOverlayShowing(bool is_showing) override; int GetToolResourceId() override; ui::ElementIdentifier GetViewContainerId() const override; SidePanelType GetSidePanelType() override; bool ShouldCloseSidePanel() override; void StartScreenshotFlow() override; void FinishedWaitingForReflow(base::TimeTicks reflow_start_time) override; bool ShouldShowPreselectionBubble() override; void ShowPreselectionBubble() override; bool UseOverlayBlur() override; void NotifyPageNavigated() override; void NotifyOverlayClosing() override; void NotifyTabForegrounded() override; void NotifyTabWillEnterBackground() override; PreselectionUIConfig GetPreselectionBubbleConfig() override; bool IsOverlayViewShared() const override; // content::WebContentsDelegate: bool HandleContextMenu(content::RenderFrameHost& render_frame_host, const content::ContextMenuParams& params) override; bool HandleKeyboardEvent(content::WebContents* source, const input::NativeWebKeyboardEvent& event) override; // Called when the browser window's fullscreen state changes. void OnFullscreenStateChanged(); // OmniboxTabHelper::Observer: void OnOmniboxInputStateChanged() override {} void OnOmniboxInputInProgress(bool in_progress) override {} void OnOmniboxFocusChanged(OmniboxFocusState state, OmniboxFocusChangeReason reason) override; void OnOmniboxPopupVisibilityChanged(bool popup_is_open) override {} // find_in_page::FindResultObserver: void OnFindEmptyText(content::WebContents* web_contents) override; void OnFindResultAvailable(content::WebContents* web_contents) override; // Called when the Lens backend handshake is complete. void OnHandshakeComplete(); // Gets the page classification from the searchbox controller. metrics::OmniboxEventProto::PageClassification GetPageClassification() const; // Adds a callback to be called when the Lens backend handshake is finished. // If the handshake is already finished, the callback will be called // immediately. void OnLensBackendHandshakeFinished(base::OnceClosure callback); // Gets the ui scale factor of the page. float GetUiScaleFactor(); // Suggest a name for the save as image feature incorporating the hostname of // the page. Protocol, TLD, etc are not taken into consideration. Duplicate // names get automatic suffixes. static const std::u16string GetFilenameForURL(const GURL& url); // lens::mojom::LensPageHandler overrides. void ActivityRequestedByOverlay( ui::mojom::ClickModifiersPtr click_modifiers) override; void AddBackgroundBlur() override; void SetLiveBlur(bool enabled) override; void ClosePreselectionBubble() override; void CloseRequestedByOverlayCloseButton() override; void CloseRequestedByOverlayBackgroundClick() override; void CopyImage(lens::mojom::CenterRotatedBoxPtr region) override; void CopyText(const std::string& text) override; void FeedbackRequestedByOverlay() override; void GetOverlayInvocationSource( GetOverlayInvocationSourceCallback callback) override; void InfoRequestedByOverlay( ui::mojom::ClickModifiersPtr click_modifiers) override; void IssueLensObjectRequest(lens::mojom::CenterRotatedBoxPtr region, bool is_mask_click) override; void IssueLensRegionRequest(lens::mojom::CenterRotatedBoxPtr region, bool is_click) override; void IssueTextSelectionRequest(const std::string& text_query, int selection_start_index, int selection_end_index, bool is_translate) override; void IssueTranslateFullPageRequest( const std::string& source_language, const std::string& target_language) override; void IssueEndTranslateModeRequest() override; void IssueTranslateSelectionRequest(const std::string& text_query, const std::string& content_language, int selection_start_index, int selection_end_index) override; void IssueMathSelectionRequest(const std::string& query, const std::string& formula, int selection_start_index, int selection_end_index) override; void NotifyOverlayInitialized() override; void RecordUkmAndTaskCompletionForLensOverlayInteraction( lens::mojom::UserAction user_action) override; void RecordLensOverlaySemanticEvent( lens::mojom::SemanticEvent event) override; void SaveAsImage(lens::mojom::CenterRotatedBoxPtr region) override; void MaybeShowTranslateFeaturePromo() override; void MaybeCloseTranslateFeaturePromo(bool feature_engaged) override; void FetchSupportedLanguages( FetchSupportedLanguagesCallback callback) override; void FinishReshowOverlay() override; void AcceptPrivacyNotice() override; void DismissPrivacyNotice() override; // content::WebContentsObserver: void DidFinishNavigation( content::NavigationHandle* navigation_handle) override; // Tries to show the translate feature promo after the translate button // element is shown. void TryShowTranslateFeaturePromo(ui::TrackedElement* element); // Performs shared logic for IssueTextSelectionRequest() and // IssueTranslateSelectionRequest(). void IssueTextSelectionRequestInner(base::Time query_start_time, const std::string& text_query, int selection_start_index, int selection_end_index); // Handles a request (either region or multimodal) trigger by sending // the request to the query controller. void IssueSearchBoxRequestPart2( base::Time query_start_time, const std::string& search_box_text, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, std::map additional_query_params, std::optional invocation_source); // Launches the Lens overlay HaTS survey if eligible. void MaybeLaunchSurvey(); // Initialize the tutorial IPH URL matcher from finch config. void InitializeTutorialIPHUrlMatcher(); // Returns true if tutorial IPH is eligible to be shown for the given URL. bool IsUrlEligibleForTutorialIPH(const GURL& url); // Shows the tutorial IPH. void ShowTutorialIPH(); // Notifies the user education service that the overlay has been used. void NotifyUserEducationAboutOverlayUsed(); // Notifies the overlay or side panel that the page content type has changed. void NotifyPageContentUpdated(); // Notifies the entry point controller to update the state of the entry // points since the state of the overlay has changed. void UpdateEntryPointsState(); // Callback to run when the partial page text is retrieved from the PDF. void OnPdfPartialPageTextRetrieved( std::vector pdf_pages_text); // Callback to run when the page context has been updated and the suggestion // query should now be issued. void OnPageContextUpdatedForSuggestion( base::Time query_start_time, std::string query_text, std::map additional_query_parameters, AutocompleteMatchType::Type match_type, bool is_zero_prefix_suggestion, lens::LensOverlayInvocationSource invocation_source); // Called by LensSearchContextualizationController after taking a screenshot. void OnScreenshotTaken(base::TimeTicks screenshot_start_time, const SkBitmap& bitmap, const std::vector& all_bounds, std::optional pdf_current_page); // Part 2 of reshowing the overlay. Called after the screenshot and page // context has been updated. void ReshowOverlayPart2(); // Part 3 of reshowing the overlay. Called after the RGB bitmap has been // created. void ReshowOverlayPart3(SkBitmap rgb_bitmap); // Starts the query flow. void StartQueryFlow(); // Fetches the partial PDF text if the page is a PDF. void FetchPdfTextIfEligible(); // For the current session only, grants the permissions needed for // contextualization if the non-blocking privacy notice is being used and the // permissions have not already been permanently granted. virtual void MaybeGrantLensOverlayPermissionsForSession( std::optional invocation_source); static lens::LensOverlayDismissalSource ConvertDismissalSource( DismissalSource dismissal_source); // Shorthand to grab the LensSearchboxController for this instance of Lens. lens::LensSearchboxController* GetLensSearchboxController(); // Shorthand to grab the LensOverlaySidePanelCoordinator for this instance of // Lens. lens::LensOverlaySidePanelCoordinator* GetLensOverlaySidePanelCoordinator(); // Shorthand to grab the LensResultsPanelRouter for this instance of Lens. lens::LensResultsPanelRouter* GetLensResultsPanelRouter(); // Shorthand to grab the LensOverlayQueryController for this instance of Lens. lens::LensOverlayQueryController* GetLensOverlayQueryController(); // Shorthand to grab the LensQueryFlowRouter for this instance of Lens. lens::LensQueryFlowRouter* GetLensQueryFlowRouter(); // Shorthand to grab the LensSearchContextualizationController for this // instance of Lens. lens::LensSearchContextualizationController* GetContextualizationController(); // Shorthand to grab the LensSessionMetricsLogger for this instance of Lens. lens::LensSessionMetricsLogger* GetLensSessionMetricsLogger(); // Owns this class. raw_ptr lens_search_controller_; // The assembly data needed for the overlay to be created and shown. std::unique_ptr initialization_data_; // Invocation source for the lens overlay. lens::LensOverlayInvocationSource invocation_source_ = lens::LensOverlayInvocationSource::kAppMenu; // A contextual search request to be issued once the overlay is initialized. base::OnceClosure pending_contextual_search_request_; // Pending region to search after the overlay loads. lens::mojom::CenterRotatedBoxPtr pending_region_; // The bitmap for the pending region stored in pending_region_. // pending_region_ and pending_region_bitmap_ are correlated and their // lifecycles are should stay in sync. SkBitmap pending_region_bitmap_; // The selection type of the current Lens request. If the // user is not currently viewing results for a Lens query, this will be // set to UNKNOWN_SELECTION_TYPE. lens::LensOverlaySelectionType lens_selection_type_ = lens::UNKNOWN_SELECTION_TYPE; // Connections to and from the overlay WebUI. Only valid while // `overlay_view_` is showing, and after the WebUI has started executing JS // and has bound the connection. mojo::Receiver receiver_{this}; mojo::Remote page_; // Whether the OCR DOM similarity has been recorded in the current session. bool ocr_dom_similarity_recorded_in_session_ = false; // The time at which the overlay was invoked. Used to compute timing metrics. base::TimeTicks invocation_time_; // The time at which the overlay was invoked, since epoch. Used to calculate // timeToWebUIReady on the WebUI side. base::Time invocation_time_since_epoch_; // The time at which the webUI binding was invoked. Used to compute timing // metrics. base::TimeTicks invocation_time_for_webui_binding_; // Indicates whether this is the first upload handler event received. This is // used to determine whether to show the upload progress bar. bool is_first_upload_handler_event_ = true; // Indicates whether the upload progress bar is currently being shown for this // upload. bool is_upload_progress_bar_shown_ = true; // Indicates whether the user is currently on a context eligible page. bool is_page_context_eligible_ = true; // TODO(384778180): The two `pre_initialization_*` fields below are used to // store data that came back before the initialization data was ready. This // should be refactored into one struct to make it cleaner. // // The stored objects response to be attached to the initialization data // if the object response came back before the initialization data was ready. std::optional> pre_initialization_objects_; // The stored text response to be attached to the initialization data // if the text response came back before the initialization data was ready. std::optional pre_initialization_text_; // The callback subscription for the element shown callback used to show the // translate feature promo. base::CallbackListSubscription translate_button_shown_subscription_; // Matcher for URLs that are eligible to have the tutorial IPH shown. std::unique_ptr tutorial_iph_url_matcher_; // Matcher for URLs that are do not need to pass the check for allowed paths. // Instead, if they match the tutorial_iph_url_matcher_` and do not contain // any of the blocked paths, they are considered matches. std::unique_ptr forced_url_matcher_; // Matcher for URL paths that are eligible to have the tutorial IPH shown. std::unique_ptr page_path_allow_matcher_; // Matcher for URL paths that are not eligible to have the tutorial IPH shown. std::unique_ptr page_path_block_matcher_; // Filters used by the URL matcher. Used to look up if a matching filter is an // allow filter or a block filter. std::map iph_url_filters_; // Used to cancel showing a queued tutorial IPH. base::OneShotTimer tutorial_iph_timer_; // ---------------Browser window scoped state: START--------------------- // State that is scoped to the browser window must be reset when the tab is // backgrounded, since the tab may move between browser windows. // Subscription to be notified when the browser window enters fullscreen. base::CallbackListSubscription fullscreen_subscription_; // Observer to check if the user is using CTRL/CMD+F while the overlay is // open. base::ScopedObservation find_tab_observer_{this}; base::ScopedObservation omnibox_tab_helper_observer_{this}; // The controller for sending gen204 pings. Owned by the overlay controller // so that the life cycle outlasts the query controller, allowing gen204 // requests to be sent upon query end. std::unique_ptr gen204_controller_; // The controller for sending requests to get the list of supported languages. // Requests are only made if the WebUI has not already cached the languages // and none of the update cache conditions are met. std::unique_ptr languages_controller_; // Whether to use AIM for visual searches. bool use_aim_for_visual_search_ = false; // Whether the user performed an interaction without accepting the privacy // notice. bool user_interacted_without_accepting_privacy_notice = false; // --------------------Browser window scoped state: END--------------------- // Must be the last member. base::WeakPtrFactory weak_factory_{this}; }; #endif // CHROME_BROWSER_UI_LENS_LENS_OVERLAY_CONTROLLER_H_