// Copyright 2016 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_COMPOSITOR_FRAME_SINK_SUPPORT_H_ #define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_COMPOSITOR_FRAME_SINK_SUPPORT_H_ #include #include #include #include #include #include "base/containers/circular_deque.h" #include "base/containers/flat_set.h" #include "base/functional/callback.h" #include "base/memory/raw_ptr.h" #include "base/memory/read_only_shared_memory_region.h" #include "base/memory/weak_ptr.h" #include "base/time/time.h" #include "components/viz/common/frame_sinks/begin_frame_source.h" #include "components/viz/common/frame_timing_details_map.h" #include "components/viz/common/quads/compositor_frame.h" #include "components/viz/common/surfaces/frame_sink_bundle_id.h" #include "components/viz/common/surfaces/region_capture_bounds.h" #include "components/viz/common/surfaces/surface_info.h" #include "components/viz/common/surfaces/surface_range.h" #include "components/viz/common/surfaces/video_capture_target.h" #include "components/viz/service/frame_sinks/begin_frame_tracker.h" #include "components/viz/service/frame_sinks/frame_sink_throttler.h" #include "components/viz/service/frame_sinks/surface_resource_holder.h" #include "components/viz/service/frame_sinks/surface_resource_holder_client.h" #include "components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h" #include "components/viz/service/hit_test/hit_test_aggregator.h" #include "components/viz/service/surfaces/frame_index_constants.h" #include "components/viz/service/surfaces/surface_client.h" #include "components/viz/service/surfaces/surface_observer.h" #include "components/viz/service/transitions/surface_animation_manager.h" #include "components/viz/service/viz_service_export.h" #include "services/viz/public/mojom/compositing/compositor_frame_sink.mojom.h" #include "services/viz/public/mojom/compositing/layer_context.mojom.h" #include "services/viz/public/mojom/hit_test/hit_test_region_list.mojom.h" namespace viz { class FrameSinkManagerImpl; class LatestLocalSurfaceIdLookupDelegate; class LayerContextImpl; class Surface; class SurfaceManager; // Possible outcomes of MaybeSubmitCompositorFrame(). // These values were previously persisted to logs. Entries should not be // renumbered and numeric values should never be reused. enum class SubmitResult { ACCEPTED = 0, COPY_OUTPUT_REQUESTS_NOT_ALLOWED = 1, // SURFACE_INVARIANTS_VIOLATION = 2 is deprecated. SIZE_MISMATCH = 3, SURFACE_ID_DECREASED = 4, SURFACE_OWNED_BY_ANOTHER_CLIENT = 5, HIT_TEST_DATA_INVALID = 6, INVALID_FRAME = 7, INVALID_DISPLAY_TRANSFORM = 8, INVALID_BEGIN_FRAME_ACK = 9, // Magic constant used by the histogram macros. kMaxValue = INVALID_BEGIN_FRAME_ACK, }; class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport : public BeginFrameObserver, public SurfaceResourceHolderClient, public SurfaceClient, public CapturableFrameSink { public: using AggregatedDamageCallback = base::RepeatingCallback; // Determines maximum number of allowed undrawn frames. Once this limit is // exceeded, we throttle sBeginFrames to 1 per second. Limit must be at least // 1, as the relative ordering of renderer / browser frame submissions allows // us to have one outstanding undrawn frame under normal operation. static constexpr uint32_t kUndrawnFrameLimit = 3; CompositorFrameSinkSupport(mojom::CompositorFrameSinkClient* client, FrameSinkManagerImpl* frame_sink_manager, const FrameSinkId& frame_sink_id, bool is_root); CompositorFrameSinkSupport(const CompositorFrameSinkSupport&) = delete; CompositorFrameSinkSupport& operator=(const CompositorFrameSinkSupport&) = delete; ~CompositorFrameSinkSupport() override; const FrameSinkId& frame_sink_id() const { return frame_sink_id_; } const SurfaceId& last_activated_surface_id() const { return last_activated_surface_id_; } const LocalSurfaceId& last_activated_local_surface_id() const { return last_activated_surface_id_.local_surface_id(); } const SurfaceId& last_created_surface_id() const { return last_created_surface_id_; } bool is_root() const { return is_root_; } FrameSinkManagerImpl* frame_sink_manager() { return frame_sink_manager_; } BeginFrameSource* begin_frame_source() { return begin_frame_source_; } void set_resource_return_delegate(LayerContextImpl* delegate) { resource_return_delegate_ = delegate; } LayerContextImpl* resource_return_delegate() const { return resource_return_delegate_; } base::WeakPtr GetWeakPtr() { return weak_factory_.GetWeakPtr(); } const FrameTimingDetailsMap& timing_details() { return frame_timing_details_; } bool needs_begin_frame() const { return needs_begin_frame_; } [[nodiscard]] FrameTimingDetailsMap TakeFrameTimingDetailsMap(); // Viz hit-test setup is only called when |is_root_| is true (except on // android webview). void SetUpHitTest( LatestLocalSurfaceIdLookupDelegate* local_surface_id_lookup_delegate); // The provided callback will be run every time a surface owned by this object // or one of its descendents is determined to be damaged at aggregation time. void SetAggregatedDamageCallbackForTesting(AggregatedDamageCallback callback); // This allows the FrameSinkManagerImpl to pass a BeginFrameSource to use. void SetBeginFrameSource(BeginFrameSource* begin_frame_source); // Sets the ID of the FrameSinkBundle to which this sink should belong. If the // sink is incompatible with the bundle (i.e. uses a different // BeginFrameSource than this CompositorFrameSinkSupport) then the sink will // be removed from the bundle and destroyed asynchronously, disconnecting its // client. void SetBundle(const FrameSinkBundleId& bundle_id); void BindLayerContext(mojom::PendingLayerContext& context, mojom::LayerContextSettingsPtr settings); void SetThreads(bool from_untrusted_client, std::vector unverified_threads); FrameSinkThrottler& GetThrottlerForTesting() { return throttler_; } const FrameSinkThrottler& GetThrottlerForTesting() const { return throttler_; } // Sets a throttling interval to be used. If |interval| is zero, any // previously set interval is cleared. Note that the frame sink may still be // throttled by other signals (e.g. cadence) unless SetAllowThrottling(false) // is called. void SetThrottleInterval(base::TimeDelta interval); // If |allowed| is false, the begin frame interval will be base::TimeDelta() // regardless of any other throttling. void SetAllowThrottling(bool allowed); // If other clients are interactive, reduce frame cadence if `throttled`. // SurfaceClient implementation. void OnSurfaceCommitted(Surface* surface) override; void OnSurfaceActivated(Surface* surface) override; void OnSurfaceDestroyed(Surface* surface) override; void OnSurfaceWillDraw(Surface* surface) override; void RefResources( const std::vector& resources) override; void UnrefResources(std::vector resources) override; void ReturnResources(std::vector resources) override; void ReceiveFromChild( const std::vector& resources) override; // Takes the CopyOutputRequests that were requested for a surface with at // most |local_surface_id|. std::vector> TakeCopyOutputRequests( const LocalSurfaceId& local_surface_id) override; void OnFrameTokenChanged(uint32_t frame_token) override; void SendCompositorFrameAck() override; void OnSurfaceAggregatedDamage( Surface* surface, const LocalSurfaceId& local_surface_id, const CompositorFrame& frame, const gfx::Rect& damage_rect, base::TimeTicks expected_display_time) override; void OnSurfacePresented(uint32_t frame_token, base::TimeTicks draw_start_timestamp, const gfx::SwapTimings& swap_timings, const gfx::PresentationFeedback& feedback) override; bool IsVideoCaptureStarted() override; std::vector GetThreads() override; // mojom::CompositorFrameSink helpers. void SetNeedsBeginFrame(bool needs_begin_frame); void SetWantsAnimateOnlyBeginFrames(); void SetAutoNeedsBeginFrame(); void SetNoCompositorFrameAcks(); bool DidNotProduceFrame(const BeginFrameAck& ack); void SubmitCompositorFrame( const LocalSurfaceId& local_surface_id, CompositorFrame frame, std::optional hit_test_region_list, uint64_t submit_time); void SubmitCompositorFrame(const LocalSurfaceId& local_surface_id, CompositorFrame frame) { SubmitCompositorFrame(local_surface_id, std::move(frame), std::nullopt, 0); } void SubmitCompositorFrame( const LocalSurfaceId& local_surface_id, CompositorFrame frame, std::optional hit_test_region_list) { SubmitCompositorFrame(local_surface_id, std::move(frame), std::move(hit_test_region_list), 0); } void NotifyNewLocalSurfaceIdExpectedWhilePaused(); // Mark |id| and all surfaces with smaller ids for destruction. Note that |id| // doesn't have to exist at the time of calling. void EvictSurface(const LocalSurfaceId& id); // Attempts to submit a new CompositorFrame to |local_surface_id| and returns // whether the frame was accepted or the reason why it was rejected. If // |local_surface_id| hasn't been submitted before then a new Surface will be // created for it. // // This is called by SubmitCompositorFrame(), which DCHECK-fails on a // non-accepted result. Prefer calling SubmitCompositorFrame() instead of this // method unless the result value affects what the caller will do next. virtual SubmitResult MaybeSubmitCompositorFrame( const LocalSurfaceId& local_surface_id, CompositorFrame frame, std::optional hit_test_region_list, uint64_t submit_time); // CapturableFrameSink implementation. const FrameSinkId& GetFrameSinkId() const override; void AttachCaptureClient(CapturableFrameSink::Client* client) override; void DetachCaptureClient(CapturableFrameSink::Client* client) override; std::optional GetRequestRegionProperties( const VideoCaptureSubTarget& sub_target) const override; void OnClientCaptureStarted() override; void OnClientCaptureStopped() override; void RequestCopyOfOutput(std::unique_ptr pending_copy_output_request) override; const CompositorFrameMetadata* GetLastActivatedFrameMetadata() override; HitTestAggregator* GetHitTestAggregator(); // Permits submitted CompositorFrames to contain CopyOutputRequests, for // special-case testing purposes only. void set_allow_copy_output_requests_for_testing() { allow_copy_output_requests_ = true; } Surface* GetLastCreatedSurfaceForTesting(); // Maps the |result| from MaybeSubmitCompositorFrame() to a human-readable // string. static const char* GetSubmitResultAsString(SubmitResult result); const std::vector>& copy_output_requests_for_testing() const { return copy_output_requests_; } bool IsEvicted(const LocalSurfaceId& local_surface_id) const; // Clears `copy_output_requests_`. Should be called when the client or service // is shutting down. The requests demanding an exact `LocalSurfaceid` match // will be transferred to the corresponding `Surface`s void ClearAllPendingCopyOutputRequests(); const RegionCaptureBounds& current_capture_bounds() const { return current_capture_bounds_; } LayerContextImpl* layer_context_for_testing() { return layer_context_.get(); } // Subscribes or unsubscribes `layer_context_` to subsequent BeginFrames. void SetLayerContextWantsBeginFrames(bool wants_begin_frames); void RegisterSurfaceAnimationManagerNotification( base::OnceCallback callback); private: friend class AckOnSurfaceActivationWhenInteractiveTest; friend class CompositorFrameSinkSupportTestBase; friend class DisplayTest; friend class FrameSinkManagerTest; friend class LayerContextImpl; friend class SurfaceAggregatorWithResourcesTest; // Creates a surface reference from the top-level root to |surface_id|. SurfaceReference MakeTopLevelRootReference(const SurfaceId& surface_id); void ProcessCompositorFrameTransitionDirective( const CompositorFrameTransitionDirective& directive, Surface* surface); void OnSaveTransitionDirectiveProcessed( const CompositorFrameTransitionDirective& directive); void DidReceiveCompositorFrameAck(); void DidPresentCompositorFrame(uint32_t frame_token, base::TimeTicks draw_start_timestamp, const gfx::SwapTimings& swap_timings, const gfx::PresentationFeedback& feedback); void DidRejectCompositorFrame( uint32_t frame_token, std::vector frame_resource_list, std::vector latency_info); // Update the display root reference with |surface|. void UpdateDisplayRootReference(const Surface* surface); // BeginFrameObserver implementation. void OnBeginFrame(const BeginFrameArgs& args) override; const BeginFrameArgs& LastUsedBeginFrameArgs() const override; void OnBeginFrameSourcePausedChanged(bool paused) override; bool WantsAnimateOnlyBeginFrames() const override; void UpdateNeedsBeginFramesInternal(); void StartObservingBeginFrameSource(); void StopObservingBeginFrameSource(); void MaybeEvictSurfaces(); void EvictLastActiveSurface(); bool ShouldSendBeginFrame(BeginFrameId frame_id, base::TimeTicks timestamp, base::TimeDelta vsync_interval); // Checks if any of the pending surfaces should activate now because their // deadline has passed. This is called every BeginFrame. void CheckPendingSurfaces(); // When throttling is requested by a client, a BeginFrame will not be sent // until the time elapsed has passed the requested throttle interval since the // last sent BeginFrame. This function returns true if such interval has // passed and a BeginFrame should be sent. bool ShouldThrottleBeginFrameAsRequested(base::TimeTicks frame_time, base::TimeDelta vsync_interval); // Instructs the FrameSinkManager to destroy our CompositorFrameSinkImpl. // To avoid reentrancy issues, this should be called from its own task. void DestroySelf(); // Posts a task to invoke DestroySelf() ASAP. void ScheduleSelfDestruction(); void UpdateThreadIdsPostVerification(std::vector threads, bool passed_verification); void DoReturnResources(std::vector resources); base::TimeDelta begin_frame_interval() const; const raw_ptr client_; const raw_ptr frame_sink_manager_; const raw_ptr surface_manager_; raw_ptr resource_return_delegate_ = nullptr; const FrameSinkId frame_sink_id_; SurfaceId last_activated_surface_id_; SurfaceId last_created_surface_id_; // If this contains a value then a surface reference from the top-level root // to SurfaceId(frame_sink_id_, referenced_local_surface_id_.value()) was // added. This will not contain a value if |is_root_| is false. std::optional referenced_local_surface_id_; SurfaceResourceHolder surface_resource_holder_; // This has a HitTestAggregator if and only if |is_root_| is true. std::unique_ptr hit_test_aggregator_; // Keeps track of CompositorFrames that have been submitted and have not // yet received an ACK from their Surface. int pending_frames_ = 0; std::vector surface_returned_resources_; // The begin frame source being observered. Null if none. raw_ptr begin_frame_source_ = nullptr; // The last begin frame args generated by the begin frame source. BeginFrameArgs last_begin_frame_args_; // Whether a request for begin frames has been issued. bool client_needs_begin_frame_ = false; bool handling_auto_needs_begin_frame_ = false; // Whether the sink currently needs begin frames for any reason. bool needs_begin_frame_ = false; // Whether or not a frame observer has been added. bool added_frame_observer_ = false; bool wants_animate_only_begin_frames_ = false; bool auto_needs_begin_frame_ = false; bool no_compositor_frame_acks_ = false; // Indicates the FrameSinkBundle to which this sink belongs, if any. std::optional bundle_id_; const bool is_root_; // Device scale factor associated with this CompositorFrameSinkSupport. // Default value is 1.0. float device_scale_factor_ = 1.0; bool is_mobile_optimized_ = false; // By default, this is equivalent to |is_root_|, but may be overridden for // testing. Generally, for non-roots, there must not be any CopyOutputRequests // contained within submitted CompositorFrames. Otherwise, unprivileged // clients would be able to capture content for which they are not authorized. bool allow_copy_output_requests_; // Used for tests only. AggregatedDamageCallback aggregated_damage_callback_; uint32_t last_frame_index_ = kFrameIndexStart - 1; // The video capture clients hooking into this instance to observe frame // begins and damage, and then make CopyOutputRequests on the appropriate // frames. std::vector> capture_clients_; // These are the CopyOutputRequests made on the frame sink (as opposed to // being included as a part of a CompositorFrame). They stay here until a // Surface with a LocalSurfaceId which is at least the stored LocalSurfaceId // takes them. For example, for a stored PendingCopyOutputRequest, a surface // with LocalSurfaceId >= PendingCopyOutputRequest::local_surface_id will take // it, but a surface with LocalSurfaceId < // PendingCopyOutputRequest::local_surface_id will not. Note that if the // PendingCopyOutputRequest::local_surface_id is default initialized, then the // next surface will take it regardless of its LocalSurfaceId. std::vector> copy_output_requests_; bool callback_received_begin_frame_ = true; bool callback_received_receive_ack_ = true; uint32_t trace_sequence_ = 0; BeginFrameTracker begin_frame_tracker_; // Keeps track of the timestamps for a specific frame's submission and // embedding. Observes the surface if the frame hasn't been embedded yet. class PendingFrameDetails : SurfaceObserver { public: PendingFrameDetails(base::TimeTicks frame_submit_timestamp, TreesInVizTiming frame_timing_details, SurfaceManager* surface_manager); ~PendingFrameDetails() override; // Sets the surface ID and adds/observes the embedded timestamp for the // surface, to set `frame_embed_timestamp_`. void set_surface_id(SurfaceId surface_id); void set_frame_id(BeginFrameId frame_id); base::TimeTicks frame_submit_timestamp() const { return frame_submit_timestamp_; } base::TimeTicks start_update_display_tree() const { return trees_in_viz_timing_details_.start_update_display_tree; } base::TimeTicks start_prepare_to_draw() const { return trees_in_viz_timing_details_.start_prepare_to_draw; } base::TimeTicks start_draw_layers() const { return trees_in_viz_timing_details_.start_draw_layers; } base::TimeTicks submit_compositor_frame() const { return trees_in_viz_timing_details_.submit_compositor_frame; } base::TimeTicks frame_embed_timestamp() const { return frame_embed_timestamp_; } BeginFrameId frame_id() const { return frame_id_; } private: void OnAddedSurfaceReference(const SurfaceId& parent_id, const SurfaceId& child_id) override; // Either sets `frame_embed_timestamp_` to the time the surface with ID // `surface_id_` is embedded (if it's embedded already), or starts // observing `OnAddedSurfaceReference()` to watch and set the timestamp // for when that surface gets embedded later. void SetOrObserveFrameEmbedTimeStamp(); const base::TimeTicks frame_submit_timestamp_; // TreesInViz related timestamps. const TreesInVizTiming trees_in_viz_timing_details_; base::TimeTicks frame_embed_timestamp_; // The surface ID that is associated with the frame. SurfaceId surface_id_; const raw_ptr surface_manager_; // This frame id maps to BeginFrameId during compositor frame submission in // order to preserve frame submission info across atypical rendering flows. // Values will be propagated through presentation promise feedback to // renderer main thread, and being useful there as an indicator for // unreliable presentation time. BeginFrameId frame_id_; }; // Maps |frame_token| to the timestamps when that frame was received and // embedded. These timestamps will be combined with the information received // in OnSurfacePresented() and stored in |frame_timing_details_|. base::flat_map> pending_received_frame_times_; FrameTimingDetailsMap frame_timing_details_; LocalSurfaceId last_evicted_local_surface_id_; base::TimeTicks last_frame_time_; // Initialize |last_drawn_frame_index_| as though the frame before the first // has been drawn. static_assert(kFrameIndexStart >= 1, "|last_drawn_frame_index| relies on kFrameIndexStart >= 1"); uint32_t last_drawn_frame_index_ = kFrameIndexStart - 1; FrameSinkThrottler throttler_; // The set of surfaces owned by this frame sink that have pending frame. base::flat_set> pending_surfaces_; // This is the last known frame interval for this sink used to decide // when to throttle begin frames. base::TimeDelta last_known_frame_interval_ = BeginFrameArgs::MinInterval(); base::flat_map> view_transition_token_to_animation_manager_; std::vector threads_; // Number of frames skipped during throttling since last BeginFrame sent. uint64_t frames_throttled_since_last_ = 0; // Number of clients that have started video capturing. uint32_t number_clients_capturing_ = 0; // Region capture bounds associated with the last surface that was aggregated. RegionCaptureBounds current_capture_bounds_; // When TreesInViz is enabled, this owns the display tree and forwards its // submitted compositor frames directly to `this`. std::unique_ptr layer_context_; bool layer_context_wants_begin_frames_ = false; // If exists, then we need to notify this callback when we have a // SurfaceAnimationManager. base::OnceCallback surface_animation_manager_callback_; base::WeakPtrFactory weak_factory_{this}; }; } // namespace viz #endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_COMPOSITOR_FRAME_SINK_SUPPORT_H_