// Copyright 2020 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "third_party/blink/renderer/modules/webcodecs/video_encoder.h" #include #include #include "base/format_macros.h" #include "base/functional/bind.h" #include "base/functional/callback.h" #include "base/functional/callback_helpers.h" #include "base/logging.h" #include "base/metrics/histogram_functions.h" #include "base/notimplemented.h" #include "base/numerics/clamped_math.h" #include "base/task/single_thread_task_runner.h" #include "base/time/time.h" #include "base/trace_event/common/trace_event_common.h" #include "base/trace_event/trace_event.h" #include "components/viz/common/gpu/raster_context_provider.h" #include "gpu/GLES2/gl2extchromium.h" #include "gpu/command_buffer/client/raster_interface.h" #include "media/base/async_destroy_video_encoder.h" #include "media/base/limits.h" #include "media/base/media_log.h" #include "media/base/media_util.h" #include "media/base/mime_util.h" #include "media/base/supported_types.h" #include "media/base/svc_scalability_mode.h" #include "media/base/timestamp_constants.h" #include "media/base/video_codecs.h" #include "media/base/video_color_space.h" #include "media/base/video_encoder.h" #include "media/base/video_util.h" #include "media/media_buildflags.h" #include "media/mojo/clients/mojo_video_encoder_metrics_provider.h" #include "media/parsers/h264_level_limits.h" #include "media/video/gpu_video_accelerator_factories.h" #include "media/video/offloading_video_encoder.h" #include "media/video/video_encode_accelerator_adapter.h" #include "media/video/video_encoder_fallback.h" #include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h" #include "third_party/blink/public/mojom/use_counter/metrics/web_feature.mojom-blink.h" #include "third_party/blink/public/platform/platform.h" #include "third_party/blink/public/platform/task_type.h" #include "third_party/blink/renderer/bindings/core/v8/native_value_traits_impl.h" #include "third_party/blink/renderer/bindings/core/v8/script_function.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h" #include "third_party/blink/renderer/bindings/core/v8/to_v8_traits.h" #include "third_party/blink/renderer/bindings/core/v8/v8_dom_exception.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_avc_encoder_config.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_chunk_metadata.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_hevc_encoder_config.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_svc_output_metadata.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_color_space_init.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_config.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options_for_av_1.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options_for_avc.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options_for_hevc.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options_for_vp_9.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_init.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_support.h" #include "third_party/blink/renderer/bindings/modules/v8/v8_video_pixel_format.h" #include "third_party/blink/renderer/core/dom/dom_exception.h" #include "third_party/blink/renderer/core/frame/local_dom_window.h" #include "third_party/blink/renderer/core/frame/local_frame.h" #include "third_party/blink/renderer/core/keywords.h" #include "third_party/blink/renderer/core/streams/readable_stream.h" #include "third_party/blink/renderer/core/streams/writable_stream.h" #include "third_party/blink/renderer/modules/event_modules.h" #include "third_party/blink/renderer/modules/webcodecs/array_buffer_util.h" #include "third_party/blink/renderer/modules/webcodecs/background_readback.h" #include "third_party/blink/renderer/modules/webcodecs/codec_state_helper.h" #include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h" #include "third_party/blink/renderer/modules/webcodecs/gpu_factories_retriever.h" #include "third_party/blink/renderer/modules/webcodecs/video_color_space.h" #include "third_party/blink/renderer/modules/webcodecs/video_encoder_buffer.h" #include "third_party/blink/renderer/platform/bindings/enumeration_base.h" #include "third_party/blink/renderer/platform/bindings/exception_state.h" #include "third_party/blink/renderer/platform/bindings/script_state.h" #include "third_party/blink/renderer/platform/graphics/gpu/shared_gpu_context.h" #include "third_party/blink/renderer/platform/graphics/web_graphics_context_3d_video_frame_pool.h" #include "third_party/blink/renderer/platform/heap/cross_thread_handle.h" #include "third_party/blink/renderer/platform/heap/heap_barrier_callback.h" #include "third_party/blink/renderer/platform/heap/persistent.h" #include "third_party/blink/renderer/platform/instrumentation/use_counter.h" #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h" #include "third_party/blink/renderer/platform/scheduler/public/thread.h" #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_base.h" #include "third_party/blink/renderer/platform/wtf/cross_thread_copier_std.h" #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h" #include "third_party/blink/renderer/platform/wtf/text/format.h" #include "third_party/perfetto/include/perfetto/tracing/track.h" #if BUILDFLAG(ENABLE_LIBAOM) #include "media/video/av1_video_encoder.h" #endif #if BUILDFLAG(ENABLE_OPENH264) #include "media/video/openh264_video_encoder.h" #endif #if BUILDFLAG(ENABLE_LIBVPX) #include "media/video/vpx_video_encoder.h" #endif namespace blink { using EncoderType = media::VideoEncodeAccelerator::Config::EncoderType; namespace { constexpr const char kCategory[] = "media"; // Controls if VideoEncoder will use timestamp from blink::VideoFrame // instead of media::VideoFrame. BASE_FEATURE(kUseBlinkTimestampForEncoding, base::FEATURE_ENABLED_BY_DEFAULT); // TODO(crbug.com/40215121): This is very similar to the method in // video_frame.cc. It should probably be a function in video_types.cc. media::VideoPixelFormat ToOpaqueMediaPixelFormat(media::VideoPixelFormat fmt) { switch (fmt) { case media::PIXEL_FORMAT_I420A: return media::PIXEL_FORMAT_I420; case media::PIXEL_FORMAT_YUV420AP10: return media::PIXEL_FORMAT_YUV420P10; case media::PIXEL_FORMAT_I422A: return media::PIXEL_FORMAT_I422; case media::PIXEL_FORMAT_YUV422AP10: return media::PIXEL_FORMAT_YUV422P10; case media::PIXEL_FORMAT_I444A: return media::PIXEL_FORMAT_I444; case media::PIXEL_FORMAT_YUV444AP10: return media::PIXEL_FORMAT_YUV444P10; case media::PIXEL_FORMAT_NV12A: return media::PIXEL_FORMAT_NV12; default: NOTIMPLEMENTED() << "Missing support for making " << fmt << " opaque."; return fmt; } } int ComputeMaxActiveEncodes(std::optional frame_delay = std::nullopt, std::optional input_capacity = std::nullopt) { constexpr int kDefaultEncoderFrameDelay = 0; // The maximum number of input frames above the encoder frame delay that we // want to be able to enqueue in |media_encoder_|. constexpr int kDefaultEncoderExtraInputCapacity = 5; const int preferred_capacity = frame_delay.value_or(kDefaultEncoderFrameDelay) + kDefaultEncoderExtraInputCapacity; return input_capacity.has_value() ? std::min(preferred_capacity, input_capacity.value()) : preferred_capacity; } media::VideoEncodeAccelerator::SupportedRateControlMode BitrateToSupportedMode( const media::Bitrate& bitrate) { switch (bitrate.mode()) { case media::Bitrate::Mode::kConstant: return media::VideoEncodeAccelerator::kConstantMode; case media::Bitrate::Mode::kVariable: return media::VideoEncodeAccelerator::kVariableMode #if BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_CHROMEOS) // On Android and ChromeOS we allow CBR-only encoders to be used // for VBR because most devices don't properly advertise support // for VBR encoding. In most cases they will initialize // successfully when configured for VBR. | media::VideoEncodeAccelerator::kConstantMode #endif // BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_CHROMEOS) ; case media::Bitrate::Mode::kExternal: return media::VideoEncodeAccelerator::kExternalMode; } } media::EncoderStatus IsAcceleratedConfigurationSupported( media::VideoCodecProfile profile, const media::VideoEncoder::Options& options, media::GpuVideoAcceleratorFactories* gpu_factories, EncoderType required_encoder_type) { if (!gpu_factories || !gpu_factories->IsGpuVideoEncodeAcceleratorEnabled()) { return media::EncoderStatus::Codes::kEncoderAccelerationSupportMissing; } // Hardware encoders only support subsamplings other than 4:2:0 for AV1 // profile 1, where we require 4:4:4. // High bit depths are supported by HEVC Main10 and AV1 Main only; AV1 // profile 1 is deliberately left out because the 4:4:4 hardware input format // is 8 bit AYUV, and 10b lacks hardware support for now. // TODO(crbug.com/537818862): Advertise chroma_sampling and bit_depth on every // SupportedProfile and drop this hardcoded allowlist. if (profile != media::HEVCPROFILE_REXT) { media::VideoChromaSampling required_sampling = (profile == media::AV1PROFILE_PROFILE_HIGH) ? media::VideoChromaSampling::k444 : media::VideoChromaSampling::k420; const int bit_depth = options.bit_depth.value_or(8); const bool bit_depth_supported = bit_depth == 8 || (bit_depth == 10 && (profile == media::HEVCPROFILE_MAIN10 || profile == media::AV1PROFILE_PROFILE_MAIN)); if ((options.subsampling.has_value() && options.subsampling.value() != required_sampling) || !bit_depth_supported) { return media::EncoderStatus::Codes::kEncoderUnsupportedConfig; } } auto supported_profiles = gpu_factories->GetVideoEncodeAcceleratorSupportedProfiles().value_or( media::VideoEncodeAccelerator::SupportedProfiles()); if (supported_profiles.empty()) { return media::EncoderStatus::Codes::kEncoderAccelerationSupportMissing; } bool found_supported_profile = false; for (auto& supported_profile : supported_profiles) { if (supported_profile.profile != profile) { continue; } if (supported_profile.is_software_codec) { if (required_encoder_type == EncoderType::kHardware) { continue; } } else if (required_encoder_type == EncoderType::kSoftware) { continue; } if (supported_profile.min_resolution.width() > options.frame_size.width() || supported_profile.min_resolution.height() > options.frame_size.height()) { continue; } if (supported_profile.max_resolution.width() < options.frame_size.width() || supported_profile.max_resolution.height() < options.frame_size.height()) { continue; } double max_supported_framerate = static_cast(supported_profile.max_framerate_numerator) / supported_profile.max_framerate_denominator; if (options.framerate.has_value() && options.framerate.value() > max_supported_framerate) { continue; } if (options.scalability_mode.has_value() && !std::ranges::contains(supported_profile.scalability_modes, options.scalability_mode.value())) { continue; } if (options.bitrate.has_value()) { auto mode = BitrateToSupportedMode(options.bitrate.value()); if (!(mode & supported_profile.rate_control_modes)) { continue; } } if (supported_profile.chroma_sampling.has_value() && supported_profile.chroma_sampling.value() != options.subsampling.value_or(media::VideoChromaSampling::k420)) { continue; } if (supported_profile.bit_depth.has_value() && supported_profile.bit_depth.value() != options.bit_depth.value_or(8)) { continue; } found_supported_profile = true; break; } return found_supported_profile ? media::EncoderStatus::Codes::kOk : media::EncoderStatus::Codes::kEncoderUnsupportedConfig; } VideoEncoderTraits::ParsedConfig* ParseConfigStatic( const VideoEncoderConfig* config, ExceptionState& exception_state) { auto* result = MakeGarbageCollected(); if (config->codec().LengthWithStrippedWhiteSpace() == 0) { exception_state.ThrowTypeError("Invalid codec; codec is required."); return nullptr; } if (config->height() == 0 || config->width() == 0) { exception_state.ThrowTypeError( "Invalid size; height and width must be greater than zero."); return nullptr; } result->options.frame_size.SetSize(config->width(), config->height()); if (config->alpha() == V8AlphaOption::Enum::kKeep) { result->not_supported_error_message = "Alpha encoding is not currently supported."; return result; } result->options.latency_mode = (config->latencyMode() == V8LatencyMode::Enum::kQuality) ? media::VideoEncoder::LatencyMode::Quality : media::VideoEncoder::LatencyMode::Realtime; if (config->hasContentHint()) { if (config->contentHint() == "detail" || config->contentHint() == keywords::kText) { result->options.content_hint = media::VideoEncoder::ContentHint::Screen; } else if (config->contentHint() == "motion") { result->options.content_hint = media::VideoEncoder::ContentHint::Camera; } } if (config->bitrateMode() == V8VideoEncoderBitrateMode::Enum::kQuantizer) { result->options.bitrate = media::Bitrate::ExternalRateControl(); } else if (config->hasBitrate()) { uint32_t bps = base::saturated_cast(config->bitrate()); if (bps == 0) { exception_state.ThrowTypeError("Bitrate must be greater than zero."); return nullptr; } if (config->bitrateMode() == V8VideoEncoderBitrateMode::Enum::kConstant) { result->options.bitrate = media::Bitrate::ConstantBitrate(bps); } else { // VBR in media:Bitrate supports both target and peak bitrate. // Currently webcodecs doesn't expose peak bitrate // (assuming unconstrained VBR), here we just set peak as 10 times // target as a good enough way of expressing unconstrained VBR. result->options.bitrate = media::Bitrate::VariableBitrate( bps, base::ClampMul(bps, 10u).RawValue()); } } if (config->hasDisplayWidth() && config->hasDisplayHeight()) { if (config->displayHeight() == 0 || config->displayWidth() == 0) { exception_state.ThrowTypeError( "Invalid display size; height and width must be greater than zero."); return nullptr; } result->display_size.emplace(config->displayWidth(), config->displayHeight()); } else if (config->hasDisplayWidth() || config->hasDisplayHeight()) { exception_state.ThrowTypeError( "Invalid display size; both height and width must be set together."); return nullptr; } if (config->hasFramerate()) { constexpr double kMinFramerate = .0001; constexpr double kMaxFramerate = 1'000'000'000; if (std::isnan(config->framerate()) || config->framerate() < kMinFramerate || config->framerate() > kMaxFramerate) { result->not_supported_error_message = Format( "Unsupported framerate; expected range from {:f} to {:f}, received " "{:f}.", kMinFramerate, kMaxFramerate, config->framerate()); return result; } result->options.framerate = config->framerate(); } else { result->options.framerate = media::VideoEncodeAccelerator::kDefaultFramerate; } // https://w3c.github.io/webrtc-svc/ if (config->hasScalabilityMode()) { if (config->scalabilityMode() == "L1T1") { result->options.scalability_mode = media::SVCScalabilityMode::kL1T1; } else if (config->scalabilityMode() == "L1T2") { result->options.scalability_mode = media::SVCScalabilityMode::kL1T2; } else if (config->scalabilityMode() == "L1T3") { result->options.scalability_mode = media::SVCScalabilityMode::kL1T3; } else if (config->scalabilityMode() == keywords::kManual) { result->options.manual_reference_buffer_control = true; } else { result->not_supported_error_message = StrCat({"Unsupported scalabilityMode: ", config->scalabilityMode()}); return result; } } // The IDL defines a default value of "no-preference". DCHECK(config->hasHardwareAcceleration()); result->hw_pref = IdlEnumToHardwarePreference(config->hardwareAcceleration()); result->codec = media::VideoCodec::kUnknown; result->profile = media::VIDEO_CODEC_PROFILE_UNKNOWN; result->level = 0; result->codec_string = config->codec(); auto parse_result = media::ParseVideoCodecString( "", config->codec().Utf8(), /*allow_ambiguous_matches=*/false); if (!parse_result) { return result; } // Some codec strings provide color space info, but for WebCodecs this is // ignored. Instead, the VideoFrames given to encode() are the source of truth // for input color space. Note also that the output color space is up to the // underlying codec impl. See https://github.com/w3c/webcodecs/issues/345. result->codec = parse_result->codec; result->profile = parse_result->profile; result->level = parse_result->level; result->options.subsampling = parse_result->subsampling; result->options.bit_depth = parse_result->bit_depth; // Ideally which profile supports a given subsampling would be checked by // ParseVideoCodecString() above. Unfortunately, ParseVideoCodecString() is // shared by many paths and enforcing profile and subsampling broke several // sites. The error messages below are more helpful anyways. switch (result->codec) { case media::VideoCodec::kH264: { if (config->hasAvc()) { switch (config->avc()->format().AsEnum()) { case V8AvcBitstreamFormat::Enum::kAvc: result->options.avc.produce_annexb = false; break; case V8AvcBitstreamFormat::Enum::kAnnexb: result->options.avc.produce_annexb = true; break; } } break; } case media::VideoCodec::kHEVC: { if (config->hasHevc()) { switch (config->hevc()->format().AsEnum()) { case V8HevcBitstreamFormat::Enum::kHevc: result->options.hevc.produce_annexb = false; break; case V8HevcBitstreamFormat::Enum::kAnnexb: result->options.hevc.produce_annexb = true; break; } } break; } default: break; } return result; } bool VerifyCodecSupportStatic(VideoEncoderTraits::ParsedConfig* config, String* js_error_message) { if (config->not_supported_error_message) { *js_error_message = *config->not_supported_error_message; return false; } const auto& frame_size = config->options.frame_size; if (frame_size.height() > media::limits::kMaxDimension) { *js_error_message = Format("Invalid height; expected range from {} to {}, received {}.", 1, media::limits::kMaxDimension, frame_size.height()); return false; } if (frame_size.width() > media::limits::kMaxDimension) { *js_error_message = Format("Invalid width; expected range from {} to {}, received {}.", 1, media::limits::kMaxDimension, frame_size.width()); return false; } if (frame_size.Area64() > media::limits::kMaxCanvas) { *js_error_message = Format( "Invalid resolution; expected range from {} to {}, received {} ({} * " "{}).", 1, media::limits::kMaxCanvas, frame_size.Area64(), frame_size.width(), frame_size.height()); return false; } switch (config->codec) { case media::VideoCodec::kAV1: case media::VideoCodec::kVP8: case media::VideoCodec::kVP9: break; #if BUILDFLAG(ENABLE_PLATFORM_HEVC) case media::VideoCodec::kHEVC: if (config->profile != media::VideoCodecProfile::HEVCPROFILE_MAIN && config->profile != media::VideoCodecProfile::HEVCPROFILE_MAIN10 && config->profile != media::VideoCodecProfile::HEVCPROFILE_REXT) { *js_error_message = "Unsupported hevc profile."; return false; } break; #endif case media::VideoCodec::kH264: { if (config->options.frame_size.width() % 2 != 0 || config->options.frame_size.height() % 2 != 0) { *js_error_message = "H264 only supports even sized frames."; return false; } // Note: This calculation is incorrect for interlaced or MBAFF encoding; // but we don't support those and likely never will. gfx::Size coded_size(base::bits::AlignUpDeprecatedDoNotUse( config->options.frame_size.width(), 16), base::bits::AlignUpDeprecatedDoNotUse( config->options.frame_size.height(), 16)); uint64_t coded_area = coded_size.Area64(); uint64_t max_coded_area = media::H264LevelToMaxFS(config->level) * 16ull * 16ull; if (coded_area > max_coded_area) { *js_error_message = Format( "The provided resolution ({}) has a coded area ({}*{}={}) which " "exceeds the maximum coded area ({}) supported by the AVC level " "({:1.1f}) indicated by the codec string (0x{:02X}). You must " "either specify a lower resolution or higher AVC level.", config->options.frame_size.ToString(), coded_size.width(), coded_size.height(), coded_area, max_coded_area, config->level / 10.0f, config->level); return false; } break; } default: *js_error_message = "Unsupported codec type."; return false; } return true; } VideoEncoderConfig* CopyConfig( const VideoEncoderConfig& config, const VideoEncoderTraits::ParsedConfig& parsed_config) { auto* result = VideoEncoderConfig::Create(); result->setCodec(config.codec()); result->setWidth(config.width()); result->setHeight(config.height()); if (config.hasDisplayWidth()) result->setDisplayWidth(config.displayWidth()); if (config.hasDisplayHeight()) result->setDisplayHeight(config.displayHeight()); if (config.hasFramerate()) result->setFramerate(config.framerate()); if (config.hasBitrate()) result->setBitrate(config.bitrate()); if (config.hasScalabilityMode()) result->setScalabilityMode(config.scalabilityMode()); if (config.hasHardwareAcceleration()) result->setHardwareAcceleration(config.hardwareAcceleration()); if (config.hasAlpha()) result->setAlpha(config.alpha()); if (config.hasBitrateMode()) result->setBitrateMode(config.bitrateMode()); if (config.hasLatencyMode()) result->setLatencyMode(config.latencyMode()); if (config.hasContentHint()) { result->setContentHint(config.contentHint()); } if (config.hasAvc() && config.avc()->hasFormat()) { auto* avc = AvcEncoderConfig::Create(); avc->setFormat(config.avc()->format()); result->setAvc(avc); } if (config.hasHevc() && config.hevc()->hasFormat()) { auto* hevc = HevcEncoderConfig::Create(); hevc->setFormat(config.hevc()->format()); result->setHevc(hevc); } return result; } bool CanUseGpuMemoryBufferReadback(media::VideoPixelFormat format, bool force_opaque) { // GMB readback only works with NV12, so only opaque buffers can be used. return (format == media::PIXEL_FORMAT_XBGR || format == media::PIXEL_FORMAT_XRGB || (force_opaque && (format == media::PIXEL_FORMAT_ABGR || format == media::PIXEL_FORMAT_ARGB))) && WebGraphicsContext3DVideoFramePool:: IsGpuMemoryBufferReadbackFromTextureEnabled(); } EncoderType GetRequiredEncoderType(media::VideoCodecProfile profile, HardwarePreference hw_pref) { if (hw_pref != HardwarePreference::kPreferHardware && media::MayHaveAndAllowSelectOSSoftwareEncoder( media::VideoCodecProfileToVideoCodec(profile))) { return hw_pref == HardwarePreference::kPreferSoftware ? EncoderType::kSoftware : EncoderType::kNoPreference; } return EncoderType::kHardware; } gfx::ColorSpace::MatrixID GetYuvMatrixForPrimaries( gfx::ColorSpace::PrimaryID primaries) { switch (primaries) { case gfx::ColorSpace::PrimaryID::BT470M: return gfx::ColorSpace::MatrixID::FCC; case gfx::ColorSpace::PrimaryID::BT470BG: return gfx::ColorSpace::MatrixID::BT470BG; case gfx::ColorSpace::PrimaryID::SMPTE170M: return gfx::ColorSpace::MatrixID::SMPTE170M; case gfx::ColorSpace::PrimaryID::SMPTE240M: return gfx::ColorSpace::MatrixID::SMPTE240M; case gfx::ColorSpace::PrimaryID::BT2020: return gfx::ColorSpace::MatrixID::BT2020_NCL; default: // Primaries without a corresponding YCbCr matrix, such as P3 and XYZ, // use BT.709 as the conversion matrix. return gfx::ColorSpace::MatrixID::BT709; } } } // namespace gfx::ColorSpace GetReadbackYuvColorSpace( const gfx::ColorSpace& source_color_space) { if (!source_color_space.IsValid()) { return gfx::ColorSpace::CreateREC709(); } const gfx::ColorSpace::MatrixID source_matrix = source_color_space.GetMatrixID(); // If the source already declares a YCbCr matrix, preserve it. The readback // frame is always video range. if (source_matrix != gfx::ColorSpace::MatrixID::RGB && source_matrix != gfx::ColorSpace::MatrixID::GBR) { return source_color_space.GetWithMatrixAndRange( source_matrix, gfx::ColorSpace::RangeID::LIMITED); } // RGB/GBR are identity matrices. Select a YCbCr matrix from the primaries // before RGB-to-YUV readback. return source_color_space.GetWithMatrixAndRange( GetYuvMatrixForPrimaries(source_color_space.GetPrimaryID()), gfx::ColorSpace::RangeID::LIMITED); } // static const char* VideoEncoderTraits::GetName() { return "VideoEncoder"; } String VideoEncoderTraits::ParsedConfig::ToString() { return Format( "{{codec: {}, profile: {}, level: {}, hw_pref: {}, options: {{{}}}, " "codec_string: {}, display_size: {}}}", media::GetCodecName(codec), media::GetProfileName(profile), level, HardwarePreferenceToIdlEnum(hw_pref).AsCStr(), options.ToString(), codec_string, display_size ? display_size->ToString() : ""); } // static VideoEncoder* VideoEncoder::Create(ScriptState* script_state, const VideoEncoderInit* init, ExceptionState& exception_state) { auto* result = MakeGarbageCollected(script_state, init, exception_state); return exception_state.HadException() ? nullptr : result; } VideoEncoder::VideoEncoder(ScriptState* script_state, const VideoEncoderInit* init, ExceptionState& exception_state) : Base(script_state, init, exception_state), max_active_encodes_(ComputeMaxActiveEncodes()) { UseCounter::Count(ExecutionContext::From(script_state), WebFeature::kWebCodecs); } VideoEncoder::~VideoEncoder() = default; VideoEncoder::ParsedConfig* VideoEncoder::OnNewConfigure( const VideoEncoderConfig* config, ExceptionState& exception_state) { first_input_transformation_.reset(); return ParseConfigStatic(config, exception_state); } bool VideoEncoder::VerifyCodecSupport(ParsedConfig* config, String* js_error_message) { return VerifyCodecSupportStatic(config, js_error_message); } media::EncoderStatus::Or> VideoEncoder::CreateAcceleratedVideoEncoder( media::VideoCodecProfile profile, const media::VideoEncoder::Options& options, media::GpuVideoAcceleratorFactories* gpu_factories, HardwarePreference hw_pref) { auto required_encoder_type = GetRequiredEncoderType(profile, hw_pref); if (media::EncoderStatus result = IsAcceleratedConfigurationSupported( profile, options, gpu_factories, required_encoder_type); !result.is_ok()) { return std::move(result); } return std::unique_ptr( std::make_unique>( std::make_unique( gpu_factories, logger_->log()->Clone(), callback_runner_, required_encoder_type))); } std::unique_ptr CreateAv1VideoEncoder() { #if BUILDFLAG(ENABLE_LIBAOM) return std::make_unique(); #else return nullptr; #endif // BUILDFLAG(ENABLE_LIBAOM) } std::unique_ptr CreateVpxVideoEncoder() { #if BUILDFLAG(ENABLE_LIBVPX) return std::make_unique(); #else return nullptr; #endif // BUILDFLAG(ENABLE_LIBVPX) } std::unique_ptr CreateOpenH264VideoEncoder() { #if BUILDFLAG(ENABLE_OPENH264) return media::IsOpenH264SoftwareEncoderEnabled() ? std::make_unique() : nullptr; #else return nullptr; #endif // BUILDFLAG(ENABLE_OPENH264) } // This method is static and takes |self| in order to make it possible to use it // with a weak |this|. It's needed in to avoid a persistent reference cycle. media::EncoderStatus::Or> VideoEncoder::CreateSoftwareVideoEncoder(VideoEncoder* self, bool fallback, media::VideoCodec codec) { DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); if (!self) return media::EncoderStatus::Codes::kEncoderIllegalState; std::unique_ptr result; switch (codec) { case media::VideoCodec::kAV1: result = CreateAv1VideoEncoder(); break; case media::VideoCodec::kVP8: case media::VideoCodec::kVP9: result = CreateVpxVideoEncoder(); break; case media::VideoCodec::kH264: result = CreateOpenH264VideoEncoder(); break; default: break; } if (!result) { return media::EncoderStatus::Codes::kEncoderUnsupportedCodec; } if (fallback) { CHECK(self->encoder_metrics_provider_); self->encoder_metrics_provider_->Initialize( self->active_config_->profile, self->active_config_->options.frame_size, /*is_hardware_encoder=*/false, self->active_config_->options.scalability_mode.value_or( media::SVCScalabilityMode::kL1T1)); } return std::unique_ptr( std::make_unique(std::move(result))); } media::EncoderStatus::Or> VideoEncoder::CreateMediaVideoEncoder( const ParsedConfig& config, media::GpuVideoAcceleratorFactories* gpu_factories, bool& is_platform_encoder) { is_platform_encoder = true; if (config.hw_pref == HardwarePreference::kPreferHardware || config.hw_pref == HardwarePreference::kNoPreference || media::MayHaveAndAllowSelectOSSoftwareEncoder(config.codec)) { auto result = CreateAcceleratedVideoEncoder(config.profile, config.options, gpu_factories, config.hw_pref); if (config.hw_pref == HardwarePreference::kPreferHardware) { return result; } else if (result.has_value()) { // 'no-preference' or 'prefer-software' and we have OS software encoders. return std::unique_ptr( std::make_unique( std::move(result).value(), ConvertToBaseOnceCallback( CrossThreadBindOnce(&VideoEncoder::CreateSoftwareVideoEncoder, MakeUnwrappingCrossThreadWeakHandle(this), /*fallback=*/true, config.codec)))); } } is_platform_encoder = false; return CreateSoftwareVideoEncoder(this, /*fallback=*/false, config.codec); } void VideoEncoder::ContinueConfigureAfterFlush(Request* request) { if (active_config_->hw_pref == HardwarePreference::kPreferSoftware && !media::MayHaveAndAllowSelectOSSoftwareEncoder(active_config_->codec)) { ContinueConfigureWithGpuFactories(request, nullptr); return; } RetrieveGpuFactoriesWithKnownEncoderSupport( CrossThreadBindOnce(&VideoEncoder::ContinueConfigureWithGpuFactories, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request))); } void VideoEncoder::ContinueConfigureWithGpuFactories( Request* request, media::GpuVideoAcceleratorFactories* gpu_factories) { DCHECK(active_config_); DCHECK_EQ(request->type, Request::Type::kConfigure); DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); bool is_platform_encoder = false; media_encoder_.reset(); auto encoder_or_error = CreateMediaVideoEncoder( *active_config_, gpu_factories, is_platform_encoder); if (!encoder_or_error.has_value()) { ReportError("Encoder creation error.", std::move(encoder_or_error).error(), /*is_error_message_from_software_codec=*/!is_platform_encoder); request->EndTracing(); return; } media_encoder_ = std::move(encoder_or_error).value(); auto info_cb = ConvertToBaseRepeatingCallback( CrossThreadBindRepeating(&VideoEncoder::OnMediaEncoderInfoChanged, MakeUnwrappingCrossThreadWeakHandle(this))); auto output_cb = ConvertToBaseRepeatingCallback(CrossThreadBindRepeating( &VideoEncoder::CallOutputCallback, MakeUnwrappingCrossThreadWeakHandle(this), // We can't use |active_config_| from |this| because it can change by // the time the callback is executed. MakeUnwrappingCrossThreadHandle(active_config_.Get()), reset_count_)); auto done_callback = [](VideoEncoder* self, Request* req, media::VideoCodec codec, const bool is_platform_encoder, media::EncoderStatus status) { if (!self || self->reset_count_ != req->reset_count) { req->EndTracing(/*aborted=*/true); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); DCHECK(self->active_config_); MEDIA_LOG(INFO, self->logger_->log()) << "Configured " << self->active_config_->ToString(); if (!status.is_ok()) { const char* error_message; switch (status.code()) { case media::EncoderStatus::Codes::kEncoderUnsupportedProfile: error_message = "Unsupported codec profile."; break; case media::EncoderStatus::Codes::kEncoderUnsupportedConfig: error_message = "Unsupported configuration parameters."; break; case media::EncoderStatus::Codes::kOutOfPlatformEncoders: error_message = "The system ran out of platform encoders."; break; default: error_message = "Encoder initialization error."; break; } self->ReportError( error_message, std::move(status), /*is_error_message_from_software_codec=*/!is_platform_encoder); } else { base::UmaHistogramEnumeration("Blink.WebCodecs.VideoEncoder.Codec", codec); } req->EndTracing(); self->blocking_request_in_progress_ = nullptr; self->ProcessRequests(); }; if (!encoder_metrics_provider_) { encoder_metrics_provider_ = CreateVideoEncoderMetricsProvider(); } if (active_config_->codec == media::VideoCodec::kAV1 && active_config_->options.bitrate.value_or(media::Bitrate()).mode() == media::Bitrate::Mode::kExternal) { // Count usages of Av1 encoding with per-frame QP, to estimate how many // users would notice if we change Av1 QP range from 0-63 to 0-255. UseCounter::Count(GetExecutionContext(), WebFeature::kWebCodecsAv1EncodingWithQP); } last_decoder_config_.reset(); first_output_after_configure_ = true; encoder_metrics_provider_->Initialize( active_config_->profile, active_config_->options.frame_size, is_platform_encoder, active_config_->options.scalability_mode.value_or( media::SVCScalabilityMode::kL1T1)); media_encoder_->Initialize( active_config_->profile, active_config_->options, std::move(info_cb), std::move(output_cb), ConvertToBaseOnceCallback(CrossThreadBindOnce( done_callback, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request), active_config_->codec, is_platform_encoder))); } std::unique_ptr VideoEncoder::CreateVideoEncoderMetricsProvider() const { mojo::PendingRemote video_encoder_metrics_provider; LocalDOMWindow* window = DomWindow(); LocalFrame* local_frame = window ? window->GetFrame() : nullptr; // There is no DOM frame if WebCodecs runs in a service worker. if (local_frame) { local_frame->GetBrowserInterfaceBroker().GetInterface( video_encoder_metrics_provider.InitWithNewPipeAndPassReceiver()); } else { Platform::Current()->GetBrowserInterfaceBroker()->GetInterface( video_encoder_metrics_provider.InitWithNewPipeAndPassReceiver()); } return base::MakeRefCounted( media::mojom::VideoEncoderUseCase::kWebCodecs, std::move(video_encoder_metrics_provider)) ->CreateVideoEncoderMetricsProvider(); } bool VideoEncoder::CanReconfigure(ParsedConfig& original_config, ParsedConfig& new_config) { // Reconfigure is intended for things that don't require changing underlying // codec implementation and can be changed on the fly. return original_config.codec == new_config.codec && original_config.profile == new_config.profile && original_config.level == new_config.level && original_config.hw_pref == new_config.hw_pref && original_config.options.avc.produce_annexb == new_config.options.avc.produce_annexb && original_config.options.hevc.produce_annexb == new_config.options.hevc.produce_annexb; } const AtomicString& VideoEncoder::InterfaceName() const { return event_target_names::kVideoEncoder; } bool VideoEncoder::HasPendingActivity() const { return (active_encodes_ > 0) || Base::HasPendingActivity(); } void VideoEncoder::Trace(Visitor* visitor) const { visitor->Trace(background_readback_); visitor->Trace(frame_reference_buffers_); Base::Trace(visitor); } void VideoEncoder::ReportError(const char* error_message, const media::EncoderStatus& status, bool is_error_message_from_software_codec) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); CHECK(!status.is_ok()); // ReportError() can be called before |encoder_metrics_provider_| is created // in media::VideoEncoder::Initialize() (e.g. there is no available // media::VideoEncoder). Since the case is about webrtc::VideoEncoder failure, // we don't record it. if (encoder_metrics_provider_) { encoder_metrics_provider_->SetError(status); } DOMException* error = nullptr; if (status.code() == media::EncoderStatus::Codes::kOutOfPlatformEncoders) { error = logger_->MakeQuotaError(error_message, status); } else { error = is_error_message_from_software_codec ? logger_->MakeSoftwareCodecOperationError(error_message, status) : logger_->MakeOperationError(error_message, status); } // We don't use `is_platform_encoder_` here since it may not match where the // error is coming from in the case of a pending configuration change. HandleError(error); } bool VideoEncoder::ReadyToProcessNextRequest() { if (active_encodes_ >= max_active_encodes_) return false; return Base::ReadyToProcessNextRequest(); } bool VideoEncoder::StartReadback(scoped_refptr frame, ReadbackDoneCallback result_cb) { // TODO(crbug.com/1195433): Once support for alpha channel encoding is // implemented, |force_opaque| must be set based on the // VideoEncoderConfig. // // TODO(crbug.com/1116564): If we ever support high bit depth read back, this // path should do something different based on options.bit_depth. const bool can_use_gmb = active_config_->options.subsampling != media::VideoChromaSampling::k444 && !disable_accelerated_frame_pool_ && CanUseGpuMemoryBufferReadback(frame->format(), /*force_opaque=*/true); if (can_use_gmb && !accelerated_frame_pool_) { if (auto wrapper = SharedGpuContext::ContextProviderWrapper()) { accelerated_frame_pool_ = std::make_unique(wrapper); } } auto [pool_result_cb, background_result_cb] = base::SplitOnceCallback(std::move(result_cb)); if (can_use_gmb && accelerated_frame_pool_) { // CopyRGBATextureToVideoFrame() operates on mailboxes and // not frames, so we must manually copy over properties relevant to // the encoder. We amend result callback to do exactly that. auto metadata_fix_lambda = [](scoped_refptr txt_frame, scoped_refptr result_frame) -> scoped_refptr { if (!result_frame) { return result_frame; } if (txt_frame->visible_rect() != gfx::Rect(txt_frame->coded_size()) || txt_frame->natural_size() != txt_frame->coded_size()) { result_frame = media::VideoFrame::WrapVideoFrame( result_frame, result_frame->format(), txt_frame->visible_rect(), txt_frame->natural_size()); if (!result_frame) { return nullptr; } } result_frame->set_timestamp(txt_frame->timestamp()); result_frame->set_hdr_metadata(txt_frame->hdr_metadata()); result_frame->metadata().MergeMetadataFrom(txt_frame->metadata()); result_frame->metadata().ClearTextureFrameMetadata(); return result_frame; }; auto callback_chain = ConvertToBaseOnceCallback( CrossThreadBindOnce(metadata_fix_lambda, frame)) .Then(std::move(pool_result_cb)); const gfx::ColorSpace readback_color_space = GetReadbackYuvColorSpace(frame->ColorSpace()); TRACE_EVENT_BEGIN( "media", "CopyRGBATextureToVideoFrame", perfetto::NamedTrack::FromPointer("blink::VideoEncoder", this), "timestamp", frame->timestamp()); if (accelerated_frame_pool_->CopyRGBATextureToVideoFrame( frame->coded_size(), frame->shared_image(), frame->acquire_sync_token(), readback_color_space, std::move(callback_chain))) { return true; } TRACE_EVENT_END( "media", /*CopyRGBATextureToVideoFrame*/ perfetto::NamedTrack::FromPointer("blink::VideoEncoder", this)); // Error occurred, fall through to normal readback path below. disable_accelerated_frame_pool_ = true; accelerated_frame_pool_.reset(); } if (!background_readback_) background_readback_ = BackgroundReadback::From(*GetExecutionContext()); if (background_readback_) { background_readback_->ReadbackTextureBackedFrameToMemoryFrame( std::move(frame), std::move(background_result_cb)); return true; } // Oh well, none of our readback mechanisms were able to succeed. return false; } void VideoEncoder::ProcessEncode(Request* request) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK_EQ(state_, V8CodecState::Enum::kConfigured); DCHECK(media_encoder_); DCHECK_EQ(request->type, Request::Type::kEncode); DCHECK_GT(requested_encodes_, 0u); if (request->encodeOpts->hasUpdateBuffer()) { auto* buffer = request->encodeOpts->updateBuffer(); if (buffer->owner() != this) { QueueHandleError(MakeGarbageCollected( DOMExceptionCode::kNotAllowedError, "updateBuffer doesn't belong to this encoder")); request->EndTracing(); return; } } if (request->encodeOpts->hasReferenceBuffers()) { for (auto& buffer : request->encodeOpts->referenceBuffers()) { if (buffer->owner() != this) { QueueHandleError(MakeGarbageCollected( DOMExceptionCode::kNotAllowedError, "one of referenceBuffers doesn't belong to this encoder")); request->EndTracing(); return; } } } auto frame = request->input->frame(); auto encode_options = CreateEncodeOptions(request); active_encodes_++; auto encode_done_callback = ConvertToBaseOnceCallback(CrossThreadBindOnce( &VideoEncoder::OnEncodeDone, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request))); auto blink_timestamp = base::Microseconds(request->input->timestamp()); if (frame->timestamp() != blink_timestamp && base::FeatureList::IsEnabled(kUseBlinkTimestampForEncoding)) { // If blink::VideFrame has the timestamp different from media::VideoFrame // we need to use blink's timestamp, because this is what JS-devs observe // and it's expected to be the timestamp of the EncodedVideoChunk. // More context about timestamp adjustments: crbug.com/333420614, // crbug.com/350780007 frame = media::VideoFrame::WrapVideoFrame( frame, frame->format(), frame->visible_rect(), frame->natural_size()); frame->set_timestamp(blink_timestamp); } base::TimeDelta frame_duration; if (frame->metadata().frame_duration && frame->metadata().frame_duration != media::kInfiniteDuration && frame->metadata().frame_duration != media::kNoTimestamp) { frame_duration = *frame->metadata().frame_duration; } // While this isn't allowed to change between calls to encode(), it may change // after a call to configure(). So we put the transform in FrameMetadata to // ensure orientation is attached to the right decoder config. media::VideoTransformation frame_transform; if (frame->metadata().transformation) { frame_transform = *frame->metadata().transformation; } frame_metadata_[frame->timestamp()] = FrameMetadata{ .duration = frame_duration, .transformation = frame_transform}; request->StartTracingVideoEncode(encode_options.key_frame, frame->timestamp()); bool mappable = frame->HasDirectCpuAccess() || frame->HasMappableSharedImage(); bool can_handle_shared_image = frame->HasSharedImage() && encoder_info_.DoesSupportGpuSharedImages(frame->shared_image()->usage(), frame->format()); // Currently underlying encoders can't handle frame backed by textures, // so let's readback pixel data to CPU memory. // TODO(crbug.com/1229845): We shouldn't be reading back frames here. if (!mappable && !can_handle_shared_image) { DCHECK(frame->HasSharedImage()); // Stall request processing while we wait for the copy to complete. It'd // be nice to not have to do this, but currently the request processing // loop must execute synchronously or flush() will miss frames. // // Note: Set this before calling StartReadback() since callbacks could // resolve synchronously. blocking_request_in_progress_ = request; auto readback_done_callback = blink::BindOnce(&VideoEncoder::OnReadbackDone, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request), frame, std::move(encode_done_callback)); if (StartReadback(std::move(frame), std::move(readback_done_callback))) { request->input->close(); } else { blocking_request_in_progress_ = nullptr; callback_runner_->PostTask( FROM_HERE, ConvertToBaseOnceCallback(CrossThreadBindOnce( &VideoEncoder::OnEncodeDone, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request), media::EncoderStatus( media::EncoderStatus::Codes::kEncoderFailedEncode, "Can't readback frame textures.")))); } return; } // Currently underlying encoders can't handle alpha channel, so let's // wrap a frame with an alpha channel into a frame without it. // For example such frames can come from 2D canvas context with alpha = true. DCHECK(mappable || can_handle_shared_image); if (media::IsYuvPlanar(frame->format()) && !media::IsOpaque(frame->format())) { frame = media::VideoFrame::WrapVideoFrame( frame, ToOpaqueMediaPixelFormat(frame->format()), frame->visible_rect(), frame->natural_size()); } if (frame->HasSharedImage()) { // This frame might have a sync token. In order to transmit this sync // token to the gpu process, it must be verified. This flushes the // renderer side command buffer and ensures tha the sync token is valid // on the gpu process side. The encoder will actually wait on the sync // token before trying to acquire the shared image. auto wrapper = SharedGpuContext::ContextProviderWrapper(); if (wrapper) { gpu::SyncToken token = frame->acquire_sync_token(); wrapper->ContextProvider().SharedImageInterface()->VerifySyncToken(token); frame->UpdateAcquireSyncToken(token); } } --requested_encodes_; ScheduleDequeueEvent(); media_encoder_->Encode(frame, encode_options, std::move(encode_done_callback)); // We passed a copy of frame() above, so this should be safe to close here. request->input->close(); } media::VideoEncoder::EncodeOptions VideoEncoder::CreateEncodeOptions( Request* request) { media::VideoEncoder::EncodeOptions result; result.key_frame = request->encodeOpts->keyFrame(); if (request->encodeOpts->hasUpdateBuffer()) { result.update_buffer = request->encodeOpts->updateBuffer()->internal_id(); } if (request->encodeOpts->hasReferenceBuffers()) { for (auto& buffer : request->encodeOpts->referenceBuffers()) { result.reference_buffers.push_back(buffer->internal_id()); } } switch (active_config_->codec) { case media::VideoCodec::kAV1: { if (!active_config_->options.bitrate.has_value() || active_config_->options.bitrate->mode() != media::Bitrate::Mode::kExternal) { break; } if (!request->encodeOpts->hasAv1() || !request->encodeOpts->av1()->hasQuantizer()) { break; } result.quantizer = request->encodeOpts->av1()->quantizer(); break; } case media::VideoCodec::kVP9: { if (!active_config_->options.bitrate.has_value() || active_config_->options.bitrate->mode() != media::Bitrate::Mode::kExternal) { break; } if (!request->encodeOpts->hasVp9() || !request->encodeOpts->vp9()->hasQuantizer()) { break; } result.quantizer = request->encodeOpts->vp9()->quantizer(); break; } case media::VideoCodec::kH264: if (!active_config_->options.bitrate.has_value() || active_config_->options.bitrate->mode() != media::Bitrate::Mode::kExternal) { break; } if (!request->encodeOpts->hasAvc() || !request->encodeOpts->avc()->hasQuantizer()) { break; } result.quantizer = request->encodeOpts->avc()->quantizer(); break; case media::VideoCodec::kHEVC: if (!active_config_->options.bitrate.has_value() || active_config_->options.bitrate->mode() != media::Bitrate::Mode::kExternal) { break; } if (!request->encodeOpts->hasHevc() || !request->encodeOpts->hevc()->hasQuantizer()) { break; } result.quantizer = request->encodeOpts->hevc()->quantizer(); break; case media::VideoCodec::kVP8: default: break; } return result; } void VideoEncoder::OnReadbackDone( Request* request, scoped_refptr txt_frame, media::VideoEncoder::EncoderStatusCB done_callback, scoped_refptr result_frame) { TRACE_EVENT_END( "media", /*CopyRGBATextureToVideoFrame*/ perfetto::NamedTrack::FromPointer("blink::VideoEncoder", this)); if (reset_count_ != request->reset_count) { return; } if (!result_frame) { callback_runner_->PostTask( FROM_HERE, ConvertToBaseOnceCallback(CrossThreadBindOnce( std::move(done_callback), media::EncoderStatus( media::EncoderStatus::Codes::kEncoderFailedEncode, "Can't readback frame textures.")))); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); auto encode_options = CreateEncodeOptions(request); --requested_encodes_; ScheduleDequeueEvent(); blocking_request_in_progress_ = nullptr; media_encoder_->Encode(std::move(result_frame), encode_options, std::move(done_callback)); ProcessRequests(); } void VideoEncoder::OnEncodeDone(Request* request, media::EncoderStatus status) { if (reset_count_ != request->reset_count) { request->EndTracing(/*aborted=*/true); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); active_encodes_--; if (!status.is_ok()) { ReportError("Encoding error.", std::move(status), /*is_error_message_from_software_codec=*/!is_platform_encoder_); } request->EndTracing(); ProcessRequests(); } void VideoEncoder::ProcessConfigure(Request* request) { DCHECK_NE(state_.AsEnum(), V8CodecState::Enum::kClosed); DCHECK_EQ(request->type, Request::Type::kConfigure); DCHECK(request->config); DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); request->StartTracing(); blocking_request_in_progress_ = request; active_config_ = request->config; String js_error_message; if (!VerifyCodecSupport(active_config_, &js_error_message)) { QueueHandleError(MakeGarbageCollected( DOMExceptionCode::kNotSupportedError, js_error_message)); request->EndTracing(); return; } // TODO(crbug.com/347676170): remove this hack when we make // getAllFrameBuffers() async and can asynchronously get the number of // encoder buffers. if (active_config_->options.manual_reference_buffer_control && (active_config_->codec == media::VideoCodec::kAV1 || active_config_->codec == media::VideoCodec::kH264 #if BUILDFLAG(ENABLE_PLATFORM_HEVC) || active_config_->codec == media::VideoCodec::kHEVC #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) )) { frame_reference_buffers_.clear(); for (size_t i = 0; i < 3; ++i) { auto* buffer = MakeGarbageCollected(this, i); frame_reference_buffers_.push_back(buffer); } } if (!media_encoder_) { ContinueConfigureAfterFlush(request); return; } auto flush_done_callback = [](VideoEncoder* self, Request* req, media::EncoderStatus status) { if (!self || self->reset_count_ != req->reset_count) { req->EndTracing(/*aborted=*/true); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); if (!status.is_ok()) { self->ReportError( "Encoder flush error.", std::move(status), /*is_error_message_from_software_codec=*/!self->is_platform_encoder_); self->blocking_request_in_progress_ = nullptr; req->EndTracing(); return; } self->ContinueConfigureAfterFlush(req); }; media_encoder_->Flush(BindOnce(flush_done_callback, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request))); } void VideoEncoder::ProcessReconfigure(Request* request) { DCHECK_EQ(state_.AsEnum(), V8CodecState::Enum::kConfigured); DCHECK_EQ(request->type, Request::Type::kReconfigure); DCHECK(request->config); DCHECK(media_encoder_); DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); request->StartTracing(); String js_error_message; if (!VerifyCodecSupport(request->config, &js_error_message)) { QueueHandleError(MakeGarbageCollected( DOMExceptionCode::kNotSupportedError, js_error_message)); request->EndTracing(); return; } auto reconf_done_callback = [](VideoEncoder* self, Request* req, media::EncoderStatus status) { if (!self || self->reset_count_ != req->reset_count) { req->EndTracing(/*aborted=*/true); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); DCHECK(self->active_config_); req->EndTracing(); if (status.is_ok()) { self->blocking_request_in_progress_ = nullptr; self->ProcessRequests(); } else { // Reconfiguration failed. Either encoder doesn't support changing options // or it didn't like this particular change. Let's try to configure it // from scratch. req->type = Request::Type::kConfigure; self->ProcessConfigure(req); } }; auto flush_done_callback = [](VideoEncoder* self, Request* req, decltype(reconf_done_callback) reconf_callback, bool is_platform_encoder, media::EncoderStatus status) { if (!self || self->reset_count_ != req->reset_count) { req->EndTracing(/*aborted=*/true); return; } DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_); if (!status.is_ok()) { self->ReportError( "Encoder initialization error.", std::move(status), /*is_error_message_from_software_codec=*/!is_platform_encoder); self->blocking_request_in_progress_ = nullptr; req->EndTracing(); return; } self->active_config_ = req->config; auto output_cb = ConvertToBaseRepeatingCallback(CrossThreadBindRepeating( &VideoEncoder::CallOutputCallback, MakeUnwrappingCrossThreadWeakHandle(self), // We can't use |active_config_| from |this| because it can change // by the time the callback is executed. MakeUnwrappingCrossThreadHandle(self->active_config_.Get()), self->reset_count_)); if (!self->encoder_metrics_provider_) { self->encoder_metrics_provider_ = self->CreateVideoEncoderMetricsProvider(); } self->encoder_metrics_provider_->Initialize( self->active_config_->profile, self->active_config_->options.frame_size, is_platform_encoder, self->active_config_->options.scalability_mode.value_or( media::SVCScalabilityMode::kL1T1)); self->first_output_after_configure_ = true; self->media_encoder_->ChangeOptions( self->active_config_->options, std::move(output_cb), ConvertToBaseOnceCallback(CrossThreadBindOnce( reconf_callback, MakeUnwrappingCrossThreadWeakHandle(self), MakeUnwrappingCrossThreadHandle(req)))); }; blocking_request_in_progress_ = request; media_encoder_->Flush( BindOnce(flush_done_callback, MakeUnwrappingCrossThreadWeakHandle(this), MakeUnwrappingCrossThreadHandle(request), std::move(reconf_done_callback), is_platform_encoder_)); } void VideoEncoder::OnMediaEncoderInfoChanged( const media::VideoEncoderInfo& encoder_info) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); if (encoder_info.is_hardware_accelerated) ApplyCodecPressure(); else ReleaseCodecPressure(); encoder_info_ = encoder_info; media::MediaLog* log = logger_->log(); log->SetProperty( encoder_info.implementation_name); log->SetProperty( encoder_info.is_hardware_accelerated); is_platform_encoder_ = encoder_info.is_hardware_accelerated; max_active_encodes_ = ComputeMaxActiveEncodes(encoder_info.frame_delay, encoder_info.input_capacity); if (active_config_->options.manual_reference_buffer_control) { frame_reference_buffers_.clear(); for (size_t i = 0; i < encoder_info.number_of_manual_reference_buffers; ++i) { auto* buffer = MakeGarbageCollected(this, i); frame_reference_buffers_.push_back(buffer); } } // We may have increased our capacity for active encodes. ProcessRequests(); } void VideoEncoder::CallOutputCallback( ParsedConfig* active_config, uint32_t reset_count, media::VideoEncoderOutput output, std::optional codec_desc) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(active_config); if (!script_state_->ContextIsValid() || !output_callback_ || state_.AsEnum() != V8CodecState::Enum::kConfigured || reset_count != reset_count_) { return; } MarkCodecActive(); if (output.data.empty()) { // The encoder drops a frame.WebCodecs doesn't specify a way of signaling // a frame was dropped. For now, the output callback is not invoked for the // dropped frame. TODO(https://www.w3.org/TR/webcodecs/#encodedvideochunk): // Notify a client that a frame is dropped. return; } auto buffer = media::DecoderBuffer::FromArray(std::move(output.data)); buffer->set_timestamp(output.timestamp); buffer->set_is_key_frame(output.key_frame); auto output_transform = media::kNoTransformation; const auto it = frame_metadata_.find(output.timestamp); if (it != frame_metadata_.end()) { if (!it->second.duration.is_zero()) { buffer->set_duration(it->second.duration); } output_transform = it->second.transformation; // While encoding happens in presentation order, outputs may be out of order // for some codec configurations. The maximum number of reordered outputs is // 16, so we can clear everything before that. if (it - frame_metadata_.begin() > 16) { frame_metadata_.erase(frame_metadata_.begin(), it + 1); } } auto* chunk = MakeGarbageCollected(std::move(buffer)); auto* metadata = EncodedVideoChunkMetadata::Create(); if (active_config->options.scalability_mode.has_value()) { auto* svc_metadata = SvcOutputMetadata::Create(); svc_metadata->setTemporalLayerId(output.temporal_id); metadata->setSvc(svc_metadata); } // TODO(https://crbug.com/1241448): All encoders should output color space. // For now, fallback to 601 since that is correct most often. gfx::ColorSpace output_color_space = output.color_space.IsValid() ? output.color_space : gfx::ColorSpace::CreateREC601(); auto video_color_space = media::VideoColorSpace::FromGfxColorSpace(output_color_space); auto encoded_size = output.encoded_size.value_or(active_config->options.frame_size); std::vector extra_data; if (codec_desc.has_value()) { extra_data = std::move(codec_desc.value()); codec_desc.reset(); } else if (last_decoder_config_.has_value()) { extra_data = last_decoder_config_->extra_data(); } media::VideoDecoderConfig current_decoder_config( active_config->codec, active_config->profile, media::VideoDecoderConfig::AlphaMode::kIsOpaque, video_color_space, output_transform, encoded_size, gfx::Rect(encoded_size), active_config->display_size.value_or(encoded_size), extra_data, media::EncryptionScheme::kUnencrypted); // We need to emit new config in the following cases: // 1. We haven't emitted a decoder config yet (first output) // 2. Decoder config has changed. if (!last_decoder_config_.has_value() || !last_decoder_config_->Matches(current_decoder_config)) { #if DCHECK_IS_ON() if (!last_decoder_config_.has_value() || video_color_space != last_decoder_config_->color_space_info()) { DCHECK(output.key_frame) << "Encoders should generate a keyframe when " << "changing color space." << " output_color_space: " << output_color_space.ToString() << " last_output_color_space: " << (last_decoder_config_.has_value() ? last_decoder_config_->color_space_info() .ToGfxColorSpace() .ToString() : std::string("none")); } if (active_config->codec == media::VideoCodec::kH264) { DCHECK(active_config->options.avc.produce_annexb || extra_data.size() > 0u); } #endif // DCHECK_IS_ON() auto* decoder_config = VideoDecoderConfig::Create(); decoder_config->setCodec(active_config->codec_string); decoder_config->setCodedHeight(encoded_size.height()); decoder_config->setCodedWidth(encoded_size.width()); decoder_config->setRotation(output_transform.rotation); decoder_config->setFlip(output_transform.mirrored); if (active_config->display_size.has_value()) { decoder_config->setDisplayAspectHeight( active_config->display_size.value().height()); decoder_config->setDisplayAspectWidth( active_config->display_size.value().width()); } VideoColorSpace* color_space = MakeGarbageCollected(output_color_space); decoder_config->setColorSpace(color_space->toJSON()); if (extra_data.size() > 0) { auto* desc_array_buf = DOMArrayBuffer::Create(extra_data); decoder_config->setDescription( MakeGarbageCollected(desc_array_buf)); } metadata->setDecoderConfig(decoder_config); last_decoder_config_ = current_decoder_config; } encoder_metrics_provider_->IncrementEncodedFrameCount(); TRACE_EVENT(kCategory, perfetto::StaticString(GetTraceNames()->output.c_str()), "timestamp", chunk->timestamp()); ScriptState::Scope scope(script_state_); output_callback_->InvokeAndReportException(nullptr, chunk, metadata); } void VideoEncoder::ResetInternal(DOMException* ex) { Base::ResetInternal(ex); active_encodes_ = 0; last_decoder_config_.reset(); } void VideoEncoder::OnNewEncode(InputType* input, ExceptionState& exception_state) { auto frame = input->frame(); if (!frame) { // Let the invalid frame path be taken by calling code. return; } auto frame_transform = frame->metadata().transformation.value_or(media::kNoTransformation); if (!first_input_transformation_) { first_input_transformation_ = frame_transform; return; } if (first_input_transformation_ == frame_transform) { return; } exception_state.ThrowDOMException( DOMExceptionCode::kDataError, "Encoding frames with different orientations is not allowed. You must " "either reorient the frames or reconfigure the encoder."); } void FindAnySupported( ScriptPromiseResolver* resolver, const GCedHeapVector>& supports) { VideoEncoderSupport* result = nullptr; for (auto& support : supports) { result = support; if (result->supported()) { break; } } resolver->Resolve(result); } static void isConfigSupportedWithSoftwareOnly( ScriptState* script_state, base::OnceCallback callback, VideoEncoderSupport* support, VideoEncoderTraits::ParsedConfig* config) { std::unique_ptr software_encoder; switch (config->codec) { case media::VideoCodec::kAV1: software_encoder = CreateAv1VideoEncoder(); break; case media::VideoCodec::kVP8: case media::VideoCodec::kVP9: software_encoder = CreateVpxVideoEncoder(); break; case media::VideoCodec::kH264: software_encoder = CreateOpenH264VideoEncoder(); break; default: break; } if (!software_encoder) { support->setSupported(false); std::move(callback).Run(support); return; } auto done_callback = [](std::unique_ptr encoder, CrossThreadOnceFunction callback, scoped_refptr runner, VideoEncoderSupport* support, media::EncoderStatus status) { support->setSupported(status.is_ok()); std::move(callback).Run(support); runner->DeleteSoon(FROM_HERE, std::move(encoder)); }; auto* context = ExecutionContext::From(script_state); auto runner = context->GetTaskRunner(TaskType::kInternalDefault); auto* software_encoder_raw = software_encoder.get(); software_encoder_raw->Initialize( config->profile, config->options, /*info_cb=*/base::DoNothing(), /*output_cb=*/base::DoNothing(), ConvertToBaseOnceCallback(CrossThreadBindOnce( done_callback, std::move(software_encoder), CrossThreadBindOnce(std::move(callback)), std::move(runner), MakeUnwrappingCrossThreadHandle(support)))); } static void isConfigSupportedWithHardwareOnly( CrossThreadOnceFunction callback, VideoEncoderSupport* support, VideoEncoderTraits::ParsedConfig* config, media::GpuVideoAcceleratorFactories* gpu_factories) { auto required_encoder_type = GetRequiredEncoderType(config->profile, config->hw_pref); bool supported = IsAcceleratedConfigurationSupported(config->profile, config->options, gpu_factories, required_encoder_type) .is_ok(); support->setSupported(supported); std::move(callback).Run(support); } // static ScriptPromise VideoEncoder::isConfigSupported( ScriptState* script_state, const VideoEncoderConfig* config, ExceptionState& exception_state) { auto* parsed_config = ParseConfigStatic(config, exception_state); if (!parsed_config) { DCHECK(exception_state.HadException()); return EmptyPromise(); } auto* config_copy = CopyConfig(*config, *parsed_config); // Run very basic coarse synchronous validation String unused_js_error_message; if (!VerifyCodecSupportStatic(parsed_config, &unused_js_error_message)) { auto* support = VideoEncoderSupport::Create(); support->setConfig(config_copy); support->setSupported(false); return ToResolvedPromise(script_state, support); } // Schedule tasks for determining hardware and software encoding support and // register them with HeapBarrierCallback. wtf_size_t num_callbacks = 0; if (parsed_config->hw_pref != HardwarePreference::kPreferSoftware || media::MayHaveAndAllowSelectOSSoftwareEncoder(parsed_config->codec)) { ++num_callbacks; } if (parsed_config->hw_pref != HardwarePreference::kPreferHardware) { ++num_callbacks; } auto* resolver = MakeGarbageCollected>( script_state); auto promise = resolver->Promise(); auto find_any_callback = HeapBarrierCallback( num_callbacks, BindOnce(&FindAnySupported, MakeUnwrappingCrossThreadHandle(resolver))); if (parsed_config->hw_pref != HardwarePreference::kPreferSoftware || media::MayHaveAndAllowSelectOSSoftwareEncoder(parsed_config->codec)) { // Hardware support not denied, detect support by hardware encoders. auto* support = VideoEncoderSupport::Create(); support->setConfig(config_copy); auto gpu_retrieved_callback = CrossThreadBindOnce(isConfigSupportedWithHardwareOnly, CrossThreadBindOnce(find_any_callback), MakeUnwrappingCrossThreadHandle(support), MakeUnwrappingCrossThreadHandle(parsed_config)); RetrieveGpuFactoriesWithKnownEncoderSupport( std::move(gpu_retrieved_callback)); } if (parsed_config->hw_pref != HardwarePreference::kPreferHardware) { // Hardware support not required, detect support by software encoders. auto* support = VideoEncoderSupport::Create(); support->setConfig(config_copy); isConfigSupportedWithSoftwareOnly(script_state, find_any_callback, support, parsed_config); } return promise; } HeapVector> VideoEncoder::getAllFrameBuffers( ScriptState*, ExceptionState& exception_state) { if (!active_config_->options.manual_reference_buffer_control) { exception_state.ThrowDOMException( DOMExceptionCode::kNotSupportedError, "getAllFrameBuffers() only supported with manual scalability mode."); return {}; } return frame_reference_buffers_; } } // namespace blink