// Copyright 2012 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/renderers/paint_canvas_video_renderer.h" #include #include #include #include #include #include "base/barrier_closure.h" #include "base/compiler_specific.h" #include "base/containers/auto_spanification_helper.h" #include "base/containers/span.h" #include "base/functional/bind.h" #include "base/functional/callback_helpers.h" #include "base/logging.h" #include "base/memory/aligned_memory.h" #include "base/memory/ptr_util.h" #include "base/memory/raw_ptr.h" #include "base/memory/scoped_refptr.h" #include "base/numerics/byte_conversions.h" #include "base/numerics/checked_math.h" #include "base/sequence_checker.h" #include "base/synchronization/waitable_event.h" #include "base/system/sys_info.h" #include "base/task/thread_pool.h" #include "base/threading/thread_restrictions.h" #include "build/build_config.h" #include "cc/paint/paint_canvas.h" #include "cc/paint/paint_flags.h" #include "cc/paint/paint_image_builder.h" #include "cc/paint/texture_backing.h" #include "components/viz/common/gpu/raster_context_provider.h" #include "components/viz/common/resources/shared_image_format.h" #include "gpu/GLES2/gl2extchromium.h" #include "gpu/command_buffer/client/client_shared_image.h" #include "gpu/command_buffer/client/context_support.h" #include "gpu/command_buffer/client/gles2_interface.h" #include "gpu/command_buffer/client/raster_interface.h" #include "gpu/command_buffer/client/shared_image_interface.h" #include "gpu/command_buffer/common/capabilities.h" #include "gpu/command_buffer/common/mailbox.h" #include "gpu/command_buffer/common/shared_image_capabilities.h" #include "gpu/command_buffer/common/shared_image_usage.h" #include "media/base/data_buffer.h" #include "media/base/format_utils.h" #include "media/base/video_frame.h" #include "media/base/video_transformation.h" #include "media/base/video_util.h" #include "media/base/wait_and_replace_sync_token_client.h" #include "third_party/fp16/src/include/fp16.h" #include "third_party/libyuv/include/libyuv.h" #include "third_party/skia/include/core/SkBitmap.h" #include "third_party/skia/include/core/SkColorSpace.h" #include "third_party/skia/include/core/SkImage.h" #include "third_party/skia/include/core/SkImageGenerator.h" #include "third_party/skia/include/core/SkImageInfo.h" #include "third_party/skia/include/gpu/ganesh/GrBackendSurface.h" #include "third_party/skia/include/gpu/ganesh/GrDirectContext.h" #include "third_party/skia/include/gpu/ganesh/SkImageGanesh.h" #include "third_party/skia/include/gpu/ganesh/gl/GrGLBackendSurface.h" #include "third_party/skia/include/gpu/ganesh/gl/GrGLTypes.h" #include "ui/gfx/geometry/rect_f.h" #include "ui/gfx/geometry/skia_conversions.h" // Skia internal format depends on a platform. On Android it is ABGR, on others // it's ARGB. YUV_ORDER() conditionally remap YUV to YVU for ABGR. #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ SK_A32_SHIFT == 24 #define OUTPUT_ARGB 1 #define LIBYUV_ABGR_TO_ARGB libyuv::ABGRToARGB #define YUV_ORDER(y, y_stride, u, u_stride, v, v_stride) \ (y), (y_stride), (u), (u_stride), (v), (v_stride) #define GBR_TO_RGB_ORDER(y, y_stride, u, u_stride, v, v_stride) \ (v), (v_stride), (y), (y_stride), (u), (u_stride) #define LIBYUV_NV12_TO_ARGB_MATRIX libyuv::NV12ToARGBMatrix #define SHARED_IMAGE_FORMAT viz::SinglePlaneFormat::kBGRA_8888 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ SK_A32_SHIFT == 24 #define OUTPUT_ARGB 0 #define LIBYUV_ABGR_TO_ARGB libyuv::ARGBToABGR #define YUV_ORDER(y, y_stride, u, u_stride, v, v_stride) \ (y), (y_stride), (v), (v_stride), (u), (u_stride) #define GBR_TO_RGB_ORDER(y, y_stride, u, u_stride, v, v_stride) \ (u), (u_stride), (y), (y_stride), (v), (v_stride) #define LIBYUV_NV12_TO_ARGB_MATRIX libyuv::NV21ToARGBMatrix #define SHARED_IMAGE_FORMAT viz::SinglePlaneFormat::kRGBA_8888 #else #error Unexpected Skia ARGB_8888 layout! #endif namespace media { namespace { // Converts YUV video frames to RGB format and stores the results in the // provided destination shared image. The caller of this function maintains // ownership of the destination shared image. Automatically handles upload of // CPU memory backed VideoFrames in multiplanar format that do not have shared // image. Converting CPU backed VideoFrames requires creation of YUV shared // images to upload the frame to the GPU where the conversion takes place. This // will not perform any color space conversion besides the YUV to RGB conversion // (it will ignore the color space of the destination shared image). IMPORTANT: // Callers of this function can pass in `shared_image_cache` to prevent repeated // creation/deletion of YUV shared images. gpu::SyncToken ConvertYuvVideoFrameToRgbSharedImage( const VideoFrame* video_frame, viz::RasterContextProvider* raster_context_provider, scoped_refptr dest_shared_image, const gpu::SyncToken& dest_sync_token, bool use_visible_rect, VideoFrameSharedImageCache* shared_image_cache) { CHECK(video_frame); CHECK(!video_frame->HasSharedImage()); DCHECK(!video_frame->coded_size().IsEmpty()) << "|video_frame| must have an area > 0"; DCHECK(raster_context_provider); // Callers may choose to provide cache which ensures that the source yuv // shared images are cached across convert calls. std::unique_ptr local_si_cache; if (!shared_image_cache) { local_si_cache = std::make_unique(); shared_image_cache = local_si_cache.get(); } auto* ri = raster_context_provider->RasterInterface(); DCHECK(ri); auto source_rect = use_visible_rect ? video_frame->visible_rect() : gfx::Rect(video_frame->coded_size()); // This SharedImage will be written to (and later read from) via the raster // interface. gpu::SharedImageUsageSet src_usage = gpu::SHARED_IMAGE_USAGE_RASTER_READ | gpu::SHARED_IMAGE_USAGE_RASTER_WRITE; // For pure software pixel upload path with video frame that does not have // textures. auto [src_shared_image, si_sync_token, status] = shared_image_cache->GetOrCreateSharedImage( video_frame, raster_context_provider, src_usage); CHECK(src_shared_image); if (status == VideoFrameSharedImageCache::Status::kMatchedVideoFrameId) { // Since the video frame id matches, no need to upload pixels or copy shared // image again. return dest_sync_token; } const viz::SharedImageFormat si_format = src_shared_image->format(); constexpr SkAlphaType kPlaneAlphaType = kUnpremul_SkAlphaType; std::array pixmaps = {}; for (int plane = 0; plane < si_format.NumberOfPlanes(); ++plane) { SkColorType color_type = viz::ToClosestSkColorType(si_format, plane); gfx::Size plane_size = si_format.GetPlaneSize(plane, video_frame->coded_size()); SkImageInfo info = SkImageInfo::Make(gfx::SizeToSkISize(plane_size), color_type, kPlaneAlphaType); pixmaps[plane] = SkPixmap(info, video_frame->data(plane), video_frame->stride(plane)); } // Prepare the SkYUVAInfo SkISize video_size = gfx::SizeToSkISize(video_frame->coded_size()); SkYUVAInfo::PlaneConfig plane_config = ToSkYUVAPlaneConfig(si_format); SkYUVAInfo::Subsampling subsampling = ToSkYUVASubsampling(si_format); // TODO(crbug.com/41380578): This should really default to rec709. SkYUVColorSpace color_space = kRec601_SkYUVColorSpace; video_frame->ColorSpace().ToSkYUVColorSpace(video_frame->BitDepth(), &color_space); SkYUVAInfo yuva_info = SkYUVAInfo(video_size, plane_config, subsampling, color_space); SkYUVAPixmaps yuv_pixmap = SkYUVAPixmaps::FromExternalPixmaps(yuva_info, pixmaps.data()); std::unique_ptr ri_access = src_shared_image->BeginRasterAccess(ri, si_sync_token, /*readonly=*/false); ri->WritePixelsYUV(src_shared_image->mailbox(), yuv_pixmap); gpu::SyncToken ri_sync_token = gpu::RasterScopedAccess::EndAccess(std::move(ri_access)); std::unique_ptr src_ri_access = src_shared_image->BeginRasterAccess(ri, ri_sync_token, /*readonly=*/true); std::unique_ptr dst_ri_access = dest_shared_image->BeginRasterAccess(ri, dest_sync_token, /*readonly=*/false); ri->CopySharedImage(src_shared_image->mailbox(), dest_shared_image->mailbox(), 0, 0, source_rect.x(), source_rect.y(), source_rect.width(), source_rect.height()); gpu::RasterScopedAccess::EndAccess(std::move(dst_ri_access)); ri_sync_token = gpu::RasterScopedAccess::EndAccess(std::move(src_ri_access)); shared_image_cache->UpdateSyncToken(ri_sync_token); return ri_sync_token; } // Update |video_frame|'s release sync token to reflect the work done in |ri|, // and ensure that |video_frame| be kept remain alive until |ri|'s commands have // been completed. This is implemented for both gpu::gles2::GLES2Interface and // gpu::raster::RasterInterface. This function is critical to ensure that // |video_frame|'s resources not be returned until they are no longer in use. // https://crbug.com/819914 (software video decode frame corruption) // https://crbug.com/1237100 (camera capture reuse corruption) void SynchronizeVideoFrameRead( scoped_refptr video_frame, gpu::raster::RasterInterface* ri, gpu::ContextSupport* context_support, std::unique_ptr ri_access) { WaitAndReplaceSyncTokenClient client(ri, std::move(ri_access)); video_frame->UpdateReleaseSyncToken(&client); if (!video_frame->metadata().read_lock_fences_enabled) { return; } unsigned query_id = 0; ri->GenQueriesEXT(1, &query_id); DCHECK(query_id); ri->BeginQueryEXT(GL_COMMANDS_COMPLETED_CHROMIUM, query_id); ri->EndQueryEXT(GL_COMMANDS_COMPLETED_CHROMIUM); // |on_query_done_cb| will keep |video_frame| alive. auto on_query_done_cb = base::BindOnce( [](scoped_refptr video_frame, gpu::raster::RasterInterface* ri, unsigned query_id) { ri->DeleteQueriesEXT(1, &query_id); }, video_frame, ri, query_id); context_support->SignalQuery(query_id, std::move(on_query_done_cb)); } libyuv::FilterMode ToLibyuvFilterMode( PaintCanvasVideoRenderer::FilterMode filter) { switch (filter) { case PaintCanvasVideoRenderer::kFilterNone: return libyuv::kFilterNone; case PaintCanvasVideoRenderer::kFilterBilinear: return libyuv::kFilterBilinear; } } size_t NumConvertVideoFrameToRGBPixelsTasks(const VideoFrame* video_frame) { constexpr size_t kTaskBytes = 1024 * 1024; // 1 MiB const size_t frame_size = VideoFrame::AllocationSize( video_frame->format(), video_frame->visible_rect().size()); const size_t n_tasks = std::max(1, frame_size / kTaskBytes); return std::min(n_tasks, base::SysInfo::NumberOfProcessors()); } void ConvertVideoFrameToRGBPixelsTask(const VideoFrame* video_frame, const SkPixmap& dst_pixmap, libyuv::FilterMode filter, size_t task_index, size_t n_tasks, base::RepeatingClosure* done) { const VideoPixelFormat format = video_frame->format(); const int width = video_frame->visible_rect().width(); const int height = video_frame->visible_rect().height(); size_t rows_per_chunk = 1; for (size_t plane = 0; plane < VideoFrame::kMaxPlanes; ++plane) { if (VideoFrame::IsValidPlane(format, plane)) { rows_per_chunk = std::lcm(rows_per_chunk, VideoFrame::SampleSize(format, plane).height()); } } base::CheckedNumeric chunks = height / rows_per_chunk; const size_t chunk_start = (chunks * task_index / n_tasks).ValueOrDie(); const size_t chunk_end = (chunks * (task_index + 1) / n_tasks).ValueOrDie(); // Indivisible heights must process any remaining rows in the last task. size_t rows = (chunk_end - chunk_start) * rows_per_chunk; if (task_index + 1 == n_tasks) { rows += height % rows_per_chunk; } const auto chunk_subrect_of_visible_rect = SkIRect::MakeXYWH(0, chunk_start * rows_per_chunk, width, rows); struct PlaneMetaData { size_t stride; raw_ptr data; }; std::array plane_meta; for (size_t plane = 0; plane < VideoFrame::kMaxPlanes; ++plane) { if (VideoFrame::IsValidPlane(format, plane)) { plane_meta[plane] = { // Note: Unlike |data|, stride does not need to be adjusted by the // visible rect and sample size. Adding the full frame stride to a // pixel on row N and column M will wrap to column M on row N + 1. .stride = video_frame->stride(plane), .data = UNSAFE_TODO( video_frame->visible_data(plane) + video_frame->stride(plane) * (chunk_start * rows_per_chunk) / VideoFrame::SampleSize(format, plane).height())}; } else { plane_meta[plane] = {.stride = 0, .data = nullptr}; } } SkPixmap dst_chunk_pixmap; bool dst_extract_subset_result = dst_pixmap.extractSubset( &dst_chunk_pixmap, chunk_subrect_of_visible_rect); CHECK(dst_extract_subset_result); if (format == PIXEL_FORMAT_ARGB || format == PIXEL_FORMAT_XRGB || format == PIXEL_FORMAT_ABGR || format == PIXEL_FORMAT_XBGR || format == PIXEL_FORMAT_RGBAF16) { SkPixmap src_pm( SkImageInfo::Make(width, rows, SkColorTypeForPlane(format, VideoFrame::Plane::kARGB), media::IsOpaque(format) ? kOpaque_SkAlphaType : kUnpremul_SkAlphaType), plane_meta[VideoFrame::Plane::kARGB].data, plane_meta[VideoFrame::Plane::kARGB].stride); src_pm.readPixels(dst_chunk_pixmap); done->Run(); return; } // At this point, the dest must be N32 for YUV formats to write. CHECK_EQ(dst_pixmap.colorType(), kN32_SkColorType); uint8_t* const pixels = reinterpret_cast(dst_chunk_pixmap.writable_addr()); const size_t row_bytes = dst_chunk_pixmap.rowBytes(); const bool premultiply_alpha = dst_chunk_pixmap.alphaType() == kPremul_SkAlphaType; // TODO(crbug.com/41380578): This should default to BT.709 color space. auto yuv_cs = kRec601_SkYUVColorSpace; video_frame->ColorSpace().ToSkYUVColorSpace(video_frame->BitDepth(), &yuv_cs); const libyuv::YuvConstants* matrix = GetYuvContantsForColorSpace(yuv_cs, OUTPUT_ARGB); if (!video_frame->data(VideoFrame::Plane::kU) && !video_frame->data(VideoFrame::Plane::kV)) { DCHECK_EQ(format, PIXEL_FORMAT_I420); // For monochrome content ARGB and ABGR are interchangeable. libyuv::I400ToARGBMatrix(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, pixels, row_bytes, matrix, width, rows); done->Run(); return; } auto convert_yuv = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kU].data, plane_meta[VideoFrame::Plane::kU].stride, plane_meta[VideoFrame::Plane::kV].data, plane_meta[VideoFrame::Plane::kV].stride), pixels, row_bytes, matrix, width, rows); }; auto convert_yuv_with_filter = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kU].data, plane_meta[VideoFrame::Plane::kU].stride, plane_meta[VideoFrame::Plane::kV].data, plane_meta[VideoFrame::Plane::kV].stride), pixels, row_bytes, matrix, width, rows, filter); }; auto convert_yuva = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kU].data, plane_meta[VideoFrame::Plane::kU].stride, plane_meta[VideoFrame::Plane::kV].data, plane_meta[VideoFrame::Plane::kV].stride), plane_meta[VideoFrame::Plane::kA].data, plane_meta[VideoFrame::Plane::kA].stride, pixels, row_bytes, matrix, width, rows, premultiply_alpha); }; auto convert_yuva_with_filter = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kU].data, plane_meta[VideoFrame::Plane::kU].stride, plane_meta[VideoFrame::Plane::kV].data, plane_meta[VideoFrame::Plane::kV].stride), plane_meta[VideoFrame::Plane::kA].data, plane_meta[VideoFrame::Plane::kA].stride, pixels, row_bytes, matrix, width, rows, premultiply_alpha, filter); }; auto convert_yuv16 = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(reinterpret_cast( plane_meta[VideoFrame::Plane::kY].data.get()), plane_meta[VideoFrame::Plane::kY].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kU].data.get()), plane_meta[VideoFrame::Plane::kU].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kV].data.get()), plane_meta[VideoFrame::Plane::kV].stride / 2), pixels, row_bytes, matrix, width, rows); }; auto convert_yuv16_with_filter = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(reinterpret_cast( plane_meta[VideoFrame::Plane::kY].data.get()), plane_meta[VideoFrame::Plane::kY].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kU].data.get()), plane_meta[VideoFrame::Plane::kU].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kV].data.get()), plane_meta[VideoFrame::Plane::kV].stride / 2), pixels, row_bytes, matrix, width, rows, filter); }; auto convert_yuva16 = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(reinterpret_cast( plane_meta[VideoFrame::Plane::kY].data.get()), plane_meta[VideoFrame::Plane::kY].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kU].data.get()), plane_meta[VideoFrame::Plane::kU].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kV].data.get()), plane_meta[VideoFrame::Plane::kV].stride / 2), reinterpret_cast( plane_meta[VideoFrame::Plane::kA].data.get()), plane_meta[VideoFrame::Plane::kA].stride / 2, pixels, row_bytes, matrix, width, rows, premultiply_alpha); }; auto convert_yuva16_with_filter = [&](const libyuv::YuvConstants* matrix, auto&& func) { func(YUV_ORDER(reinterpret_cast( plane_meta[VideoFrame::Plane::kY].data.get()), plane_meta[VideoFrame::Plane::kY].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kU].data.get()), plane_meta[VideoFrame::Plane::kU].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kV].data.get()), plane_meta[VideoFrame::Plane::kV].stride / 2), reinterpret_cast( plane_meta[VideoFrame::Plane::kA].data.get()), plane_meta[VideoFrame::Plane::kA].stride / 2, pixels, row_bytes, matrix, width, rows, premultiply_alpha, filter); }; switch (format) { case PIXEL_FORMAT_YV12: case PIXEL_FORMAT_I420: convert_yuv_with_filter(matrix, libyuv::I420ToARGBMatrixFilter); break; case PIXEL_FORMAT_I422: convert_yuv_with_filter(matrix, libyuv::I422ToARGBMatrixFilter); break; case PIXEL_FORMAT_I444: // Special case for 4:4:4 RGB encoded videos. if (video_frame->ColorSpace().GetMatrixID() == gfx::ColorSpace::MatrixID::GBR) { libyuv::MergeARGBPlane( GBR_TO_RGB_ORDER(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kU].data, plane_meta[VideoFrame::Plane::kU].stride, plane_meta[VideoFrame::Plane::kV].data, plane_meta[VideoFrame::Plane::kV].stride), plane_meta[VideoFrame::Plane::kA].data, plane_meta[VideoFrame::Plane::kA].stride, pixels, row_bytes, width, rows); } else { convert_yuv(matrix, libyuv::I444ToARGBMatrix); } break; case PIXEL_FORMAT_YUV420P10: convert_yuv16_with_filter(matrix, libyuv::I010ToARGBMatrixFilter); break; case PIXEL_FORMAT_YUV422P10: convert_yuv16_with_filter(matrix, libyuv::I210ToARGBMatrixFilter); break; case PIXEL_FORMAT_YUV444P10: convert_yuv16(matrix, libyuv::I410ToARGBMatrix); break; case PIXEL_FORMAT_YUV420P12: convert_yuv16(matrix, libyuv::I012ToARGBMatrix); break; case PIXEL_FORMAT_I420A: convert_yuva_with_filter(matrix, libyuv::I420AlphaToARGBMatrixFilter); break; case PIXEL_FORMAT_I422A: convert_yuva_with_filter(matrix, libyuv::I422AlphaToARGBMatrixFilter); break; case PIXEL_FORMAT_I444A: convert_yuva(matrix, libyuv::I444AlphaToARGBMatrix); break; case PIXEL_FORMAT_YUV420AP10: convert_yuva16_with_filter(matrix, libyuv::I010AlphaToARGBMatrixFilter); break; case PIXEL_FORMAT_YUV422AP10: convert_yuva16_with_filter(matrix, libyuv::I210AlphaToARGBMatrixFilter); break; case PIXEL_FORMAT_YUV444AP10: convert_yuva16(matrix, libyuv::I410AlphaToARGBMatrix); break; case PIXEL_FORMAT_NV12: LIBYUV_NV12_TO_ARGB_MATRIX(plane_meta[VideoFrame::Plane::kY].data, plane_meta[VideoFrame::Plane::kY].stride, plane_meta[VideoFrame::Plane::kUV].data, plane_meta[VideoFrame::Plane::kUV].stride, pixels, row_bytes, matrix, width, rows); break; case PIXEL_FORMAT_P010LE: libyuv::P010ToARGBMatrix( reinterpret_cast( plane_meta[VideoFrame::Plane::kY].data.get()), plane_meta[VideoFrame::Plane::kY].stride / 2, reinterpret_cast( plane_meta[VideoFrame::Plane::kUV].data.get()), plane_meta[VideoFrame::Plane::kUV].stride / 2, pixels, row_bytes, matrix, width, rows); if (!OUTPUT_ARGB) { libyuv::ARGBToABGR(pixels, row_bytes, pixels, row_bytes, width, rows); } break; case PIXEL_FORMAT_YUV422P12: case PIXEL_FORMAT_YUV444P12: case PIXEL_FORMAT_Y16: NOTREACHED() << "These cases should be handled in ConvertVideoFrameToRGBPixels"; case PIXEL_FORMAT_UYVY: case PIXEL_FORMAT_NV21: case PIXEL_FORMAT_NV16: case PIXEL_FORMAT_NV24: case PIXEL_FORMAT_NV12A: case PIXEL_FORMAT_P210LE: case PIXEL_FORMAT_P410LE: case PIXEL_FORMAT_YUY2: case PIXEL_FORMAT_ARGB: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_XRGB: case PIXEL_FORMAT_RGB24: case PIXEL_FORMAT_MJPEG: case PIXEL_FORMAT_ABGR: case PIXEL_FORMAT_XBGR: case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_RGBAF16: case PIXEL_FORMAT_UNKNOWN: NOTREACHED() << "Only YUV formats and Y16 are supported, got: " << media::VideoPixelFormatToString(format); } done->Run(); } std::tuple VideoPixelFormatAsSkYUVAInfoValues(VideoPixelFormat format) { // The 9, 10, and 12 bit formats could be added here if GetYUVAPlanes() were // updated to convert data to unorm16/float16. Similarly, alpha planes and // formats with interleaved planes (e.g. NV12) could be supported if that // function were updated to not assume 3 separate Y, U, and V planes. Also, // GpuImageDecodeCache would need be able to handle plane configurations // other than 3 separate y, u, and v planes (crbug.com/910276). switch (format) { case PIXEL_FORMAT_I420: return {SkYUVAInfo::PlaneConfig::kY_U_V, SkYUVAInfo::Subsampling::k420}; case PIXEL_FORMAT_I422: return {SkYUVAInfo::PlaneConfig::kY_U_V, SkYUVAInfo::Subsampling::k422}; case PIXEL_FORMAT_I444: return {SkYUVAInfo::PlaneConfig::kY_U_V, SkYUVAInfo::Subsampling::k444}; default: return {SkYUVAInfo::PlaneConfig::kUnknown, SkYUVAInfo::Subsampling::kUnknown}; } } SkImageInfo GetVideoImageGeneratorSkImageInfo( const scoped_refptr& frame) { const auto frame_color_space = frame->CompatRGBColorSpace(); const auto color_type = frame->format() == PIXEL_FORMAT_RGBAF16 ? kRGBA_F16_SkColorType : kN32_SkColorType; const auto alpha_type = media::IsOpaque(frame->format()) ? kOpaque_SkAlphaType : kPremul_SkAlphaType; return SkImageInfo::Make(frame->visible_rect().width(), frame->visible_rect().height(), color_type, alpha_type, frame_color_space.ToSkColorSpace()); } } // anonymous namespace // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU. class VideoImageGenerator : public cc::PaintImageGenerator { public: VideoImageGenerator() = delete; explicit VideoImageGenerator(scoped_refptr frame) : cc::PaintImageGenerator(GetVideoImageGeneratorSkImageInfo(frame), frame->hdr_metadata()), frame_(std::move(frame)) { DCHECK(!frame_->HasSharedImage()); } VideoImageGenerator(const VideoImageGenerator&) = delete; VideoImageGenerator& operator=(const VideoImageGenerator&) = delete; ~VideoImageGenerator() override = default; sk_sp GetEncodedData() const override { return nullptr; } bool GetPixels(SkPixmap dst_pixmap, size_t frame_index, cc::PaintImage::GeneratorClientId client_id, uint32_t lazy_pixel_ref) override { DCHECK_EQ(frame_index, 0u); if (IsYuvPlanar(frame_->format()) && dst_pixmap.colorType() != kN32_SkColorType) { SkBitmap temp_bitmap; if (!temp_bitmap.tryAllocPixels( dst_pixmap.info() .makeColorType(kN32_SkColorType) .makeColorSpace(GetSkImageInfo().refColorSpace()))) { return false; } PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels( frame_.get(), UNSAFE_SKBITMAP_TO_BYTES_SPAN(temp_bitmap), temp_bitmap.rowBytes(), kN32_SkColorType); return temp_bitmap.pixmap().readPixels(dst_pixmap); } // If skia couldn't do the YUV conversion on GPU, we will on CPU. PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels( frame_.get(), UNSAFE_SKPIXMAP_TO_BYTES_SPAN(dst_pixmap), dst_pixmap.rowBytes(), dst_pixmap.colorType()); if (!SkColorSpace::Equals(GetSkImageInfo().colorSpace(), dst_pixmap.colorSpace())) { SkPixmap src(GetSkImageInfo(), dst_pixmap.writable_addr(), dst_pixmap.rowBytes()); if (!src.readPixels(dst_pixmap)) { return false; } } return true; } bool QueryYUVA(const SkYUVAPixmapInfo::SupportedDataTypes&, SkYUVAPixmapInfo* info) const override { SkYUVAInfo::PlaneConfig plane_config; SkYUVAInfo::Subsampling subsampling; std::tie(plane_config, subsampling) = VideoPixelFormatAsSkYUVAInfoValues(frame_->format()); if (plane_config == SkYUVAInfo::PlaneConfig::kUnknown) { return false; } // Don't use the YUV conversion path for multi-plane RGB frames. if (frame_->format() == PIXEL_FORMAT_I444 && frame_->ColorSpace().GetMatrixID() == gfx::ColorSpace::MatrixID::GBR) { return false; } if (!info) { return true; } SkYUVColorSpace yuv_color_space; if (!frame_->ColorSpace().ToSkYUVColorSpace(frame_->BitDepth(), &yuv_color_space)) { // TODO(crbug.com/41380578): This should default to BT.709 color space. yuv_color_space = kRec601_SkYUVColorSpace; } auto yuva_info = SkYUVAInfo( {frame_->visible_rect().width(), frame_->visible_rect().height()}, plane_config, subsampling, yuv_color_space); *info = SkYUVAPixmapInfo(yuva_info, SkYUVAPixmapInfo::DataType::kUnorm8, /*rowBytes=*/nullptr); return true; } bool GetYUVAPlanes(const SkYUVAPixmaps& pixmaps, size_t frame_index, uint32_t lazy_pixel_ref, cc::PaintImage::GeneratorClientId client_id) override { DCHECK_EQ(frame_index, 0u); DCHECK_EQ(pixmaps.numPlanes(), 3); #if DCHECK_IS_ON() SkYUVAInfo::PlaneConfig plane_config; SkYUVAInfo::Subsampling subsampling; std::tie(plane_config, subsampling) = VideoPixelFormatAsSkYUVAInfoValues(frame_->format()); DCHECK_EQ(plane_config, pixmaps.yuvaInfo().planeConfig()); DCHECK_EQ(subsampling, pixmaps.yuvaInfo().subsampling()); #endif for (int plane = VideoFrame::Plane::kY; plane <= VideoFrame::Plane::kV; ++plane) { const auto plane_size = VideoFrame::PlaneSizeInSamples( frame_->format(), plane, frame_->visible_rect().size()); if (plane_size.width() != pixmaps.plane(plane).width() || plane_size.height() != pixmaps.plane(plane).height()) { return false; } const auto& out_plane = pixmaps.plane(plane); // Copy the frame to the supplied memory. It'd be nice to avoid this copy, // but the memory is externally owned so we can't w/o an API change. libyuv::CopyPlane(frame_->visible_data(plane), frame_->stride(plane), reinterpret_cast(out_plane.writable_addr()), out_plane.rowBytes(), plane_size.width(), plane_size.height()); } return true; } private: scoped_refptr frame_; }; class VideoTextureBacking : public cc::TextureBacking { public: explicit VideoTextureBacking( scoped_refptr raster_context_provider, const gfx::Size& coded_size, const gfx::ColorSpace& color_space) : shared_image_( raster_context_provider->SharedImageInterface()->CreateSharedImage( {SHARED_IMAGE_FORMAT, coded_size, color_space, gpu::SHARED_IMAGE_USAGE_GLES2_READ | gpu::SHARED_IMAGE_USAGE_RASTER_READ | gpu::SHARED_IMAGE_USAGE_RASTER_WRITE, "PaintCanvasVideoRenderer"}, gpu::kNullSurfaceHandle)), sk_image_info_(SkImageInfo::Make(gfx::SizeToSkISize(coded_size), kRGBA_8888_SkColorType, shared_image_->alpha_type(), color_space.ToSkColorSpace())) { raster_context_provider_ = std::move(raster_context_provider); CHECK(shared_image_); sync_token_ = shared_image_->creation_sync_token(); } VideoTextureBacking(VideoTextureBacking&& other) : shared_image_(std::move(other.shared_image_)), sk_image_info_(std::move(other.sk_image_info_)), sync_token_(other.sync_token_), acquired_(true) { if (other.ri_access_) { sync_token_ = gpu::RasterScopedAccess::EndAccess(std::move(other.ri_access_)); } other.raster_context_provider_ = nullptr; other.sync_token_.Clear(); } void Bind(scoped_refptr context) override { CHECK(!acquired_ || !raster_context_provider_); // If this has not been passed across threads then binding is unnecessary. if (!acquired_) { return; } scoped_refptr raster_context_provider; if (context) { auto* wrapper = static_cast(context.get()); raster_context_provider = wrapper->provider(); } if (raster_context_provider == raster_context_provider_) { return; } raster_context_provider_ = std::move(raster_context_provider); if (raster_context_provider_) { BeginAccess(); } } void Unbind() override { if (acquired_) { clear_access(); raster_context_provider_ = nullptr; } } ~VideoTextureBacking() override { CHECK(!acquired_ || !ri_access_); if (ri_access_) { gpu::SyncToken sync_token = gpu::RasterScopedAccess::EndAccess(std::move(ri_access_)); shared_image_->UpdateDestructionSyncToken(sync_token); } } const SkImageInfo& GetSkImageInfo() override { return sk_image_info_; } gpu::Mailbox GetMailbox() const override { return shared_image_->mailbox(); } const scoped_refptr& GetSharedImage() const { return shared_image_; } const scoped_refptr& raster_context_provider() const { return raster_context_provider_; } void BeginAccess() { CHECK(raster_context_provider_); CHECK(!ri_access_); auto* ri = raster_context_provider_->RasterInterface(); ri_access_ = shared_image_->BeginRasterAccess(ri, sync_token_, /*readonly=*/true); } void clear_access() { if (ri_access_) { sync_token_ = gpu::RasterScopedAccess::EndAccess(std::move(ri_access_)); } } sk_sp GetSkImageViaReadback() override { sk_sp image_pixels = SkData::MakeUninitialized(sk_image_info_.computeMinByteSize()); if (!readPixels(sk_image_info_, image_pixels->writable_data(), sk_image_info_.minRowBytes(), 0, 0)) { DLOG(ERROR) << "VideoTextureBacking::GetSkImageViaReadback failed."; return nullptr; } return SkImages::RasterFromData(sk_image_info_, std::move(image_pixels), sk_image_info_.minRowBytes()); } bool readPixels(const SkImageInfo& dst_info, void* dst_pixels, size_t dst_row_bytes, int src_x, int src_y) override { CHECK(raster_context_provider_); gpu::raster::RasterInterface* ri = raster_context_provider_->RasterInterface(); return ri->ReadbackImagePixels(shared_image_->mailbox(), dst_info, dst_info.minRowBytes(), src_x, src_y, /*plane_index=*/0, dst_pixels); } const gpu::SyncToken& sync_token() { return sync_token_; } void UpdateSyncToken(const gpu::SyncToken& sync_token) { sync_token_ = sync_token; } private: // This is a newly allocated shared image if a copy or conversion was // necessary. scoped_refptr shared_image_; const SkImageInfo sk_image_info_; scoped_refptr raster_context_provider_; std::unique_ptr ri_access_; gpu::SyncToken sync_token_; bool acquired_ = false; }; PaintCanvasVideoRenderer::PaintCanvasVideoRenderer() : renderer_stable_id_(cc::PaintImage::GetNextId()) {} PaintCanvasVideoRenderer::~PaintCanvasVideoRenderer() = default; void PaintCanvasVideoRenderer::Paint( scoped_refptr video_frame, cc::PaintCanvas* canvas, const cc::PaintFlags& flags, const PaintParams& params, viz::RasterContextProvider* raster_context_provider) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); if (flags.isFullyTransparent()) { return; } if (video_frame && video_frame->HasSharedImage()) { if (!raster_context_provider) { DLOG(ERROR) << "Can't render textured frames w/o viz::RasterContextProvider"; return; // Unable to get/create a shared main thread context. } } gfx::RectF dest_rect = params.dest_rect.value_or( video_frame ? gfx::RectF(gfx::SizeF(video_frame->visible_rect().size())) : gfx::RectF()); SkRect dest; dest.setLTRB(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); // Paint black rectangle if there isn't a frame available or the // frame has an unexpected format. if (!video_frame.get() || video_frame->natural_size().IsEmpty() || !(media::IsYuvPlanar(video_frame->format()) || video_frame->format() == PIXEL_FORMAT_Y16 || video_frame->format() == PIXEL_FORMAT_ARGB || video_frame->format() == PIXEL_FORMAT_XRGB || video_frame->format() == PIXEL_FORMAT_ABGR || video_frame->format() == PIXEL_FORMAT_XBGR || video_frame->format() == PIXEL_FORMAT_RGBAF16 || video_frame->HasSharedImage())) { cc::PaintFlags black_with_alpha_flags; black_with_alpha_flags.setAlphaf(flags.getAlphaf()); canvas->drawRect(dest, black_with_alpha_flags); canvas->flush(); return; } // We want to be able to cache the PaintImage, to avoid redundant readbacks if // the canvas is software. // We do not need to synchronize video frame read here since it's already // taken care of in UpdateLastImage(). if (!UpdateLastImage(video_frame, raster_context_provider)) { return; } DCHECK(cache_); cc::PaintImage image = cache_->paint_image; if (params.reinterpret_as_srgb) { image = cc::PaintImageBuilder::WithCopy(image) .set_reinterpret_as_srgb(true) .TakePaintImage(); } if (params.acquire_texture_backing && cache_->texture_backing) { cc::PaintImage::ContentId content_id = image.GetContentIdForFrame(cc::PaintImage::kDefaultFrameIndex); image = cc::PaintImageBuilder::WithCopy(image) .set_texture_backing(sk_make_sp( std::move(*cache_->texture_backing)), content_id) .TakePaintImage(); ResetCache(); } DCHECK(image); cc::PaintFlags video_flags; video_flags.setAlphaf(flags.getAlphaf()); video_flags.setBlendMode(flags.getBlendMode()); video_flags.setTargetedHdrHeadroom(flags.getTargetedHdrHeadroom()); const bool need_rotation = params.transformation.rotation != VIDEO_ROTATION_0; const bool need_scaling = dest_rect.size() != gfx::SizeF(video_frame->visible_rect().size()); const bool need_translation = !dest_rect.origin().IsOrigin(); const bool needs_mirror = params.transformation.mirrored; bool need_transform = need_rotation || need_scaling || need_translation || needs_mirror; if (need_transform) { canvas->save(); canvas->translate( SkFloatToScalar(dest_rect.x() + (dest_rect.width() * 0.5f)), SkFloatToScalar(dest_rect.y() + (dest_rect.height() * 0.5f))); SkScalar angle = SkFloatToScalar(0.0f); switch (params.transformation.rotation) { case VIDEO_ROTATION_0: break; case VIDEO_ROTATION_90: angle = SkFloatToScalar(90.0f); break; case VIDEO_ROTATION_180: angle = SkFloatToScalar(180.0f); break; case VIDEO_ROTATION_270: angle = SkFloatToScalar(270.0f); break; } canvas->rotate(angle); gfx::SizeF rotated_dest_size = dest_rect.size(); const bool has_flipped_size = params.transformation.IsOrthogonal(); if (has_flipped_size) { rotated_dest_size = gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width()); } auto sx = SkFloatToScalar(rotated_dest_size.width() / video_frame->visible_rect().width()); auto sy = SkFloatToScalar(rotated_dest_size.height() / video_frame->visible_rect().height()); if (needs_mirror) { if (has_flipped_size) { sy *= -1; } else { sx *= -1; } } canvas->scale(sx, sy); canvas->translate( -SkFloatToScalar(video_frame->visible_rect().width() * 0.5f), -SkFloatToScalar(video_frame->visible_rect().height() * 0.5f)); } SkImageInfo info; size_t row_bytes; SkIPoint origin; base::span pixels = UNSAFE_PAINTCANVAS_TOP_LAYER_TO_BYTES_SPAN( canvas, &info, &row_bytes, &origin); // This if is a special handling of video for SkiaPaintcanvas backend, where // the video does not need any transform and it is enough to draw the frame // directly into the skia canvas const SkColorType canvas_color_type = info.colorType(); const bool is_yuv = media::IsYuvPlanar(video_frame->format()); const bool color_type_compatible = is_yuv ? (canvas_color_type == kN32_SkColorType) : (canvas_color_type == kBGRA_8888_SkColorType || canvas_color_type == kRGBA_8888_SkColorType); if (!need_transform && !params.reinterpret_as_srgb && video_frame->HasDirectCpuAccess() && flags.isOpaque() && flags.getBlendMode() == SkBlendMode::kSrc && flags.getFilterQuality() == cc::PaintFlags::FilterQuality::kLow && !pixels.empty() && color_type_compatible) { const size_t offset = info.computeOffset(origin.x(), origin.y(), row_bytes); ConvertVideoFrameToRGBPixels(video_frame.get(), pixels.subspan(offset), row_bytes, canvas_color_type); } else if (video_frame->HasSharedImage()) { DCHECK_EQ(video_frame->coded_size(), gfx::Size(image.width(), image.height())); canvas->drawImageRect(image, gfx::RectToSkRect(video_frame->visible_rect()), SkRect::MakeWH(video_frame->visible_rect().width(), video_frame->visible_rect().height()), cc::PaintFlags::FilterQualityToSkSamplingOptions( flags.getFilterQuality()), &video_flags, SkCanvas::kStrict_SrcRectConstraint); } else { DCHECK_EQ(video_frame->visible_rect().size(), gfx::Size(image.width(), image.height())); canvas->drawImage(image, 0, 0, cc::PaintFlags::FilterQualityToSkSamplingOptions( flags.getFilterQuality()), &video_flags); } if (need_transform) { canvas->restore(); } // Make sure to flush so we can remove the videoframe from the generator. canvas->flush(); } void PaintCanvasVideoRenderer::Copy( scoped_refptr video_frame, cc::PaintCanvas* canvas, viz::RasterContextProvider* raster_context_provider) { cc::PaintFlags flags; flags.setBlendMode(SkBlendMode::kSrc); flags.setFilterQuality(cc::PaintFlags::FilterQuality::kLow); Paint(std::move(video_frame), canvas, flags, PaintParams(), raster_context_provider); } namespace { // libyuv doesn't support all 9-, 10- nor 12-bit pixel formats yet. This // function creates a regular 8-bit video frame which we can give to libyuv. scoped_refptr DownShiftHighbitVideoFrame( const VideoFrame* video_frame) { VideoPixelFormat format; switch (video_frame->format()) { case PIXEL_FORMAT_YUV420P12: case PIXEL_FORMAT_YUV420P10: format = PIXEL_FORMAT_I420; break; case PIXEL_FORMAT_YUV422P12: case PIXEL_FORMAT_YUV422P10: format = PIXEL_FORMAT_I422; break; case PIXEL_FORMAT_YUV444P12: case PIXEL_FORMAT_YUV444P10: format = PIXEL_FORMAT_I444; break; default: NOTREACHED(); } const int scale = 1 << (24 - video_frame->BitDepth()); scoped_refptr ret = VideoFrame::CreateFrame( format, video_frame->coded_size(), video_frame->visible_rect(), video_frame->natural_size(), video_frame->timestamp()); if (!ret) { return nullptr; } ret->set_color_space(video_frame->ColorSpace()); // Copy all metadata. // (May be enough to copy color space) ret->metadata().MergeMetadataFrom(video_frame->metadata()); for (int plane = VideoFrame::Plane::kY; plane <= VideoFrame::Plane::kV; ++plane) { int width = VideoFrame::Columns(plane, video_frame->format(), video_frame->visible_rect().width()); int height = VideoFrame::Rows(plane, video_frame->format(), video_frame->visible_rect().height()); const uint16_t* src = reinterpret_cast(video_frame->visible_data(plane)); base::span dst = ret->GetWritableVisiblePlaneData(plane); if (!src) { // An AV1 monochrome (grayscale) frame has no U and V planes. Set all U // and V samples to the neutral value (128). DCHECK_NE(plane, VideoFrame::Plane::kY); std::ranges::fill(dst.first(height * ret->stride(plane)), 128); continue; } libyuv::Convert16To8Plane(src, video_frame->stride(plane) / 2, dst.data(), ret->stride(plane), scale, width, height); } return ret; } // Converts 16-bit data to |out| buffer of specified GL |type|. // When the |format| is RGBA, the converted value is fed as luminance. void FlipAndConvertY16(const VideoFrame* video_frame, base::span out, unsigned format, unsigned type, bool flip_y, size_t output_row_bytes) { base::span row_head = video_frame->GetVisiblePlaneData(0); const size_t stride = video_frame->stride(0); const int height = video_frame->visible_rect().height(); for (int i = 0; i < height; ++i) { base::span out_row = out.subspan( flip_y ? output_row_bytes * (height - i - 1) : output_row_bytes * i, output_row_bytes); // This code in `//media` is hot, so it's worth using // `reinterpret_span` to keep performance up. auto row = base::subtle::reinterpret_span(row_head.subspan( stride * i, video_frame->visible_rect().width() * sizeof(uint16_t))); if (type == GL_FLOAT) { auto out_row_float = base::subtle::reinterpret_span(out_row); if (format == GL_RGBA) { std::array temp_buffer = {}; temp_buffer[3] = 1.0f; for (uint16_t value : row) { const float gray_value = value / 65535.f; temp_buffer[0] = gray_value; temp_buffer[1] = gray_value; temp_buffer[2] = gray_value; out_row_float.take_first<4u>().copy_from_nonoverlapping(temp_buffer); } continue; } else if (format == GL_RED) { std::ranges::transform(row, out_row_float.begin(), [](uint16_t value) { return value / 65535.f; }); continue; } // For other formats, hit NOTREACHED below. } else if (type == GL_UNSIGNED_BYTE) { // We take the upper 8 bits of 16-bit data and convert it as luminance to // ARGB. We loose the precision here, but it is important not to render // Y16 as RG_88. To get the full precision use float textures with WebGL1 // and e.g. R16UI or R32F textures with WebGL2. DCHECK_EQ(static_cast(GL_RGBA), format); // This code in `//media` is hot, so it's worth using // `reinterpret_span` to keep performance up. auto out_row_uint32 = base::subtle::reinterpret_span(out_row); std::ranges::transform(row, out_row_uint32.begin(), [](uint16_t value) { const uint32_t gray_value = value >> 8; return SkColorSetRGB(gray_value, gray_value, gray_value); }); continue; } NOTREACHED() << "Unsupported Y16 conversion for format: 0x" << std::hex << format << " and type: 0x" << std::hex << type; } } // Common functionality of PaintCanvasVideoRenderer's TexImage2D and // TexSubImage2D. Allocates a buffer required for conversion and converts // |frame| content to desired |format|. Returns true if calling glTex(Sub)Image // is supported for provided |frame| format and parameters. bool TexImageHelper(VideoFrame* frame, unsigned format, unsigned type, GrSurfaceOrigin dst_origin, scoped_refptr* temp_buffer) { unsigned output_bytes_per_pixel = 0; switch (frame->format()) { case PIXEL_FORMAT_Y16: // Converting single component unsigned short here to FLOAT luminance. switch (format) { case GL_RGBA: if (type == GL_FLOAT) { output_bytes_per_pixel = 4 * sizeof(GLfloat); break; } return false; case GL_RED: if (type == GL_FLOAT) { output_bytes_per_pixel = sizeof(GLfloat); break; } return false; default: return false; } break; default: return false; } // VideoFrame that isn't backed by shared image always top-left, so if // destination is bottom-left we need to flip. const bool flip_y = dst_origin != kTopLeft_GrSurfaceOrigin; size_t output_row_bytes = frame->visible_rect().width() * output_bytes_per_pixel; *temp_buffer = base::MakeRefCounted( output_row_bytes * frame->visible_rect().height()); FlipAndConvertY16(frame, (*temp_buffer)->writable_data(), format, type, flip_y, output_row_bytes); return true; } // Upload the |frame| data to temporary texture of |temp_format|, // |temp_internalformat| and |temp_type| and then copy intermediate texture // subimage to destination |texture|. The destination |texture| is bound to the // |target| before the call. void TextureSubImageUsingIntermediate(unsigned target, unsigned texture, gpu::gles2::GLES2Interface* gl, VideoFrame* frame, int temp_internalformat, unsigned temp_format, unsigned temp_type, int level, int xoffset, int yoffset, GrSurfaceOrigin dst_origin, SkAlphaType dst_alpha_type) { unsigned temp_texture = 0; gl->GenTextures(1, &temp_texture); gl->BindTexture(target, temp_texture); gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST); gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST); gl->TexImage2D(target, 0, temp_internalformat, frame->visible_rect().width(), frame->visible_rect().height(), 0, temp_format, temp_type, frame->visible_data(0)); gl->BindTexture(target, texture); // VideoFrame that is not backed by shared image has always top-left origin. // We uploaded data to `temp_texture` as is, so need to flip when copy to // destination if it's bottom left. const bool do_flip_y = dst_origin != kTopLeft_GrSurfaceOrigin; // VideoFrame data is not premultiplied, so we need to premultiply if // requested. const bool do_premultiply_alpha = dst_alpha_type == kPremul_SkAlphaType; gl->CopySubTextureCHROMIUM(temp_texture, 0, target, texture, level, 0, 0, xoffset, yoffset, frame->visible_rect().width(), frame->visible_rect().height(), do_flip_y, do_premultiply_alpha, false); gl->DeleteTextures(1, &temp_texture); } } // anonymous namespace // static void PaintCanvasVideoRenderer::SynchronizeVideoFrameRead( scoped_refptr video_frame, gpu::gles2::GLES2Interface* gl, gpu::ContextSupport* context_support, base::OnceCallback sync_callback) { WaitAndReplaceSyncTokenClient client(gl, std::move(sync_callback)); video_frame->UpdateReleaseSyncToken(&client); if (!video_frame->metadata().read_lock_fences_enabled) { return; } unsigned query_id = 0; gl->GenQueriesEXT(1, &query_id); DCHECK(query_id); gl->BeginQueryEXT(GL_COMMANDS_COMPLETED_CHROMIUM, query_id); gl->EndQueryEXT(GL_COMMANDS_COMPLETED_CHROMIUM); // |on_query_done_cb| will keep |video_frame| alive. auto on_query_done_cb = base::DoNothingWithBoundArgs(video_frame); context_support->SignalQuery(query_id, std::move(on_query_done_cb)); // Delete the query immediately. This will cause |on_query_done_cb| to be // issued prematurely. The alternative, deleting |query_id| within // |on_query_done_cb|, can cause a crash because |gl| may not still exist // at the time of the callback. This is incorrect behavior. // https://crbug.com/1243763 gl->DeleteQueriesEXT(1, &query_id); } void PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels( const VideoFrame* video_frame, base::span rgb_pixels, size_t row_bytes, SkColorType dst_color_type, bool premultiply_alpha, FilterMode filter, bool disable_threading) { if (!video_frame->HasDirectCpuAccess()) { NOTREACHED() << "Cannot extract pixels from non-CPU frame formats."; } scoped_refptr temporary_frame; // TODO(thomasanderson): Parallelize converting these formats. switch (video_frame->format()) { case PIXEL_FORMAT_YUV422P12: case PIXEL_FORMAT_YUV444P12: temporary_frame = DownShiftHighbitVideoFrame(video_frame); video_frame = temporary_frame.get(); break; case PIXEL_FORMAT_YUV420P10: case PIXEL_FORMAT_YUV420P12: // In AV1, a monochrome (grayscale) frame is represented as a YUV 4:2:0 // frame with no U and V planes. Since there are no 10-bit and 12-bit // versions of libyuv::I400ToARGBMatrix(), convert the frame to an 8-bit // YUV 4:2:0 frame with U and V planes. if (!video_frame->data(VideoFrame::Plane::kU) && !video_frame->data(VideoFrame::Plane::kV)) { temporary_frame = DownShiftHighbitVideoFrame(video_frame); video_frame = temporary_frame.get(); } break; case PIXEL_FORMAT_Y16: // Since it is grayscale conversion, we disregard // SK_PMCOLOR_BYTE_ORDER and always use GL_RGBA. FlipAndConvertY16(video_frame, rgb_pixels, GL_RGBA, GL_UNSIGNED_BYTE, false /*flip_y*/, row_bytes); return; default: break; } // DownShiftHighbitVideoFrame may fail creation of the `temporary_frame`. if (!video_frame) { return; } const size_t n_tasks = disable_threading ? 1 : NumConvertVideoFrameToRGBPixelsTasks(video_frame); base::WaitableEvent event; base::RepeatingClosure barrier = base::BarrierClosure( n_tasks, base::BindOnce(&base::WaitableEvent::Signal, base::Unretained(&event))); SkPixmap dst_pixmap( SkImageInfo::Make( gfx::SizeToSkISize(video_frame->visible_rect().size()), dst_color_type, premultiply_alpha ? kPremul_SkAlphaType : kUnpremul_SkAlphaType), rgb_pixels.data(), row_bytes); const libyuv::FilterMode libyuv_filter = ToLibyuvFilterMode(filter); for (size_t i = 1; i < n_tasks; ++i) { base::ThreadPool::PostTask( FROM_HERE, base::BindOnce(ConvertVideoFrameToRGBPixelsTask, base::Unretained(video_frame), dst_pixmap, libyuv_filter, i, n_tasks, &barrier)); } ConvertVideoFrameToRGBPixelsTask(video_frame, dst_pixmap, libyuv_filter, 0, n_tasks, &barrier); { base::ScopedAllowBaseSyncPrimitivesOutsideBlockingScope allow_wait; event.Wait(); } } // static viz::SharedImageFormat PaintCanvasVideoRenderer::GetRGBPixelsOutputFormat() { return SHARED_IMAGE_FORMAT; } // static bool PaintCanvasVideoRenderer::MultiPlaneChannelFormatSupported( viz::RasterContextProvider* raster_context_provider, viz::SharedImageFormat::ChannelFormat channel_format) { const auto& caps = raster_context_provider->ContextCapabilities(); const auto& shared_image_caps = raster_context_provider->SharedImageInterface()->GetCapabilities(); // If one-component shared images or RG textures are unsupported, then no // channel formats are supported. if (shared_image_caps.disable_one_component_textures || !caps.texture_rg) { return false; } switch (channel_format) { case viz::SharedImageFormat::ChannelFormat::k8: return true; case viz::SharedImageFormat::ChannelFormat::k10: case viz::SharedImageFormat::ChannelFormat::k16: return caps.texture_norm16; case viz::SharedImageFormat::ChannelFormat::k16F: return shared_image_caps.is_r16f_supported; } } // static bool PaintCanvasVideoRenderer:: IsPixelFormatSupportedForYuvSharedImageConversion( VideoPixelFormat video_format) { // To expand support for additional VideoFormats expand this switch. switch (video_format) { case PIXEL_FORMAT_NV12: case PIXEL_FORMAT_P010LE: case PIXEL_FORMAT_NV16: case PIXEL_FORMAT_P210LE: case PIXEL_FORMAT_NV24: case PIXEL_FORMAT_P410LE: case PIXEL_FORMAT_NV12A: case PIXEL_FORMAT_I420: case PIXEL_FORMAT_I420A: return true; case PIXEL_FORMAT_YV12: case PIXEL_FORMAT_I422: case PIXEL_FORMAT_I444: case PIXEL_FORMAT_YUV420P10: case PIXEL_FORMAT_YUV422P10: case PIXEL_FORMAT_YUV444P10: case PIXEL_FORMAT_YUV420P12: case PIXEL_FORMAT_YUV422P12: case PIXEL_FORMAT_YUV444P12: case PIXEL_FORMAT_ARGB: case PIXEL_FORMAT_XRGB: case PIXEL_FORMAT_ABGR: case PIXEL_FORMAT_XBGR: case PIXEL_FORMAT_NV21: case PIXEL_FORMAT_UYVY: case PIXEL_FORMAT_YUY2: case PIXEL_FORMAT_RGB24: case PIXEL_FORMAT_MJPEG: case PIXEL_FORMAT_Y16: case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_RGBAF16: case PIXEL_FORMAT_I422A: case PIXEL_FORMAT_I444A: case PIXEL_FORMAT_YUV420AP10: case PIXEL_FORMAT_YUV422AP10: case PIXEL_FORMAT_YUV444AP10: case PIXEL_FORMAT_UNKNOWN: return false; } } bool PaintCanvasVideoRenderer::CopyVideoFrameYUVDataToGLTexture( viz::RasterContextProvider* raster_context_provider, gpu::gles2::GLES2Interface* destination_gl, scoped_refptr video_frame, VideoFrameSharedImageCache* rgb_si_cache, VideoFrameSharedImageCache* yuv_si_cache, unsigned int target, unsigned int texture, unsigned int internal_format, unsigned int format, unsigned int type, int level, SkAlphaType dst_alpha_type, GrSurfaceOrigin dst_origin) { if (!raster_context_provider) { return false; } #if BUILDFLAG(IS_ANDROID) // TODO(crbug.com/40751207): These formats don't work with the passthrough // command decoder on Android for some reason. const auto format_enum = static_cast(internal_format); if (format_enum == GL_RGB10_A2 || format_enum == GL_RGB565 || format_enum == GL_RGB5_A1 || format_enum == GL_RGBA4) { return false; } #endif if (!video_frame->HasDirectCpuAccess()) { return false; } if (!IsPixelFormatSupportedForYuvSharedImageConversion( video_frame->format())) { return false; } // Could handle NV12 here as well. See NewSkImageFromVideoFrameYUV. CHECK(!video_frame->HasSharedImage()); // If a suitable pixel format for the frame's planes is not present, then // YUV to RGB conversion must be done on the CPU. auto shared_image_format = VideoPixelFormatToSharedImageFormat(video_frame->format()); if (!shared_image_format.has_value()) { return false; } if (shared_image_format->is_multi_plane() && !MultiPlaneChannelFormatSupported( raster_context_provider, shared_image_format->channel_format())) { return false; } // We copy the contents of the source VideoFrame into the intermediate SI // over the raster interface and read out the contents of the intermediate // SI into the destination GL texture via the GLES2 interface. gpu::SharedImageUsageSet src_usage = gpu::SHARED_IMAGE_USAGE_RASTER_WRITE | gpu::SHARED_IMAGE_USAGE_GLES2_READ; CHECK(rgb_si_cache); CHECK(yuv_si_cache); // We need a shared image to receive the intermediate RGB result. Try to reuse // one if compatible, otherwise create a new one. auto [rgb_shared_image, rgb_sync_token, status] = rgb_si_cache->GetOrCreateSharedImage( video_frame.get(), raster_context_provider, src_usage, SHARED_IMAGE_FORMAT, kPremul_SkAlphaType, video_frame->CompatRGBColorSpace()); CHECK(rgb_shared_image); // On the source Raster context, do the YUV->RGB conversion. // Pass the rgb sync token here to be waited upon before performing raster // tasks. gpu::SyncToken post_conversion_sync_token = CopyVideoFrameToSharedImage( raster_context_provider, video_frame, rgb_shared_image, rgb_sync_token, /*use_visible_rect=*/false, yuv_si_cache); // On the destination GL context, do a copy (with cropping) into the // destination texture. destination_gl->BindTexture(target, texture); destination_gl->TexImage2D( target, level, internal_format, video_frame->visible_rect().width(), video_frame->visible_rect().height(), 0, format, type, nullptr); base::OnceCallback sync_callback = destination_gl->CopySharedImageToGLTextureViaTextureCopy( video_frame->visible_rect(), rgb_shared_image.get(), post_conversion_sync_token, target, texture, internal_format, format, type, level, dst_alpha_type, dst_origin); // Update the rgb sync token to be waited upon based on gles tasks performed // earlier. rgb_si_cache->UpdateSyncToken(std::move(sync_callback).Run()); // video_frame->UpdateReleaseSyncToken is not necessary since the video frame // data we used was CPU-side to begin with. If there were any textures, we // didn't use them. return true; } bool PaintCanvasVideoRenderer::TexImage2D( unsigned target, unsigned texture, gpu::gles2::GLES2Interface* gl, const gpu::Capabilities& gpu_capabilities, VideoFrame* frame, int level, int internalformat, unsigned format, unsigned type, GrSurfaceOrigin dst_origin, SkAlphaType dst_alpha_type) { DCHECK(frame); DCHECK(!frame->HasSharedImage()); GLint precision = 0; GLint range[2] = {0, 0}; gl->GetShaderPrecisionFormat(GL_FRAGMENT_SHADER, GL_MEDIUM_FLOAT, range, &precision); // Note: CopyTextureCHROMIUM uses mediump for color computation. Don't use // it if the precision would lead to data loss when converting 16-bit // normalized to float. medium_float.precision > 15 means that the approach // below is not used on Android, where the extension EXT_texture_norm16 is // not widely supported. It is used on Windows, Linux and OSX. // Android support is not required for now because Tango depth camera already // provides floating point data (projected point cloud). See crbug.com/674440. if (gpu_capabilities.texture_norm16 && precision > 15 && target == GL_TEXTURE_2D && (type == GL_FLOAT || type == GL_UNSIGNED_BYTE)) { // TODO(aleksandar.stojiljkovic): Extend the approach to TexSubImage2D // implementation and other types. See https://crbug.com/624436. // Allocate the destination texture. gl->TexImage2D(target, level, internalformat, frame->visible_rect().width(), frame->visible_rect().height(), 0, format, type, nullptr); // We use sized internal format GL_R16_EXT instead of unsized GL_RED. // See angleproject:1952 TextureSubImageUsingIntermediate(target, texture, gl, frame, GL_R16_EXT, GL_RED, GL_UNSIGNED_SHORT, level, 0, 0, dst_origin, dst_alpha_type); return true; } scoped_refptr temp_buffer; if (!TexImageHelper(frame, format, type, dst_origin, &temp_buffer)) { return false; } gl->TexImage2D(target, level, internalformat, frame->visible_rect().width(), frame->visible_rect().height(), 0, format, type, temp_buffer->data().data()); return true; } bool PaintCanvasVideoRenderer::TexSubImage2D(unsigned target, gpu::gles2::GLES2Interface* gl, VideoFrame* frame, int level, unsigned format, unsigned type, int xoffset, int yoffset, GrSurfaceOrigin dst_origin, SkAlphaType dst_alpha_type) { DCHECK(frame); DCHECK(!frame->HasSharedImage()); scoped_refptr temp_buffer; if (!TexImageHelper(frame, format, type, dst_origin, &temp_buffer)) { return false; } gl->TexSubImage2D( target, level, xoffset, yoffset, frame->visible_rect().width(), frame->visible_rect().height(), format, type, temp_buffer->data().data()); return true; } void PaintCanvasVideoRenderer::ResetCache() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); cache_.reset(); // ClientSharedImage destructor calls DestroySharedImage which in turn ensures // that the deferred destroy request is flushed. Thus, clients don't need to // call SharedImageInterface::Flush explicitly. } PaintCanvasVideoRenderer::Cache::Cache(VideoFrame::ID frame_id, base::RepeatingClosure on_expire) : frame_id(frame_id), timer(FROM_HERE, kTemporaryResourceDeletionDelay, std::move(on_expire)) {} PaintCanvasVideoRenderer::Cache::~Cache() = default; bool PaintCanvasVideoRenderer::Cache::Recycle() { paint_image = cc::PaintImage(); return texture_backing->unique(); } void PaintCanvasVideoRenderer::OnCacheExpired() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); cache_.reset(); } bool PaintCanvasVideoRenderer::UpdateLastImage( scoped_refptr video_frame, viz::RasterContextProvider* raster_context_provider) { // Check for a cache hit. if (cache_ && video_frame->unique_id() == cache_->frame_id && cache_->paint_image) { cache_->timer.Reset(); return true; } auto paint_image_builder = cc::PaintImageBuilder::WithDefault() .set_id(renderer_stable_id_) .set_animation_type(cc::PaintImage::AnimationType::kVideo) .set_completion_state(cc::PaintImage::CompletionState::kDone) .set_hdr_metadata(video_frame->hdr_metadata()); // Generate a new image. // Note: Skia will hold onto |video_frame| via |video_generator| only when // |video_frame| is software. // Holding |video_frame| longer than this call when using GPUVideoDecoder // could cause problems since the pool of VideoFrames has a fixed size. if (video_frame->HasSharedImage()) { CHECK(raster_context_provider); auto* ri = raster_context_provider->RasterInterface(); DCHECK(ri); const auto video_frame_si = video_frame->shared_image(); // Create or reuse a texture backing for the cached copy. if (cache_ && cache_->texture_backing && cache_->texture_backing->raster_context_provider() == raster_context_provider && cache_->coded_size == video_frame->coded_size() && cache_->Recycle()) { // We can reuse the shared image from the previous cache. cache_->frame_id = video_frame->unique_id(); // NOTE: It is necessary to let go of read access to the cached copy // here because the below copy operation takes readwrite access to that // cached copy, and requesting RW access while already holding R access // on a single service-side texture causes a DCHECK to fire. cache_->texture_backing->clear_access(); } else { cache_.emplace( video_frame->unique_id(), base::BindRepeating(&PaintCanvasVideoRenderer::OnCacheExpired, base::Unretained(this))); cache_->texture_backing = sk_make_sp( raster_context_provider, video_frame->coded_size(), video_frame->CompatRGBColorSpace()); } // Copy into the shared image backing of the cached copy. gpu::SyncToken sync_token = CopyVideoFrameToSharedImage(raster_context_provider, video_frame, cache_->texture_backing->GetSharedImage(), cache_->texture_backing->sync_token(), /*use_visible_rect=*/false); cache_->texture_backing->UpdateSyncToken(sync_token); cache_->coded_size = video_frame->coded_size(); paint_image_builder.set_texture_backing(cache_->texture_backing, cc::PaintImage::GetNextContentId()); cache_->texture_backing->BeginAccess(); } else { cache_.emplace( video_frame->unique_id(), base::BindRepeating(&PaintCanvasVideoRenderer::OnCacheExpired, base::Unretained(this))); paint_image_builder.set_paint_image_generator( sk_make_sp(video_frame)); } cache_->paint_image = paint_image_builder.TakePaintImage(); if (!cache_->paint_image) { // Couldn't create the PaintImage. cache_.reset(); return false; } cache_->timer.Reset(); return true; } bool PaintCanvasVideoRenderer::CanUseCopyVideoFrameToSharedImage( const VideoFrame& video_frame) { return video_frame.HasSharedImage() || IsPixelFormatSupportedForYuvSharedImageConversion( video_frame.format()); } gpu::SyncToken PaintCanvasVideoRenderer::CopyVideoFrameToSharedImage( viz::RasterContextProvider* raster_context_provider, scoped_refptr video_frame, scoped_refptr dest_shared_image, const gpu::SyncToken& dest_sync_token, bool use_visible_rect, VideoFrameSharedImageCache* yuv_shared_image_cache /*=nullptr*/) { auto* ri = raster_context_provider->RasterInterface(); gpu::SyncToken sync_token; // If we have single source shared image, just use CopySharedImage(). if (video_frame->HasSharedImage()) { auto source_rect = use_visible_rect ? video_frame->visible_rect() : gfx::Rect(video_frame->coded_size()); std::unique_ptr dst_ri_access = dest_shared_image->BeginRasterAccess(ri, dest_sync_token, /*readonly=*/false); std::unique_ptr src_ri_access = video_frame->shared_image()->BeginRasterAccess( ri, video_frame->acquire_sync_token(), /*readonly=*/true); ri->CopySharedImage(video_frame->shared_image()->mailbox(), dest_shared_image->mailbox(), 0, 0, source_rect.x(), source_rect.y(), source_rect.width(), source_rect.height()); sync_token = gpu::RasterScopedAccess::EndAccess(std::move(dst_ri_access)); // If VideoFrame has textures, we need to update SyncToken or to keep frame // alive until gpu is done with copy if `read_lock_fences_enabled` is set. // This is to make sure decoder doesn't re-use frame before copy is done. ::media::SynchronizeVideoFrameRead( std::move(video_frame), ri, raster_context_provider->ContextSupport(), std::move(src_ri_access)); } else { sync_token = ConvertYuvVideoFrameToRgbSharedImage( video_frame.get(), raster_context_provider, dest_shared_image, dest_sync_token, use_visible_rect, yuv_shared_image_cache); } return sync_token; } gfx::Size PaintCanvasVideoRenderer::LastImageDimensionsForTesting() { DCHECK(cache_); DCHECK(cache_->paint_image); return gfx::Size(cache_->paint_image.width(), cache_->paint_image.height()); } } // namespace media