From c2a9a9716ec72a8dd20fa40d055782444ba5c491 Mon Sep 17 00:00:00 2001 From: Brian Salomon Date: Tue, 15 Sep 2020 11:24:28 -0400 Subject: [PATCH] New version of SkImage::MakeFromYUVAPixmaps() Takes SkYUVAPixmaps. Still implemented in terms of SkYUVAIndex[4] internally. Replace all internal use cases except wacky_yuv_formats. Takes GrRecordingContext rather than GrContext. SkVideoDecoder updated to take GrRecordingContext. Bug: skia:10632 Change-Id: I6e9b9b6a4f11333bce6f87c1ebff0acb297f6540 Reviewed-on: https://skia-review.googlesource.com/c/skia/+/316837 Commit-Queue: Brian Salomon Reviewed-by: Adlai Holler Reviewed-by: Robert Phillips --- RELEASE_NOTES.txt | 6 +++ experimental/ffmpeg/SkVideoDecoder.cpp | 43 +++++++-------- experimental/ffmpeg/SkVideoDecoder.h | 6 +-- gm/video_decoder.cpp | 6 +-- include/core/SkImage.h | 30 +++++++++++ include/core/SkYUVAPixmaps.h | 2 +- src/core/SkYUVAPixmaps.cpp | 27 +++++----- src/image/SkImage.cpp | 8 +++ src/image/SkImage_GpuYUVA.cpp | 75 ++++++++++++++++++++++++-- tests/ImageTest.cpp | 11 ++-- tools/gpu/YUVUtils.cpp | 9 +--- 11 files changed, 160 insertions(+), 63 deletions(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index edf7e7c9cc..c4c59e6e2c 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -9,6 +9,12 @@ Milestone 87 * + * Alternate SkImage::MakeFromYUVAPixmaps signature. Takes SkYUVAPixmaps, which specifies + planar configuration in a more structured manner. Currently limited to tri-planar + configurations without alpha but will be expanded. Older signature will become + deprecated when the new signature supports a broad range of planar configurations. + https://review.skia.org/316837 + * Switch to using Microsoft::WRL::ComPtr for wrapping D3D12 objects. https://review.skia.org/314957 diff --git a/experimental/ffmpeg/SkVideoDecoder.cpp b/experimental/ffmpeg/SkVideoDecoder.cpp index 835a9b50f3..2d46f66185 100644 --- a/experimental/ffmpeg/SkVideoDecoder.cpp +++ b/experimental/ffmpeg/SkVideoDecoder.cpp @@ -9,6 +9,7 @@ #include "include/core/SkColorSpace.h" #include "include/core/SkImage.h" #include "include/core/SkYUVAIndex.h" +#include "include/core/SkYUVAPixmaps.h" static SkYUVColorSpace get_yuvspace(AVColorSpace space) { // this is pretty incomplete -- TODO: look to convert more AVColorSpaces @@ -159,29 +160,23 @@ static int64_t skstream_seek_packet(void* ctx, int64_t pos, int whence) { return stream->seek(SkToSizeT(pos)) ? pos : -1; } -static sk_sp make_yuv_420(GrContext* gr, int w, int h, - uint8_t* const data[], int const strides[], - SkYUVColorSpace yuv_space, +static sk_sp make_yuv_420(GrRecordingContext* rContext, + int w, int h, + uint8_t* const data[], + int const strides[], + SkYUVColorSpace yuvSpace, sk_sp cs) { - SkImageInfo info[3]; - info[0] = SkImageInfo::Make(w, h, kGray_8_SkColorType, kOpaque_SkAlphaType); - info[1] = SkImageInfo::Make(w/2, h/2, kGray_8_SkColorType, kOpaque_SkAlphaType); - info[2] = SkImageInfo::Make(w/2, h/2, kGray_8_SkColorType, kOpaque_SkAlphaType); + SkYUVAInfo yuvaInfo({w, h}, SkYUVAInfo::PlanarConfig::kY_U_V_420, yuvSpace); + SkPixmap pixmaps[3]; + pixmaps[0].reset(SkImageInfo::MakeA8(w, h), data[0], strides[0]); + w = (w + 1)/2; + h = (h + 1)/2; + pixmaps[1].reset(SkImageInfo::MakeA8(w, h), data[1], strides[1]); + pixmaps[2].reset(SkImageInfo::MakeA8(w, h), data[2], strides[2]); + auto yuvaPixmaps = SkYUVAPixmaps::FromExternalPixmaps(yuvaInfo, pixmaps); - SkPixmap pm[4]; - for (int i = 0; i < 3; ++i) { - pm[i] = SkPixmap(info[i], data[i], strides[i]); - } - pm[3].reset(); // no alpha - - SkYUVAIndex indices[4]; - indices[SkYUVAIndex::kY_Index] = {0, SkColorChannel::kR}; - indices[SkYUVAIndex::kU_Index] = {1, SkColorChannel::kR}; - indices[SkYUVAIndex::kV_Index] = {2, SkColorChannel::kR}; - indices[SkYUVAIndex::kA_Index] = {-1, SkColorChannel::kR}; - - return SkImage::MakeFromYUVAPixmaps(gr, yuv_space, pm, indices, {w, h}, - kTopLeft_GrSurfaceOrigin, false, false, cs); + return SkImage::MakeFromYUVAPixmaps( + rContext, yuvaPixmaps, GrMipMapped::kNo, false, std::move(cs)); } // Init with illegal values, so our first compare will fail, forcing us to compute @@ -222,8 +217,8 @@ sk_sp SkVideoDecoder::convertFrame(const AVFrame* frame) { switch (frame->format) { case AV_PIX_FMT_YUV420P: - if (auto image = make_yuv_420(fGr, frame->width, frame->height, frame->data, - frame->linesize, yuv_space, fCSCache.fCS)) { + if (auto image = make_yuv_420(fRecordingContext, frame->width, frame->height, + frame->data, frame->linesize, yuv_space, fCSCache.fCS)) { return image; } break; @@ -311,7 +306,7 @@ sk_sp SkVideoDecoder::nextImage(double* timeStamp) { return nullptr; } -SkVideoDecoder::SkVideoDecoder(GrContext* gr) : fGr(gr) {} +SkVideoDecoder::SkVideoDecoder(GrRecordingContext* rContext) : fRecordingContext(rContext) {} SkVideoDecoder::~SkVideoDecoder() { this->reset(); diff --git a/experimental/ffmpeg/SkVideoDecoder.h b/experimental/ffmpeg/SkVideoDecoder.h index 022009db01..85c25a748a 100644 --- a/experimental/ffmpeg/SkVideoDecoder.h +++ b/experimental/ffmpeg/SkVideoDecoder.h @@ -20,11 +20,11 @@ extern "C" { class SkVideoDecoder { public: - SkVideoDecoder(GrContext* gr = nullptr); + SkVideoDecoder(GrRecordingContext* = nullptr); ~SkVideoDecoder(); void reset(); - void setGrContext(GrContext* gr) { fGr = gr; } + void setGrContext(GrRecordingContext* rContext) { fRecordingContext = rContext; } bool loadStream(std::unique_ptr); bool rewind(); @@ -52,7 +52,7 @@ private: void update(AVColorPrimaries, AVColorTransferCharacteristic); }; - GrContext* fGr = nullptr; // not owned by us + GrRecordingContext* fRecordingContext = nullptr; // not owned by us std::unique_ptr fStream; diff --git a/gm/video_decoder.cpp b/gm/video_decoder.cpp index c2480f0418..cd4aa996a1 100644 --- a/gm/video_decoder.cpp +++ b/gm/video_decoder.cpp @@ -34,12 +34,12 @@ protected: } void onDraw(SkCanvas* canvas) override { - GrContext* gr = canvas->getGrContext(); - if (!gr) { + GrContext* rContext = canvas->recordingContext(); + if (!rContext) { return; } - fDecoder.setGrContext(gr); // gr can change over time in viewer + fDecoder.setGrContext(rContext); // context can change over time in viewer double timeStamp; auto img = fDecoder.nextImage(&timeStamp); diff --git a/include/core/SkImage.h b/include/core/SkImage.h index 7b12a829a9..b5dd49fe5e 100644 --- a/include/core/SkImage.h +++ b/include/core/SkImage.h @@ -31,6 +31,7 @@ class SkMipmap; class SkPaint; class SkPicture; class SkSurface; +class SkYUVAPixmaps; class GrBackendTexture; class GrContext; class GrDirectContext; @@ -505,6 +506,35 @@ public: GrSurfaceOrigin imageOrigin, bool buildMips, bool limitToMaxTextureSize = false, sk_sp imageColorSpace = nullptr); + /** Creates SkImage from SkYUVAPixmaps. + + The image will remain planar with each plane converted to a texture using the passed + GrRecordingContext. + + SkYUVAPixmaps has a SkYUVAInfo which specifies the transformation from YUV to RGB. + The SkColorSpace of the resulting RGB values is specified by imageColorSpace. This will + be the SkColorSpace reported by the image and when drawn the RGB values will be converted + from this space into the destination space (if the destination is tagged). + + Currently, this is only supported using the GPU backend and will fail if context is nullptr. + + SkYUVAPixmaps does not need to remain valid after this returns. + + @param context GPU context + @param pixmaps The planes as pixmaps with supported SkYUVAInfo that + specifies conversion to RGB. + @param buildMips create internal YUVA textures as mip map if kYes. This is + silently ignored if the context does not support mip maps. + @param limitToMaxTextureSize downscale image to GPU maximum texture size, if necessary + @param imageColorSpace range of colors of the resulting image; may be nullptr + @return created SkImage, or nullptr + */ + static sk_sp MakeFromYUVAPixmaps(GrRecordingContext* context, + const SkYUVAPixmaps& pixmaps, + GrMipMapped buildMips = GrMipmapped::kNo, + bool limitToMaxTextureSize = false, + sk_sp imageColorSpace = nullptr); + /** To be deprecated. */ static sk_sp MakeFromYUVTexturesCopyWithExternalBackend( diff --git a/include/core/SkYUVAPixmaps.h b/include/core/SkYUVAPixmaps.h index 51ed9e40cc..7d5d0ae433 100644 --- a/include/core/SkYUVAPixmaps.h +++ b/include/core/SkYUVAPixmaps.h @@ -232,7 +232,7 @@ public: /** * Conversion to legacy SkYUVA data structures. */ - bool toLegacy(SkYUVASizeInfo*, SkYUVAIndex[4]); + bool toLegacy(SkYUVASizeInfo*, SkYUVAIndex[4]) const; private: SkYUVAPixmaps(const SkYUVAPixmapInfo&, sk_sp); diff --git a/src/core/SkYUVAPixmaps.cpp b/src/core/SkYUVAPixmaps.cpp index 9dfa4a569f..afd015a401 100644 --- a/src/core/SkYUVAPixmaps.cpp +++ b/src/core/SkYUVAPixmaps.cpp @@ -129,7 +129,7 @@ SkYUVAPixmapInfo::SkYUVAPixmapInfo(const SkYUVAInfo& yuvaInfo, SkYUVAPixmapInfo::SkYUVAPixmapInfo(const SkYUVAInfo& yuvaInfo, DataType dataType, const size_t rowBytes[kMaxPlanes]) { - SkColorType colorTypes[kMaxPlanes]; + SkColorType colorTypes[kMaxPlanes] = {}; int n = yuvaInfo.numPlanes(); for (int i = 0; i < n; ++i) { // Currently all PlanarConfigs have 1 channel per plane. @@ -216,9 +216,10 @@ SkYUVAPixmaps SkYUVAPixmaps::FromExternalMemory(const SkYUVAPixmapInfo& yuvaPixm SkYUVAPixmaps SkYUVAPixmaps::FromExternalPixmaps(const SkYUVAInfo& yuvaInfo, const SkPixmap pixmaps[kMaxPlanes]) { - SkColorType colorTypes[kMaxPlanes]; - size_t rowBytes[kMaxPlanes]; - for (int i = 0; i < kMaxPlanes; ++i) { + SkColorType colorTypes[kMaxPlanes] = {}; + size_t rowBytes[kMaxPlanes] = {}; + int numPlanes = yuvaInfo.numPlanes(); + for (int i = 0; i < numPlanes; ++i) { colorTypes[i] = pixmaps[i].colorType(); rowBytes[i] = pixmaps[i].rowBytes(); } @@ -239,10 +240,10 @@ SkYUVAPixmaps::SkYUVAPixmaps(const SkYUVAPixmapInfo& yuvaPixmapInfo, sk_spisValid()) { return false; } @@ -293,14 +294,12 @@ bool SkYUVAPixmaps::toLegacy(SkYUVASizeInfo* yuvaSizeInfo, SkYUVAIndex yuvaIndic if (!ok) { return false; } - yuvaSizeInfo->fOrigin = fYUVAInfo.origin(); - SkISize planeDimensions[SkYUVAInfo::kMaxPlanes]; - int n = fYUVAInfo.planeDimensions(planeDimensions); - for (int i = 0; i < n; ++i) { - yuvaSizeInfo->fSizes[i] = fPlanes[i].dimensions(); - yuvaSizeInfo->fWidthBytes[i] = fPlanes[i].rowBytes(); - if (fPlanes[i].dimensions() != planeDimensions[i]) { - return false; + if (yuvaSizeInfo) { + yuvaSizeInfo->fOrigin = fYUVAInfo.origin(); + int n = fYUVAInfo.numPlanes(); + for (int i = 0; i < n; ++i) { + yuvaSizeInfo->fSizes[i] = fPlanes[i].dimensions(); + yuvaSizeInfo->fWidthBytes[i] = fPlanes[i].rowBytes(); } } return true; diff --git a/src/image/SkImage.cpp b/src/image/SkImage.cpp index 37d3dd26d4..8674de4b1b 100644 --- a/src/image/SkImage.cpp +++ b/src/image/SkImage.cpp @@ -593,6 +593,14 @@ sk_sp SkImage::MakeFromYUVATexturesCopyWithExternalBackend( return nullptr; } +sk_sp SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context, + const SkYUVAPixmaps& pixmaps, + GrMipMapped buildMips, + bool limitToMaxTextureSize, + sk_sp imageColorSpace) { + return nullptr; +} + sk_sp SkImage::MakeFromYUVTexturesCopyWithExternalBackend( GrContext*, SkYUVColorSpace, const GrBackendTexture[3], GrSurfaceOrigin, const GrBackendTexture&, sk_sp) { diff --git a/src/image/SkImage_GpuYUVA.cpp b/src/image/SkImage_GpuYUVA.cpp index 546c821b6d..cbd78ec95c 100644 --- a/src/image/SkImage_GpuYUVA.cpp +++ b/src/image/SkImage_GpuYUVA.cpp @@ -9,6 +9,7 @@ #include #include +#include "include/core/SkYUVAPixmaps.h" #include "include/core/SkYUVASizeInfo.h" #include "include/gpu/GrDirectContext.h" #include "include/gpu/GrRecordingContext.h" @@ -290,10 +291,10 @@ sk_sp SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context, // Make proxies GrSurfaceProxyView tempViews[4]; + int maxTextureSize = context->priv().caps()->maxTextureSize(); for (int i = 0; i < numPixmaps; ++i) { const SkPixmap* pixmap = &yuvaPixmaps[i]; SkAutoPixmapStorage resized; - int maxTextureSize = context->priv().caps()->maxTextureSize(); int maxDim = std::max(yuvaPixmaps[i].width(), yuvaPixmaps[i].height()); if (limitToMaxTextureSize && maxDim > maxTextureSize) { float scale = static_cast(maxTextureSize) / maxDim; @@ -312,7 +313,6 @@ sk_sp SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context, bmp.installPixels(*pixmap); GrBitmapTextureMaker bitmapMaker(context, bmp, GrImageTexGenPolicy::kNew_Uncached_Budgeted); GrMipmapped mipMapped = buildMips ? GrMipmapped::kYes : GrMipmapped::kNo; - GrSurfaceProxyView view; tempViews[i] = bitmapMaker.view(mipMapped); if (!tempViews[i]) { return nullptr; @@ -321,7 +321,76 @@ sk_sp SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context, return sk_make_sp(sk_ref_sp(context), imageSize, kNeedNewImageUniqueID, yuvColorSpace, tempViews, numPixmaps, yuvaIndices, - imageOrigin, imageColorSpace); + imageOrigin, std::move(imageColorSpace)); +} + +sk_sp SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context, + const SkYUVAPixmaps& pixmaps, + GrMipMapped buildMips, + bool limitToMaxTextureSize, + sk_sp imageColorSpace) { + if (!context) { + return nullptr; // until we impl this for raster backend + } + + if (!pixmaps.isValid()) { + return nullptr; + } + + SkYUVAIndex yuvaIndices[4]; + if (!pixmaps.toLegacy(nullptr, yuvaIndices)) { + return nullptr; + } + + // SkImage_GpuYUVA doesn't yet support different encoded origins. + if (pixmaps.yuvaInfo().origin() != kTopLeft_SkEncodedOrigin) { + return nullptr; + } + + if (!context->priv().caps()->mipmapSupport()) { + buildMips = GrMipMapped::kNo; + } + + // Make proxies + GrSurfaceProxyView tempViews[4]; + int numPlanes = pixmaps.numPlanes(); + int maxTextureSize = context->priv().caps()->maxTextureSize(); + for (int i = 0; i < numPlanes; ++i) { + const SkPixmap* pixmap = &pixmaps.plane(i); + SkAutoPixmapStorage resized; + int maxDim = std::max(pixmap->width(), pixmap->height()); + if (maxDim > maxTextureSize) { + if (!limitToMaxTextureSize) { + return nullptr; + } + float scale = static_cast(maxTextureSize)/maxDim; + int newWidth = std::min(static_cast(pixmap->width() *scale), maxTextureSize); + int newHeight = std::min(static_cast(pixmap->height()*scale), maxTextureSize); + SkImageInfo info = pixmap->info().makeWH(newWidth, newHeight); + if (!resized.tryAlloc(info) || !pixmap->scalePixels(resized, kLow_SkFilterQuality)) { + return nullptr; + } + pixmap = &resized; + } + // Turn the pixmap into a GrTextureProxy + SkBitmap bmp; + bmp.installPixels(*pixmap); + GrBitmapTextureMaker bitmapMaker(context, bmp, GrImageTexGenPolicy::kNew_Uncached_Budgeted); + tempViews[i] = bitmapMaker.view(buildMips); + if (!tempViews[i]) { + return nullptr; + } + } + + return sk_make_sp(sk_ref_sp(context), + pixmaps.yuvaInfo().dimensions(), + kNeedNewImageUniqueID, + pixmaps.yuvaInfo().yuvColorSpace(), + tempViews, + numPlanes, + yuvaIndices, + kTopLeft_GrSurfaceOrigin, + std::move(imageColorSpace)); } ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/tests/ImageTest.cpp b/tests/ImageTest.cpp index 25c35435e7..b7dd5b99d6 100644 --- a/tests/ImageTest.cpp +++ b/tests/ImageTest.cpp @@ -1366,14 +1366,11 @@ DEF_TEST(Image_nonfinite_dst, reporter) { static sk_sp make_yuva_image(GrDirectContext* dContext) { SkAutoPixmapStorage pm; pm.alloc(SkImageInfo::Make(1, 1, kAlpha_8_SkColorType, kPremul_SkAlphaType)); - const SkPixmap pmaps[] = {pm, pm, pm, pm}; - SkYUVAIndex indices[] = {{0, SkColorChannel::kA}, - {1, SkColorChannel::kA}, - {2, SkColorChannel::kA}, - {3, SkColorChannel::kA}}; + SkYUVAInfo yuvaInfo({1, 1}, SkYUVAInfo::PlanarConfig::kY_U_V_444, kJPEG_Full_SkYUVColorSpace); + const SkPixmap pmaps[] = {pm, pm, pm}; + auto yuvaPixmaps = SkYUVAPixmaps::FromExternalPixmaps(yuvaInfo, pmaps); - return SkImage::MakeFromYUVAPixmaps(dContext, kJPEG_SkYUVColorSpace, pmaps, indices, - SkISize::Make(1, 1), kTopLeft_GrSurfaceOrigin, false); + return SkImage::MakeFromYUVAPixmaps(dContext, yuvaPixmaps); } DEF_GPUTEST_FOR_ALL_CONTEXTS(ImageFlush, reporter, ctxInfo) { diff --git a/tools/gpu/YUVUtils.cpp b/tools/gpu/YUVUtils.cpp index 87ce7120b1..c2e2288cf5 100644 --- a/tools/gpu/YUVUtils.cpp +++ b/tools/gpu/YUVUtils.cpp @@ -75,14 +75,7 @@ bool LazyYUVImage::ensureYUVImage(GrRecordingContext* rContext) { return true; // Have already made a YUV image valid for this context. } // Try to make a new YUV image for this context. - fYUVImage = SkImage::MakeFromYUVAPixmaps(rContext, - fPixmaps.yuvaInfo().yuvColorSpace(), - fPixmaps.planes().data(), - fComponents, - fSizeInfo.fSizes[0], - kTopLeft_GrSurfaceOrigin, - static_cast(fMipmapped), - false); + fYUVImage = SkImage::MakeFromYUVAPixmaps(rContext, fPixmaps, fMipmapped, false, nullptr); return fYUVImage != nullptr; }