New version of SkImage::MakeFromYUVAPixmaps()

Takes SkYUVAPixmaps. Still implemented in terms of SkYUVAIndex[4]
internally.

Replace all internal use cases except wacky_yuv_formats.

Takes GrRecordingContext rather than GrContext.

SkVideoDecoder updated to take GrRecordingContext.

Bug: skia:10632

Change-Id: I6e9b9b6a4f11333bce6f87c1ebff0acb297f6540
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/316837
Commit-Queue: Brian Salomon <bsalomon@google.com>
Reviewed-by: Adlai Holler <adlai@google.com>
Reviewed-by: Robert Phillips <robertphillips@google.com>

RELEASE_NOTES.txt

@@ -9,6 +9,12 @@ Milestone 87
 
   * <insert new release notes here>
 
+  * Alternate SkImage::MakeFromYUVAPixmaps signature. Takes SkYUVAPixmaps, which specifies
+    planar configuration in a more structured manner. Currently limited to tri-planar
+    configurations without alpha but will be expanded. Older signature will become
+    deprecated when the new signature supports a broad range of planar configurations.
+    https://review.skia.org/316837
+
   * Switch to using Microsoft::WRL::ComPtr for wrapping D3D12 objects.
     https://review.skia.org/314957
 

experimental/ffmpeg/SkVideoDecoder.cpp

@@ -9,6 +9,7 @@
 #include "include/core/SkColorSpace.h"
 #include "include/core/SkImage.h"
 #include "include/core/SkYUVAIndex.h"
+#include "include/core/SkYUVAPixmaps.h"
 
 static SkYUVColorSpace get_yuvspace(AVColorSpace space) {
     // this is pretty incomplete -- TODO: look to convert more AVColorSpaces
@@ -159,29 +160,23 @@ static int64_t skstream_seek_packet(void* ctx, int64_t pos, int whence) {
     return stream->seek(SkToSizeT(pos)) ? pos : -1;
 }
 
-static sk_sp<SkImage> make_yuv_420(GrContext* gr, int w, int h,
+static sk_sp<SkImage> make_yuv_420(GrRecordingContext* rContext,
-                                   uint8_t* const data[], int const strides[],
+                                   int w, int h,
-                                   SkYUVColorSpace yuv_space,
+                                   uint8_t* const data[],
+                                   int const strides[],
+                                   SkYUVColorSpace yuvSpace,
                                    sk_sp<SkColorSpace> cs) {
-    SkImageInfo info[3];
+    SkYUVAInfo yuvaInfo({w, h}, SkYUVAInfo::PlanarConfig::kY_U_V_420, yuvSpace);
-    info[0] = SkImageInfo::Make(w, h, kGray_8_SkColorType, kOpaque_SkAlphaType);
+    SkPixmap pixmaps[3];
-    info[1] = SkImageInfo::Make(w/2, h/2, kGray_8_SkColorType, kOpaque_SkAlphaType);
+    pixmaps[0].reset(SkImageInfo::MakeA8(w, h), data[0], strides[0]);
-    info[2] = SkImageInfo::Make(w/2, h/2, kGray_8_SkColorType, kOpaque_SkAlphaType);
+    w = (w + 1)/2;
+    h = (h + 1)/2;
+    pixmaps[1].reset(SkImageInfo::MakeA8(w, h), data[1], strides[1]);
+    pixmaps[2].reset(SkImageInfo::MakeA8(w, h), data[2], strides[2]);
+    auto yuvaPixmaps = SkYUVAPixmaps::FromExternalPixmaps(yuvaInfo, pixmaps);
 
-    SkPixmap pm[4];
+    return SkImage::MakeFromYUVAPixmaps(
-    for (int i = 0; i < 3; ++i) {
+            rContext, yuvaPixmaps, GrMipMapped::kNo, false, std::move(cs));
-        pm[i] = SkPixmap(info[i], data[i], strides[i]);
-    }
-    pm[3].reset();  // no alpha
-
-    SkYUVAIndex indices[4];
-    indices[SkYUVAIndex::kY_Index] = {0, SkColorChannel::kR};
-    indices[SkYUVAIndex::kU_Index] = {1, SkColorChannel::kR};
-    indices[SkYUVAIndex::kV_Index] = {2, SkColorChannel::kR};
-    indices[SkYUVAIndex::kA_Index] = {-1, SkColorChannel::kR};
-
-    return SkImage::MakeFromYUVAPixmaps(gr, yuv_space, pm, indices, {w, h},
-                                        kTopLeft_GrSurfaceOrigin, false, false, cs);
 }
 
 // Init with illegal values, so our first compare will fail, forcing us to compute
@@ -222,8 +217,8 @@ sk_sp<SkImage> SkVideoDecoder::convertFrame(const AVFrame* frame) {
 
     switch (frame->format) {
         case AV_PIX_FMT_YUV420P:
-            if (auto image = make_yuv_420(fGr, frame->width, frame->height, frame->data,
+            if (auto image = make_yuv_420(fRecordingContext, frame->width, frame->height,
-                                          frame->linesize, yuv_space, fCSCache.fCS)) {
+                                          frame->data, frame->linesize, yuv_space, fCSCache.fCS)) {
                 return image;
             }
             break;
@@ -311,7 +306,7 @@ sk_sp<SkImage> SkVideoDecoder::nextImage(double* timeStamp) {
     return nullptr;
 }
 
-SkVideoDecoder::SkVideoDecoder(GrContext* gr) : fGr(gr) {}
+SkVideoDecoder::SkVideoDecoder(GrRecordingContext* rContext) : fRecordingContext(rContext) {}
 
 SkVideoDecoder::~SkVideoDecoder() {
     this->reset();

experimental/ffmpeg/SkVideoDecoder.h

@@ -20,11 +20,11 @@ extern "C" {
 
 class SkVideoDecoder {
 public:
-    SkVideoDecoder(GrContext* gr = nullptr);
+    SkVideoDecoder(GrRecordingContext* = nullptr);
     ~SkVideoDecoder();
 
     void reset();
-    void setGrContext(GrContext* gr) { fGr = gr; }
+    void setGrContext(GrRecordingContext* rContext) { fRecordingContext = rContext; }
 
     bool loadStream(std::unique_ptr<SkStream>);
     bool rewind();
@@ -52,7 +52,7 @@ private:
         void update(AVColorPrimaries, AVColorTransferCharacteristic);
     };
 
-    GrContext*          fGr = nullptr;  // not owned by us
+    GrRecordingContext* fRecordingContext = nullptr;  // not owned by us
 
     std::unique_ptr<SkStream>   fStream;
 

gm/video_decoder.cpp

@@ -34,12 +34,12 @@ protected:
     }
 
     void onDraw(SkCanvas* canvas) override {
-        GrContext* gr = canvas->getGrContext();
+        GrContext* rContext = canvas->recordingContext();
-        if (!gr) {
+        if (!rContext) {
             return;
         }
 
-        fDecoder.setGrContext(gr); // gr can change over time in viewer
+        fDecoder.setGrContext(rContext);  // context can change over time in viewer
 
         double timeStamp;
         auto img = fDecoder.nextImage(&timeStamp);

include/core/SkImage.h

@@ -31,6 +31,7 @@ class SkMipmap;
 class SkPaint;
 class SkPicture;
 class SkSurface;
+class SkYUVAPixmaps;
 class GrBackendTexture;
 class GrContext;
 class GrDirectContext;
@@ -505,6 +506,35 @@ public:
             GrSurfaceOrigin imageOrigin, bool buildMips, bool limitToMaxTextureSize = false,
             sk_sp<SkColorSpace> imageColorSpace = nullptr);
 
+    /** Creates SkImage from SkYUVAPixmaps.
+
+        The image will remain planar with each plane converted to a texture using the passed
+        GrRecordingContext.
+
+        SkYUVAPixmaps has a SkYUVAInfo which specifies the transformation from YUV to RGB.
+        The SkColorSpace of the resulting RGB values is specified by imageColorSpace. This will
+        be the SkColorSpace reported by the image and when drawn the RGB values will be converted
+        from this space into the destination space (if the destination is tagged).
+
+        Currently, this is only supported using the GPU backend and will fail if context is nullptr.
+
+        SkYUVAPixmaps does not need to remain valid after this returns.
+
+        @param context                GPU context
+        @param pixmaps                The planes as pixmaps with supported SkYUVAInfo that
+                                      specifies conversion to RGB.
+        @param buildMips              create internal YUVA textures as mip map if kYes. This is
+                                      silently ignored if the context does not support mip maps.
+        @param limitToMaxTextureSize  downscale image to GPU maximum texture size, if necessary
+        @param imageColorSpace        range of colors of the resulting image; may be nullptr
+        @return                       created SkImage, or nullptr
+    */
+    static sk_sp<SkImage> MakeFromYUVAPixmaps(GrRecordingContext* context,
+                                              const SkYUVAPixmaps& pixmaps,
+                                              GrMipMapped buildMips = GrMipmapped::kNo,
+                                              bool limitToMaxTextureSize = false,
+                                              sk_sp<SkColorSpace> imageColorSpace = nullptr);
+
     /** To be deprecated.
     */
     static sk_sp<SkImage> MakeFromYUVTexturesCopyWithExternalBackend(

include/core/SkYUVAPixmaps.h

@@ -232,7 +232,7 @@ public:
     /**
      * Conversion to legacy SkYUVA data structures.
      */
-    bool toLegacy(SkYUVASizeInfo*, SkYUVAIndex[4]);
+    bool toLegacy(SkYUVASizeInfo*, SkYUVAIndex[4]) const;
 
 private:
     SkYUVAPixmaps(const SkYUVAPixmapInfo&, sk_sp<SkData>);

src/core/SkYUVAPixmaps.cpp

@@ -129,7 +129,7 @@ SkYUVAPixmapInfo::SkYUVAPixmapInfo(const SkYUVAInfo& yuvaInfo,
 SkYUVAPixmapInfo::SkYUVAPixmapInfo(const SkYUVAInfo& yuvaInfo,
                                    DataType dataType,
                                    const size_t rowBytes[kMaxPlanes]) {
-    SkColorType colorTypes[kMaxPlanes];
+    SkColorType colorTypes[kMaxPlanes] = {};
     int n = yuvaInfo.numPlanes();
     for (int i = 0; i < n; ++i) {
         // Currently all PlanarConfigs have 1 channel per plane.
@@ -216,9 +216,10 @@ SkYUVAPixmaps SkYUVAPixmaps::FromExternalMemory(const SkYUVAPixmapInfo& yuvaPixm
 
 SkYUVAPixmaps SkYUVAPixmaps::FromExternalPixmaps(const SkYUVAInfo& yuvaInfo,
                                                  const SkPixmap pixmaps[kMaxPlanes]) {
-    SkColorType colorTypes[kMaxPlanes];
+    SkColorType colorTypes[kMaxPlanes] = {};
-    size_t rowBytes[kMaxPlanes];
+    size_t rowBytes[kMaxPlanes] = {};
-    for (int i = 0; i < kMaxPlanes; ++i) {
+    int numPlanes = yuvaInfo.numPlanes();
+    for (int i = 0; i < numPlanes; ++i) {
         colorTypes[i] = pixmaps[i].colorType();
         rowBytes[i] = pixmaps[i].rowBytes();
     }
@@ -239,10 +240,10 @@ SkYUVAPixmaps::SkYUVAPixmaps(const SkYUVAPixmapInfo& yuvaPixmapInfo, sk_sp<SkDat
 
 SkYUVAPixmaps::SkYUVAPixmaps(const SkYUVAInfo& yuvaInfo, const SkPixmap pixmaps[kMaxPlanes])
         : fYUVAInfo(yuvaInfo) {
-    std::copy_n(pixmaps, kMaxPlanes, fPlanes.data());
+    std::copy_n(pixmaps, yuvaInfo.numPlanes(), fPlanes.data());
 }
 
-bool SkYUVAPixmaps::toLegacy(SkYUVASizeInfo* yuvaSizeInfo, SkYUVAIndex yuvaIndices[4]) {
+bool SkYUVAPixmaps::toLegacy(SkYUVASizeInfo* yuvaSizeInfo, SkYUVAIndex yuvaIndices[4]) const {
     if (!this->isValid()) {
         return false;
     }
@@ -293,14 +294,12 @@ bool SkYUVAPixmaps::toLegacy(SkYUVASizeInfo* yuvaSizeInfo, SkYUVAIndex yuvaIndic
     if (!ok) {
         return false;
     }
-    yuvaSizeInfo->fOrigin = fYUVAInfo.origin();
+    if (yuvaSizeInfo) {
-    SkISize planeDimensions[SkYUVAInfo::kMaxPlanes];
+        yuvaSizeInfo->fOrigin = fYUVAInfo.origin();
-    int n = fYUVAInfo.planeDimensions(planeDimensions);
+        int n = fYUVAInfo.numPlanes();
-    for (int i = 0; i < n; ++i) {
+        for (int i = 0; i < n; ++i) {
-        yuvaSizeInfo->fSizes[i] = fPlanes[i].dimensions();
+            yuvaSizeInfo->fSizes[i] = fPlanes[i].dimensions();
-        yuvaSizeInfo->fWidthBytes[i] = fPlanes[i].rowBytes();
+            yuvaSizeInfo->fWidthBytes[i] = fPlanes[i].rowBytes();
-        if (fPlanes[i].dimensions() != planeDimensions[i]) {
-            return false;
         }
     }
     return true;

src/image/SkImage.cpp

@@ -593,6 +593,14 @@ sk_sp<SkImage> SkImage::MakeFromYUVATexturesCopyWithExternalBackend(
     return nullptr;
 }
 
+sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
+                                            const SkYUVAPixmaps& pixmaps,
+                                            GrMipMapped buildMips,
+                                            bool limitToMaxTextureSize,
+                                            sk_sp<SkColorSpace> imageColorSpace) {
+    return nullptr;
+}
+
 sk_sp<SkImage> SkImage::MakeFromYUVTexturesCopyWithExternalBackend(
         GrContext*, SkYUVColorSpace, const GrBackendTexture[3], GrSurfaceOrigin,
         const GrBackendTexture&, sk_sp<SkColorSpace>) {

src/image/SkImage_GpuYUVA.cpp

@@ -9,6 +9,7 @@
 #include <cstring>
 #include <type_traits>
 
+#include "include/core/SkYUVAPixmaps.h"
 #include "include/core/SkYUVASizeInfo.h"
 #include "include/gpu/GrDirectContext.h"
 #include "include/gpu/GrRecordingContext.h"
@@ -290,10 +291,10 @@ sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
 
     // Make proxies
     GrSurfaceProxyView tempViews[4];
+    int maxTextureSize = context->priv().caps()->maxTextureSize();
     for (int i = 0; i < numPixmaps; ++i) {
         const SkPixmap* pixmap = &yuvaPixmaps[i];
         SkAutoPixmapStorage resized;
-        int maxTextureSize = context->priv().caps()->maxTextureSize();
         int maxDim = std::max(yuvaPixmaps[i].width(), yuvaPixmaps[i].height());
         if (limitToMaxTextureSize && maxDim > maxTextureSize) {
             float scale = static_cast<float>(maxTextureSize) / maxDim;
@@ -312,7 +313,6 @@ sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
         bmp.installPixels(*pixmap);
         GrBitmapTextureMaker bitmapMaker(context, bmp, GrImageTexGenPolicy::kNew_Uncached_Budgeted);
         GrMipmapped mipMapped = buildMips ? GrMipmapped::kYes : GrMipmapped::kNo;
-        GrSurfaceProxyView view;
         tempViews[i] = bitmapMaker.view(mipMapped);
         if (!tempViews[i]) {
             return nullptr;
@@ -321,7 +321,76 @@ sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
 
     return sk_make_sp<SkImage_GpuYUVA>(sk_ref_sp(context), imageSize, kNeedNewImageUniqueID,
                                        yuvColorSpace, tempViews, numPixmaps, yuvaIndices,
-                                       imageOrigin, imageColorSpace);
+                                       imageOrigin, std::move(imageColorSpace));
+}
+
+sk_sp<SkImage> SkImage::MakeFromYUVAPixmaps(GrRecordingContext* context,
+                                            const SkYUVAPixmaps& pixmaps,
+                                            GrMipMapped buildMips,
+                                            bool limitToMaxTextureSize,
+                                            sk_sp<SkColorSpace> imageColorSpace) {
+    if (!context) {
+        return nullptr;  // until we impl this for raster backend
+    }
+
+    if (!pixmaps.isValid()) {
+        return nullptr;
+    }
+
+    SkYUVAIndex yuvaIndices[4];
+    if (!pixmaps.toLegacy(nullptr, yuvaIndices)) {
+        return nullptr;
+    }
+
+    // SkImage_GpuYUVA doesn't yet support different encoded origins.
+    if (pixmaps.yuvaInfo().origin() != kTopLeft_SkEncodedOrigin) {
+        return nullptr;
+    }
+
+    if (!context->priv().caps()->mipmapSupport()) {
+        buildMips = GrMipMapped::kNo;
+    }
+
+    // Make proxies
+    GrSurfaceProxyView tempViews[4];
+    int numPlanes = pixmaps.numPlanes();
+    int maxTextureSize = context->priv().caps()->maxTextureSize();
+    for (int i = 0; i < numPlanes; ++i) {
+        const SkPixmap* pixmap = &pixmaps.plane(i);
+        SkAutoPixmapStorage resized;
+        int maxDim = std::max(pixmap->width(), pixmap->height());
+        if (maxDim > maxTextureSize) {
+            if (!limitToMaxTextureSize) {
+                return nullptr;
+            }
+            float scale = static_cast<float>(maxTextureSize)/maxDim;
+            int newWidth  = std::min(static_cast<int>(pixmap->width() *scale), maxTextureSize);
+            int newHeight = std::min(static_cast<int>(pixmap->height()*scale), maxTextureSize);
+            SkImageInfo info = pixmap->info().makeWH(newWidth, newHeight);
+            if (!resized.tryAlloc(info) || !pixmap->scalePixels(resized, kLow_SkFilterQuality)) {
+                return nullptr;
+            }
+            pixmap = &resized;
+        }
+        // Turn the pixmap into a GrTextureProxy
+        SkBitmap bmp;
+        bmp.installPixels(*pixmap);
+        GrBitmapTextureMaker bitmapMaker(context, bmp, GrImageTexGenPolicy::kNew_Uncached_Budgeted);
+        tempViews[i] = bitmapMaker.view(buildMips);
+        if (!tempViews[i]) {
+            return nullptr;
+        }
+    }
+
+    return sk_make_sp<SkImage_GpuYUVA>(sk_ref_sp(context),
+                                       pixmaps.yuvaInfo().dimensions(),
+                                       kNeedNewImageUniqueID,
+                                       pixmaps.yuvaInfo().yuvColorSpace(),
+                                       tempViews,
+                                       numPlanes,
+                                       yuvaIndices,
+                                       kTopLeft_GrSurfaceOrigin,
+                                       std::move(imageColorSpace));
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////

tests/ImageTest.cpp

@@ -1366,14 +1366,11 @@ DEF_TEST(Image_nonfinite_dst, reporter) {
 static sk_sp<SkImage> make_yuva_image(GrDirectContext* dContext) {
     SkAutoPixmapStorage pm;
     pm.alloc(SkImageInfo::Make(1, 1, kAlpha_8_SkColorType, kPremul_SkAlphaType));
-    const SkPixmap pmaps[] = {pm, pm, pm, pm};
+    SkYUVAInfo yuvaInfo({1, 1}, SkYUVAInfo::PlanarConfig::kY_U_V_444, kJPEG_Full_SkYUVColorSpace);
-    SkYUVAIndex indices[] = {{0, SkColorChannel::kA},
+    const SkPixmap pmaps[] = {pm, pm, pm};
-                             {1, SkColorChannel::kA},
+    auto yuvaPixmaps = SkYUVAPixmaps::FromExternalPixmaps(yuvaInfo, pmaps);
-                             {2, SkColorChannel::kA},
-                             {3, SkColorChannel::kA}};
 
-    return SkImage::MakeFromYUVAPixmaps(dContext, kJPEG_SkYUVColorSpace, pmaps, indices,
+    return SkImage::MakeFromYUVAPixmaps(dContext, yuvaPixmaps);
-                                        SkISize::Make(1, 1), kTopLeft_GrSurfaceOrigin, false);
 }
 
 DEF_GPUTEST_FOR_ALL_CONTEXTS(ImageFlush, reporter, ctxInfo) {

tools/gpu/YUVUtils.cpp

@@ -75,14 +75,7 @@ bool LazyYUVImage::ensureYUVImage(GrRecordingContext* rContext) {
         return true; // Have already made a YUV image valid for this context.
     }
     // Try to make a new YUV image for this context.
-    fYUVImage = SkImage::MakeFromYUVAPixmaps(rContext,
+    fYUVImage = SkImage::MakeFromYUVAPixmaps(rContext, fPixmaps, fMipmapped, false, nullptr);
-                                             fPixmaps.yuvaInfo().yuvColorSpace(),
-                                             fPixmaps.planes().data(),
-                                             fComponents,
-                                             fSizeInfo.fSizes[0],
-                                             kTopLeft_GrSurfaceOrigin,
-                                             static_cast<bool>(fMipmapped),
-                                             false);
     return fYUVImage != nullptr;
 }