Add CubicResampler to makeShader

Only works on raster backend for the now.

Bug: skia:10344
Change-Id: I2c82d5345ae83a2bb2744ab27e3d971c57ea989e
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/303271
Commit-Queue: Mike Reed <reed@google.com>
Reviewed-by: Brian Salomon <bsalomon@google.com>
Reviewed-by: Mike Klein <mtklein@google.com>

gm/bicubic.cpp

@@ -6,23 +6,42 @@
  */
 
 #include "gm/gm.h"
-#include "include/core/SkBitmap.h"
 #include "include/core/SkCanvas.h"
+#include "include/core/SkSurface.h"
 
-DEF_SIMPLE_GM(bicubic, canvas, 300, 64) {
+DEF_SIMPLE_GM(bicubic, canvas, 300, 320) {
     canvas->clear(SK_ColorBLACK);
 
-    SkBitmap bmp;
+    auto make_img = []() {
-    bmp.allocN32Pixels(8, 1);
+        auto surf = SkSurface::MakeRasterN32Premul(7, 7);
-    bmp.eraseColor(0);
+        surf->getCanvas()->drawColor(SK_ColorBLACK);
-    *bmp.getAddr32(3, 0) = 0xFFFFFFFF;
 
+        SkPaint paint;
+        paint.setColor(SK_ColorWHITE);
+        surf->getCanvas()->drawLine(3.5f, 0, 3.5f, 8, paint);
+        return surf->makeImageSnapshot();
+    };
+
+    auto img = make_img();
+
+    canvas->scale(40, 8);
+    for (auto q : {kNone_SkFilterQuality, kLow_SkFilterQuality, kHigh_SkFilterQuality}) {
+        SkPaint p;
+        p.setFilterQuality(q);
+        canvas->drawImage(img, 0, 0, &p);
+        canvas->translate(0, img->height() + 1.0f);
+    }
+
+    const SkRect r = SkRect::MakeIWH(img->width(), img->height());
     SkPaint paint;
-    paint.setFilterQuality(kHigh_SkFilterQuality);
 
-    for (int i = 0; i < 64; ++i) {
+    SkImage::CubicResampler cubics[] = {
-        float x = 1.0f + i/63.0f;
+        {      0, 1.0f/2 },
-        float y = i;
+        { 1.0f/3, 1.0f/3 },
-        canvas->drawBitmapRect(bmp, SkRect::MakeXYWH(x, y, 512, 1), &paint);
+    };
+    for (auto c : cubics) {
+        paint.setShader(img->makeShader(SkTileMode::kClamp, SkTileMode::kClamp, c));
+        canvas->drawRect(r, paint);
+        canvas->translate(0, img->height() + 1.0f);
     }
 }

include/core/SkImage.h

@@ -11,6 +11,7 @@
 #include "include/core/SkFilterQuality.h"
 #include "include/core/SkImageEncoder.h"
 #include "include/core/SkImageInfo.h"
+#include "include/core/SkM44.h"
 #include "include/core/SkRefCnt.h"
 #include "include/core/SkScalar.h"
 #include "include/core/SkShader.h"
@@ -785,9 +786,40 @@ public:
     */
     bool isOpaque() const { return SkAlphaTypeIsOpaque(this->alphaType()); }
 
+    /**
+     *  Make a shader with the specified tiling and mipmap sampling.
+     */
     sk_sp<SkShader> makeShader(SkTileMode tmx, SkTileMode tmy, const SkFilterOptions&,
                                const SkMatrix* localMatrix = nullptr) const;
 
+    /*
+     *  Specify B and C (each between 0...1) to create a shader that applies the corresponding
+     *  cubic reconstruction filter to the image.
+     *
+     *  Example values:
+     *      B = 1/3, C = 1/3        "Mitchell" filter
+     *      B = 0,   C = 1/2        "Catmull-Rom" filter
+     *
+     *  See "Reconstruction Filters in Computer Graphics"
+     *          Don P. Mitchell
+     *          Arun N. Netravali
+     *          1988
+     *  https://www.cs.utexas.edu/~fussell/courses/cs384g-fall2013/lectures/mitchell/Mitchell.pdf
+     *
+     *  Desmos worksheet https://www.desmos.com/calculator/aghdpicrvr
+     *  Nice overview https://entropymine.com/imageworsener/bicubic/
+     */
+    struct CubicResampler {
+        float B, C;
+    };
+
+    /**
+     *  Make a shader with the specified tiling and CubicResampler parameters.
+     *  Returns nullptr if the resampler values are outside of [0...1]
+     */
+    sk_sp<SkShader> makeShader(SkTileMode tmx, SkTileMode tmy, CubicResampler,
+                                      const SkMatrix* localMatrix = nullptr) const;
+
     /** Creates SkShader from SkImage. SkShader dimensions are taken from SkImage. SkShader uses
         SkTileMode rules to fill drawn area outside SkImage. localMatrix permits
         transforming SkImage before SkCanvas matrix is applied.

src/core/SkPicturePriv.h

@@ -79,6 +79,7 @@ public:
     // V76: Add filtering enum to ImageShader
     // V77: Explicit filtering options on imageshaders
     // V78: Serialize skmipmap data for images that have it
+    // V79: Cubic Resampler option on imageshader
 
     enum Version {
         kMorphologyTakesScalar_Version      = 74,
@@ -86,10 +87,11 @@ public:
         kFilterEnumInImageShader_Version    = 76,
         kFilterOptionsInImageShader_Version = 77,
         kSerializeMipmaps_Version           = 78,
+        kCubicResamplerImageShader_Version  = 79,
 
         // Only SKPs within the min/current picture version range (inclusive) can be read.
         kMin_Version     = kMorphologyTakesScalar_Version,
-        kCurrent_Version = kSerializeMipmaps_Version
+        kCurrent_Version = kCubicResamplerImageShader_Version
     };
 };
 

src/image/SkImage.cpp

@@ -142,6 +142,12 @@ sk_sp<SkShader> SkImage::makeShader(SkTileMode tmx, SkTileMode tmy,
                                SkImageShader::FilterEnum(filtering));
 }
 
+sk_sp<SkShader> SkImage::makeShader(SkTileMode tmx, SkTileMode tmy, CubicResampler cubic,
+                                           const SkMatrix* localMatrix) const {
+    return SkImageShader::Make(sk_ref_sp(const_cast<SkImage*>(this)), tmx, tmy, cubic,
+                               localMatrix);
+}
+
 sk_sp<SkData> SkImage::encodeToData(SkEncodedImageFormat type, int quality) const {
     SkBitmap bm;
     if (as_IB(this)->getROPixels(&bm)) {

src/shaders/SkImageShader.cpp

@@ -22,6 +22,16 @@
 #include "src/shaders/SkBitmapProcShader.h"
 #include "src/shaders/SkEmptyShader.h"
 
+SkM44 SkImageShader::CubicResamplerMatrix(float B, float C) {
+    const float scale = 1.0f/18;
+    B *= scale;
+    C *= scale;
+    return SkM44(    3*B, -9*B - 18*C,       9*B + 36*C,      -3*B - 18*C,
+                 1 - 6*B,           0, -3 + 36*B + 18*C,  2 - 27*B - 18*C,
+                     3*B,  9*B + 18*C,  3 - 45*B - 36*C, -2 + 27*B + 18*C,
+                       0,           0,            -18*C,       3*B + 18*C);
+}
+
 /**
  *  We are faster in clamp, so always use that tiling when we can.
  */
@@ -47,7 +57,8 @@ SkImageShader::SkImageShader(sk_sp<SkImage> img,
     , fTileModeY(optimize(tmy, fImage->height()))
     , fFilterEnum(filtering)
     , fClampAsIfUnpremul(clampAsIfUnpremul)
-    , fFilterOptions({})   // ignored
+    , fFilterOptions({})    // ignored
+    , fCubic({})            // ignored
 {
     SkASSERT(filtering != kUseFilterOptions);
 }
@@ -63,6 +74,21 @@ SkImageShader::SkImageShader(sk_sp<SkImage> img,
     , fFilterEnum(FilterEnum::kUseFilterOptions)
     , fClampAsIfUnpremul(false)
     , fFilterOptions(options)
+    , fCubic({})    // ignored
+{}
+
+SkImageShader::SkImageShader(sk_sp<SkImage> img,
+                             SkTileMode tmx, SkTileMode tmy,
+                             SkImage::CubicResampler cubic,
+                             const SkMatrix* localMatrix)
+    : INHERITED(localMatrix)
+    , fImage(std::move(img))
+    , fTileModeX(optimize(tmx, fImage->width()))
+    , fTileModeY(optimize(tmy, fImage->height()))
+    , fFilterEnum(FilterEnum::kUseCubicResampler)
+    , fClampAsIfUnpremul(false)
+    , fFilterOptions({})    // ignored
+    , fCubic(cubic)
 {}
 
 // fClampAsIfUnpremul is always false when constructed through public APIs,
@@ -74,13 +100,30 @@ sk_sp<SkFlattenable> SkImageShader::CreateProc(SkReadBuffer& buffer) {
 
     FilterEnum fe = kInheritFromPaint;
     if (!buffer.isVersionLT(SkPicturePriv::kFilterEnumInImageShader_Version)) {
-        fe = buffer.read32LE<FilterEnum>(kUseFilterOptions);
+        fe = buffer.read32LE<FilterEnum>(kLast);
     }
 
-    SkFilterOptions fo = { SkSamplingMode::kNearest, SkMipmapMode::kNone };
+    SkFilterOptions fo{ SkSamplingMode::kNearest, SkMipmapMode::kNone };
-    if (!buffer.isVersionLT(SkPicturePriv::kFilterOptionsInImageShader_Version)) {
+    SkImage::CubicResampler cubic{};
-        fo.fSampling = buffer.read32LE<SkSamplingMode>(SkSamplingMode::kLinear);
+
-        fo.fMipmap   = buffer.read32LE<SkMipmapMode>(SkMipmapMode::kLinear);
+    if (buffer.isVersionLT(SkPicturePriv::kCubicResamplerImageShader_Version)) {
+        if (!buffer.isVersionLT(SkPicturePriv::kFilterOptionsInImageShader_Version)) {
+            fo.fSampling = buffer.read32LE<SkSamplingMode>(SkSamplingMode::kLinear);
+            fo.fMipmap   = buffer.read32LE<SkMipmapMode>(SkMipmapMode::kLinear);
+        }
+    } else {
+        switch (fe) {
+            case kUseFilterOptions:
+                fo.fSampling = buffer.read32LE<SkSamplingMode>(SkSamplingMode::kLinear);
+                fo.fMipmap   = buffer.read32LE<SkMipmapMode>(SkMipmapMode::kLinear);
+                break;
+            case kUseCubicResampler:
+                cubic.B = buffer.readScalar();
+                cubic.C = buffer.readScalar();
+                break;
+            default:
+                break;
+        }
     }
 
     SkMatrix localMatrix;
@@ -90,17 +133,33 @@ sk_sp<SkFlattenable> SkImageShader::CreateProc(SkReadBuffer& buffer) {
         return nullptr;
     }
 
-    return (fe == kUseFilterOptions)
+    switch (fe) {
-          ? SkImageShader::Make(std::move(img), tmx, tmy, fo, &localMatrix)
+        case kUseFilterOptions:
-          : SkImageShader::Make(std::move(img), tmx, tmy, &localMatrix, fe);
+            return SkImageShader::Make(std::move(img), tmx, tmy, fo, &localMatrix);
+        case kUseCubicResampler:
+            return SkImageShader::Make(std::move(img), tmx, tmy, cubic, &localMatrix);
+        default:
+            break;
+    }
+    return SkImageShader::Make(std::move(img), tmx, tmy, &localMatrix, fe);
 }
 
 void SkImageShader::flatten(SkWriteBuffer& buffer) const {
     buffer.writeUInt((unsigned)fTileModeX);
     buffer.writeUInt((unsigned)fTileModeY);
     buffer.writeUInt((unsigned)fFilterEnum);
-    buffer.writeUInt((unsigned)fFilterOptions.fSampling);
+    switch (fFilterEnum) {
-    buffer.writeUInt((unsigned)fFilterOptions.fMipmap);
+        case kUseCubicResampler:
+            buffer.writeScalar(fCubic.B);
+            buffer.writeScalar(fCubic.C);
+            break;
+        case kUseFilterOptions:
+            buffer.writeUInt((unsigned)fFilterOptions.fSampling);
+            buffer.writeUInt((unsigned)fFilterOptions.fMipmap);
+            break;
+        default:
+            break;
+    }
     buffer.writeMatrix(this->getLocalMatrix());
     buffer.writeImage(fImage.get());
     SkASSERT(fClampAsIfUnpremul == false);
@@ -138,7 +197,8 @@ static bool legacy_shader_can_handle(const SkMatrix& inv) {
 
 SkShaderBase::Context* SkImageShader::onMakeContext(const ContextRec& rec,
                                                     SkArenaAlloc* alloc) const {
-    if (fFilterEnum == kUseFilterOptions) {
+    // we only support the old SkFilterQuality setting
+    if (fFilterEnum > kInheritFromPaint) {
         return nullptr;
     }
 
@@ -234,6 +294,20 @@ sk_sp<SkShader> SkImageShader::Make(sk_sp<SkImage> image,
     };
 }
 
+sk_sp<SkShader> SkImageShader::Make(sk_sp<SkImage> image, SkTileMode tmx, SkTileMode tmy,
+                                    SkImage::CubicResampler cubic, const SkMatrix* localMatrix) {
+    if (!(cubic.B >= 0 && cubic.B <= 1 &&
+          cubic.C >= 0 && cubic.C <= 1)) {
+        return nullptr;
+    }
+    if (!image) {
+        return sk_make_sp<SkEmptyShader>();
+    }
+    return sk_sp<SkShader>{
+        new SkImageShader(image, tmx, tmy, cubic, localMatrix)
+    };
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
 #if SK_SUPPORT_GPU
@@ -437,7 +511,9 @@ static void tweak_quality_and_inv_matrix(SkFilterQuality* quality, SkMatrix* mat
 bool SkImageShader::doStages(const SkStageRec& rec, SkImageStageUpdater* updater) const {
     SkFilterQuality quality;
     switch (fFilterEnum) {
-        case FilterEnum::kUseFilterOptions: return false;   // TODO: support these in stages
+        case FilterEnum::kUseFilterOptions:
+        case FilterEnum::kUseCubicResampler:
+            return false;   // TODO: support these in stages
         case FilterEnum::kInheritFromPaint:
             quality = rec.fPaint.getFilterQuality();
             break;
@@ -761,6 +837,12 @@ skvm::Color SkImageShader::onProgram(skvm::Builder* p,
         if (lowerWeight > 0) {
             lower = &access->lowerLevel();
         }
+    } else if (fFilterEnum == kUseCubicResampler){
+        auto* access = alloc->make<SkMipmapAccessor>(as_IB(fImage.get()), baseInv,
+                                                     SkMipmapMode::kNone);
+        upper = &access->level();
+        upperInv = post_scale(upper->dimensions(), baseInv);
+        sampling = SamplingEnum::kBicubic;
     } else {
         // Convert from the filter-quality enum to our working description:
         //  sampling : nearest, bilerp, bicubic
@@ -939,29 +1021,44 @@ skvm::Color SkImageShader::onProgram(skvm::Builder* p,
             // They're either the 16 corners of a 3x3 grid/ surrounding (x,y) at (0.5,0.5) off-center.
             skvm::F32 fx = fract(local.x + 0.5f),
                       fy = fract(local.y + 0.5f);
+            skvm::F32 wx[4],
+                      wy[4];
 
-            // See GrCubicEffect for details of these weights.
+            if (fFilterEnum == kUseCubicResampler) {
-            // TODO: these maybe don't seem right looking at gm/bicubic and GrBicubicEffect.
+                SkM44 weights = CubicResamplerMatrix(fCubic.B, fCubic.C);
-            auto near = [&](skvm::F32 t) {
+
-                // 1/18 + 9/18t + 27/18t^2 - 21/18t^3 == t ( t ( -21/18t + 27/18) + 9/18) + 1/18
+                auto dot = [](const skvm::F32 a[], const skvm::F32 b[]) {
-                return t * (t * (t * (-21/18.0f) + 27/18.0f) + 9/18.0f) + 1/18.0f;
+                    return a[0]*b[0] + a[1]*b[1] + a[2]*b[2] + a[3]*b[3];
-            };
+                };
-            auto far = [&](skvm::F32 t) {
+                const skvm::F32 tmpx[] =  { p->splat(1.0f), fx, fx*fx, fx*fx*fx };
-                // 0/18 + 0/18*t - 6/18t^2 + 7/18t^3 == t^2 (7/18t - 6/18)
+                const skvm::F32 tmpy[] =  { p->splat(1.0f), fy, fy*fy, fy*fy*fy };
-                return t * t * (t * (7/18.0f) - 6/18.0f);
+
-            };
+                for (int row = 0; row < 4; ++row) {
-            const skvm::F32 wx[] =  {
+                    SkV4 r = weights.row(row);
-                far (1.0f - fx),
+                    skvm::F32 ru[] = {
-                near(1.0f - fx),
+                        p->uniformF(uniforms->pushF(r[0])),
-                near(       fx),
+                        p->uniformF(uniforms->pushF(r[1])),
-                far (       fx),
+                        p->uniformF(uniforms->pushF(r[2])),
-            };
+                        p->uniformF(uniforms->pushF(r[3])),
-            const skvm::F32 wy[] = {
+                    };
-                far (1.0f - fy),
+                    wx[row] = dot(ru, tmpx);
-                near(1.0f - fy),
+                    wy[row] = dot(ru, tmpy);
-                near(       fy),
+                }
-                far (       fy),
+            } else {    // legacy hard-coded weights
-            };
+                auto near = [&](skvm::F32 t) {
+                    // 1/18 + 9/18t + 27/18t^2 - 21/18t^3 == t ( t ( -21/18t + 27/18) + 9/18) + 1/18
+                    return t * (t * (t * (-21/18.0f) + 27/18.0f) + 9/18.0f) + 1/18.0f;
+                };
+                auto far = [&](skvm::F32 t) {
+                    // 0/18 + 0/18*t - 6/18t^2 + 7/18t^3 == t^2 (7/18t - 6/18)
+                    return t * t * (t * (7/18.0f) - 6/18.0f);
+                };
+
+                wx[0] = far (1.0f - fx);  wy[0] = far (1.0f - fy);
+                wx[1] = near(1.0f - fx);  wy[1] = near(1.0f - fy);
+                wx[2] = near(       fx);  wy[2] = near(       fy);
+                wx[3] = far (       fx);  wy[3] = far (       fy);
+            }
 
             skvm::Color c;
             c.r = c.g = c.b = c.a = p->splat(0.0f);

src/shaders/SkImageShader.h

@@ -26,6 +26,10 @@ public:
         kInheritFromPaint,
         // this signals we should use the new SkFilterOptions
         kUseFilterOptions,
+        // use fCubic and ignore FilterOptions
+        kUseCubicResampler,
+
+        kLast = kUseCubicResampler,
     };
 
     static sk_sp<SkShader> Make(sk_sp<SkImage>,
@@ -41,12 +45,20 @@ public:
                                 const SkFilterOptions&,
                                 const SkMatrix* localMatrix);
 
+    static sk_sp<SkShader> Make(sk_sp<SkImage>,
+                                SkTileMode tmx,
+                                SkTileMode tmy,
+                                SkImage::CubicResampler,
+                                const SkMatrix* localMatrix);
+
     bool isOpaque() const override;
 
 #if SK_SUPPORT_GPU
     std::unique_ptr<GrFragmentProcessor> asFragmentProcessor(const GrFPArgs&) const override;
 #endif
 
+    static SkM44 CubicResamplerMatrix(float B, float C);
+
 private:
     SK_FLATTENABLE_HOOKS(SkImageShader)
 
@@ -61,6 +73,11 @@ private:
                   SkTileMode tmy,
                   const SkFilterOptions&,
                   const SkMatrix* localMatrix);
+    SkImageShader(sk_sp<SkImage>,
+                  SkTileMode tmx,
+                  SkTileMode tmy,
+                  SkImage::CubicResampler,
+                  const SkMatrix* localMatrix);
 
     void flatten(SkWriteBuffer&) const override;
 #ifdef SK_ENABLE_LEGACY_SHADERCONTEXT
@@ -80,8 +97,9 @@ private:
 
     SkFilterQuality resolveFiltering(SkFilterQuality paintQuality) const {
         switch (fFilterEnum) {
-            case kUseFilterOptions: return kNone_SkFilterQuality;   // TODO
+            case kUseCubicResampler: return kHigh_SkFilterQuality;   // TODO: handle explicitly
-            case kInheritFromPaint: return paintQuality;
+            case kUseFilterOptions:  return kNone_SkFilterQuality;   // TODO: handle explicitly
+            case kInheritFromPaint:  return paintQuality;
             default: break;
         }
         return (SkFilterQuality)fFilterEnum;
@@ -95,6 +113,8 @@ private:
 
     // only use this if fFilterEnum == kUseFilterOptions
     SkFilterOptions  fFilterOptions;
+    // only use this if fFilterEnum == kUseCubicResampler
+    SkImage::CubicResampler fCubic;
 
     friend class SkShaderBase;
     typedef SkShaderBase INHERITED;

tests/ImageTest.cpp

@@ -1471,3 +1471,33 @@ DEF_GPUTEST_FOR_ALL_CONTEXTS(ImageFlush, reporter, ctxInfo) {
     i2->flushAndSubmit(direct);
     REPORTER_ASSERT(reporter, numSubmits() == 1);
 }
+
+#include "src/shaders/SkImageShader.h"
+
+constexpr SkM44 gCentripetalCatmulRom
+    (0.0f/2, -1.0f/2,  2.0f/2, -1.0f/2,
+     2.0f/2,  0.0f/2, -5.0f/2,  3.0f/2,
+     0.0f/2,  1.0f/2,  4.0f/2, -3.0f/2,
+     0.0f/2,  0.0f/2, -1.0f/2,  1.0f/2);
+
+constexpr SkM44 gMitchellNetravali
+    ( 1.0f/18, -9.0f/18,  15.0f/18,  -7.0f/18,
+     16.0f/18,  0.0f/18, -36.0f/18,  21.0f/18,
+      1.0f/18,  9.0f/18,  27.0f/18, -21.0f/18,
+      0.0f/18,  0.0f/18,  -6.0f/18,   7.0f/18);
+
+DEF_TEST(image_cubicresampler, reporter) {
+    auto diff = [reporter](const SkM44& a, const SkM44& b) {
+        const float tolerance = 0.000001f;
+        for (int r = 0; r < 4; ++r) {
+            for (int c = 0; c < 4; ++c) {
+                float d = std::abs(a.rc(r, c) - b.rc(r, c));
+                REPORTER_ASSERT(reporter, d <= tolerance);
+            }
+        }
+    };
+
+    diff(SkImageShader::CubicResamplerMatrix(1.0f/3, 1.0f/3), gMitchellNetravali);
+
+    diff(SkImageShader::CubicResamplerMatrix(0, 1.0f/2), gCentripetalCatmulRom);
+}