Initial checkin of GM verifier framework

The goal of the verifier framework is to enable opt-in checks of the
images produced by individual GMs. The basis of verification will be
comparing the rendered output of a GM against a source-of-truth image,
such as one generated by the CPU backend.

In the short term this can enable coarse-grained sanity checks for a
subset of GMs to catch e.g. egregious rendering bugs. In the longer term
this can provide an SkQP-style suite of tests that can be run across
many/all GMs to provide a vote of confidence in the rendering
correctness of new devices.

Bug: skia:9855
Change-Id: Id7310de8005ffa7e8eb2fd0e4008f5f8db1419ab
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/267761
Commit-Queue: Derek Sollenberger <djsollen@google.com>
Auto-Submit: Tyler Denniston <tdenniston@google.com>
Reviewed-by: Derek Sollenberger <djsollen@google.com>

dm/DM.cpp

@@ -7,6 +7,7 @@
 
 #include "dm/DMJsonWriter.h"
 #include "dm/DMSrcSink.h"
+#include "gm/verifiers/gmverifier.h"
 #include "include/codec/SkCodec.h"
 #include "include/core/SkBBHFactory.h"
 #include "include/core/SkColorPriv.h"
@@ -150,6 +151,9 @@ static DEFINE_string(properties, "",
 
 static DEFINE_bool(rasterize_pdf, false, "Rasterize PDFs when possible.");
 
+static DEFINE_bool(runVerifiers, false,
+                   "if true, run SkQP-style verification of GM-produced images.");
+
 
 #if defined(__MSVC_RUNTIME_CHECKS)
 #include <rtcapi.h>
@@ -1132,6 +1136,11 @@ struct Task {
                 }
             }
 
+            // Run verifiers if specified
+            if (FLAGS_runVerifiers) {
+                RunGMVerifiers(task, bitmap);
+            }
+
             // We're likely switching threads here, so we must capture by value, [=] or [foo,bar].
             SkStreamAsset* data = stream.detachAsStream().release();
             gDefinitelyThreadSafeWork.add([task,name,bitmap,data]{
@@ -1388,6 +1397,23 @@ struct Task {
             }
         }
     }
+
+    static void RunGMVerifiers(const Task& task, const SkBitmap& actualBitmap) {
+        const SkString name = task.src->name();
+        auto verifierList = task.src->getVerifiers();
+        if (verifierList == nullptr) {
+            return;
+        }
+
+        skiagm::verifiers::VerifierResult
+            res = verifierList->verifyAll(task.sink->colorInfo(), actualBitmap);
+        if (!res.ok()) {
+            fail(
+                SkStringPrintf(
+                    "%s %s %s %s: verifier failed: %s", task.sink.tag.c_str(), task.src.tag.c_str(),
+                    task.src.options.c_str(), name.c_str(), res.message().c_str()));
+        }
+    }
 };
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

dm/DMSrcSink.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "dm/DMSrcSink.h"
+#include "gm/verifiers/gmverifier.h"
 #include "include/codec/SkAndroidCodec.h"
 #include "include/codec/SkCodec.h"
 #include "include/core/SkColorSpace.h"
@@ -108,6 +109,11 @@ void GMSrc::modifyGrContextOptions(GrContextOptions* options) const {
     gm->modifyGrContextOptions(options);
 }
 
+std::unique_ptr<skiagm::verifiers::VerifierList> GMSrc::getVerifiers() const {
+    std::unique_ptr<skiagm::GM> gm(fFactory());
+    return gm->getVerifiers();
+}
+
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
 BRDSrc::BRDSrc(Path path, Mode mode, CodecSrc::DstColorType dstColorType, uint32_t sampleSize)

dm/DMSrcSink.h

@@ -24,6 +24,12 @@
 
 //#define TEST_VIA_SVG
 
+namespace skiagm {
+namespace verifiers {
+class VerifierList;
+}
+}
+
 namespace DM {
 
 // This is just convenience.  It lets you use either return "foo" or return SkStringPrintf(...).
@@ -82,6 +88,11 @@ struct Src {
     virtual SkISize size(int) const { return this->size(); }
     // Force Tasks using this Src to run on the main thread?
     virtual bool serial() const { return false; }
+
+    /** Return a list of verifiers for the src, or null if no verifiers should be run .*/
+    virtual std::unique_ptr<skiagm::verifiers::VerifierList> getVerifiers() const {
+        return nullptr;
+    }
 };
 
 struct Sink {
@@ -97,6 +108,9 @@ struct Sink {
     virtual const char* fileExtension() const  = 0;
 
     virtual SinkFlags flags() const = 0;
+
+    /** Returns the color type and space used by the sink. */
+    virtual SkColorInfo colorInfo() const { return SkColorInfo(); }
 };
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
@@ -110,6 +124,8 @@ public:
     Name name() const override;
     void modifyGrContextOptions(GrContextOptions* options) const override;
 
+    std::unique_ptr<skiagm::verifiers::VerifierList> getVerifiers() const override;
+
 private:
     skiagm::GMFactory fFactory;
 };
@@ -351,6 +367,9 @@ public:
         return SinkFlags{ SinkFlags::kGPU, SinkFlags::kDirect, ms };
     }
     const GrContextOptions& baseContextOptions() const { return fBaseContextOptions; }
+    SkColorInfo colorInfo() const override {
+        return SkColorInfo(fColorType, fAlphaType, fColorSpace);
+    }
 
 private:
     sk_gpu_test::GrContextFactory::ContextType        fContextType;

gm/gm.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "gm/gm.h"
+#include "gm/verifiers/gmverifier.h"
 #include "include/core/SkBitmap.h"
 #include "include/core/SkBlendMode.h"
 #include "include/core/SkCanvas.h"
@@ -143,6 +144,11 @@ bool GM::animate(double nanos) { return this->onAnimate(nanos); }
 bool GM::runAsBench() const { return false; }
 void GM::modifyGrContextOptions(GrContextOptions* options) {}
 
+std::unique_ptr<verifiers::VerifierList> GM::getVerifiers() const {
+    // No verifiers by default.
+    return nullptr;
+}
+
 void GM::onOnceBeforeDraw() {}
 
 bool GM::onAnimate(double /*nanos*/) { return false; }

gm/gm.h

@@ -8,6 +8,7 @@
 #ifndef skiagm_DEFINED
 #define skiagm_DEFINED
 
+#include "gm/verifiers/gmverifier.h"
 #include "include/core/SkColor.h"
 #include "include/core/SkScalar.h"
 #include "include/core/SkSize.h"
@@ -148,6 +149,8 @@ namespace skiagm {
 
         virtual void modifyGrContextOptions(GrContextOptions*);
 
+        virtual std::unique_ptr<verifiers::VerifierList> getVerifiers() const;
+
     protected:
         virtual void onOnceBeforeDraw();
         virtual DrawResult onDraw(SkCanvas* canvas, SkString* errorMsg);
@@ -176,6 +179,12 @@ namespace skiagm {
     class GpuGM : public GM {
     public:
         GpuGM(SkColor backgroundColor = SK_ColorWHITE) : GM(backgroundColor) {}
+
+        // TODO(tdenniston): Currently GpuGMs don't have verifiers (because they do not render on
+        //   CPU), but we may want to be able to verify the output images standalone, without
+        //   requiring a gold image for comparison.
+        std::unique_ptr<verifiers::VerifierList> getVerifiers() const override { return nullptr; }
+
     private:
         using GM::onDraw;
         DrawResult onDraw(SkCanvas*, SkString* errorMsg) final;

gm/verifiers/gmverifier.cpp

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2020 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "gm/gm.h"
+#include "gm/verifiers/gmverifier.h"
+#include "include/core/SkBitmap.h"
+#include "include/core/SkCanvas.h"
+#include "include/core/SkSurface.h"
+#include "include/effects/SkImageFilters.h"
+#include "include/encode/SkPngEncoder.h"
+#include "src/utils/SkOSPath.h"
+
+/** Checks the given VerifierResult. If it is not ok, returns it. */
+#define RETURN_NOT_OK(res)  if (!(res).ok()) return (res)
+
+namespace skiagm {
+namespace verifiers {
+
+VerifierResult::VerifierResult() : VerifierResult(Code::kOk, SkString("Ok")) {}
+
+VerifierResult::VerifierResult(VerifierResult::Code code, const SkString& msg) :
+    fCode(code), fMessage(msg) {}
+
+bool VerifierResult::ok() const {
+    return fCode == Code::kOk;
+}
+
+const SkString& VerifierResult::message() const {
+    return fMessage;
+}
+
+VerifierResult VerifierResult::Ok() {
+    return VerifierResult(Code::kOk, SkString("Ok"));
+}
+
+VerifierResult VerifierResult::Fail(const SkString& msg) {
+    return VerifierResult(Code::kFail, msg);
+}
+
+GMVerifier::GMVerifier(InputType inputType) : fInputType(inputType) {}
+
+GMVerifier::~GMVerifier() {}
+
+bool GMVerifier::needsGoldImage() const {
+    return fInputType == InputType::kGoldImageRequired;
+}
+
+VerifierResult GMVerifier::verify(const SkBitmap& gold, const SkBitmap& actual) {
+    // Call into specific implementation.
+    return verifyWithGold(actual.bounds(), gold, actual);
+}
+
+VerifierResult GMVerifier::verify(const SkBitmap& actual) {
+    // Call into specific implementation.
+    return verify(actual.bounds(), actual);
+}
+
+SkBitmap GMVerifier::RenderGoldBmp(skiagm::GM* gm, const SkColorInfo& colorInfo) {
+    SkASSERT(gm);
+
+    // Call into the GM instance to get the initial image.
+    const SkISize size = gm->getISize();
+    SkBitmap goldBmp;
+    goldBmp.allocPixels(SkImageInfo::Make(size, colorInfo));
+    SkCanvas canvas(goldBmp);
+    gm->draw(&canvas);
+
+    // Convert into common verifier colorspace.
+    SkBitmap goldVerifierBmp;
+    goldVerifierBmp.allocPixels(SkImageInfo::Make(size, VerifierColorInfo()));
+    SkCanvas verifierCanvas(goldVerifierBmp);
+    verifierCanvas.drawBitmap(goldBmp, 0, 0);
+
+    return goldVerifierBmp;
+}
+
+SkColorInfo GMVerifier::VerifierColorInfo() {
+    return SkColorInfo(
+        kRGBA_F16_SkColorType, kPremul_SkAlphaType,
+        SkColorSpace::MakeRGB(SkNamedTransferFn::kRec2020, SkNamedGamut::kRec2020));
+}
+
+VerifierResult GMVerifier::makeError(const SkString& msg) const {
+    return VerifierResult::Fail(SkStringPrintf("[%s] %s", name().c_str(), msg.c_str()));
+}
+
+VerifierList::VerifierList(GM* gm) : fGM(gm), fFailedVerifier(nullptr) {}
+
+void VerifierList::add(std::unique_ptr<GMVerifier> verifier) {
+    fVerifiers.push_back(std::move(verifier));
+}
+
+bool VerifierList::needsGoldImage() const {
+    for (const auto& v : fVerifiers) {
+        if (v->needsGoldImage()) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+VerifierResult VerifierList::verifyAll(const SkColorInfo& colorInfo, const SkBitmap& actual) {
+    // Render the golden image if any verifiers need it.
+    SkBitmap goldBmp;
+    if (needsGoldImage()) {
+        goldBmp = GMVerifier::RenderGoldBmp(fGM, colorInfo);
+    }
+
+    for (const auto& v : fVerifiers) {
+        fFailedVerifier = v.get();
+        if (v->needsGoldImage()) {
+            RETURN_NOT_OK(v->verify(goldBmp, actual));
+        } else {
+            RETURN_NOT_OK(v->verify(actual));
+        }
+    }
+
+    fFailedVerifier = nullptr;
+    return VerifierResult::Ok();
+}
+
+}
+}

gm/verifiers/gmverifier.h

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2020 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef gmverifier_DEFINED
+#define gmverifier_DEFINED
+
+#include "include/core/SkColor.h"
+#include "include/core/SkRect.h"
+#include "include/core/SkString.h"
+
+#include <vector>
+
+class SkBitmap;
+
+namespace skiagm {
+
+class GM;
+
+namespace verifiers {
+
+/** Result type for GM verifiers. */
+class VerifierResult {
+public:
+    VerifierResult();
+
+    /** Returns true if the result is ok (non-error). */
+    bool ok() const;
+
+    /** Returns reference to any message associated with the result. */
+    const SkString& message() const;
+
+    /** Constructs an "ok" (non-error) result. */
+    static VerifierResult Ok();
+
+    /** Constructs a "fail" (error) result with a specific message. */
+    static VerifierResult Fail(const SkString& msg);
+
+private:
+    /** Underlying error code. */
+    enum class Code {
+        kOk, kFail
+    };
+
+    /** Result code */
+    Code fCode;
+
+    /** Result message (may be empty). */
+    SkString fMessage;
+
+    /** Private constructor for a result with a specific code and message. */
+    VerifierResult(Code code, const SkString& msg);
+};
+
+/**
+ * Abstract base class for GM verifiers. A verifier checks the rendered output image of a GM.
+ *
+ * Different verifiers perform different types of transforms and checks. Verifiers may check the
+ * output of a GM against a given "golden" image which represents the correct output, or just
+ * check the output image of the GM by itself.
+ *
+ * Most verifiers have configurable fuzziness in the comparisons performed against the golden image.
+ *
+ * Subclasses should inherit from one of StandaloneVerifier or GoldImageVerifier instead of
+ * directly from this base class.
+ */
+class GMVerifier {
+public:
+    GMVerifier() = delete;
+
+    virtual ~GMVerifier();
+
+    /** Returns the human-friendly name of the verifier. */
+    virtual SkString name() const = 0;
+
+    /** Returns true if this verifier needs the gold image as input. */
+    bool needsGoldImage() const;
+
+    /**
+     * Runs the verifier. This method should be used if the verifier needs the gold image as input.
+     *
+     * @param gold Bitmap containing the "correct" image.
+     * @param actual Bitmap containing rendered output of a GM.
+     * @return Ok if the verification passed, or an error if not.
+     */
+    VerifierResult verify(const SkBitmap& gold, const SkBitmap& actual);
+
+    /**
+     * Runs the verifier.
+     *
+     * @param actual Bitmap containing rendered output of a GM.
+     * @return Ok if the verification passed, or an error if not.
+     */
+    VerifierResult verify(const SkBitmap& actual);
+
+    /** Renders the GM using the "golden" configuration. This is common across all GMs/verifiers. */
+    static SkBitmap RenderGoldBmp(skiagm::GM* gm, const SkColorInfo& colorInfo);
+
+    /**
+     * Gets the color information that all verifier inputs should be transformed into.
+     *
+     * The primary reason for having a single shared colorspace/color type is making per-pixel
+     * comparisons easier. Both the image under test and gold image are transformed into a shared
+     * colorspace which allows for getting per-pixel colors in SkColor4f.
+     */
+    static SkColorInfo VerifierColorInfo();
+
+protected:
+    /** The type of input required for the verifier. */
+    enum class InputType {
+        kGoldImageRequired, kStandalone
+    };
+
+    /** Set depending if the verifier needs a golden image as an input. */
+    InputType fInputType;
+
+    /** Constructor. */
+    GMVerifier(InputType inputType);
+
+    /** Implementation of the verification. */
+    virtual VerifierResult verifyWithGold(
+        const SkIRect& region, const SkBitmap& gold, const SkBitmap& actual) = 0;
+
+    /** Implementation of the verification. */
+    virtual VerifierResult verify(const SkIRect& region, const SkBitmap& actual) = 0;
+
+    /** Returns an error result formatted appropriately. */
+    VerifierResult makeError(const SkString& msg) const;
+};
+
+/**
+ * A verifier that operates standalone on the given input image (no comparison against a golden
+ * image).
+ */
+class StandaloneVerifier : public GMVerifier {
+public:
+    StandaloneVerifier() : GMVerifier(InputType::kStandalone) {}
+
+protected:
+    VerifierResult verifyWithGold(const SkIRect&, const SkBitmap&, const SkBitmap&) final {
+        return makeError(SkString("Verifier does not accept gold image input"));
+    }
+};
+
+/**
+ * A verifier that operates compares input image against a golden image.
+ */
+class GoldImageVerifier : public GMVerifier {
+public:
+    GoldImageVerifier() : GMVerifier(InputType::kGoldImageRequired) {}
+
+protected:
+    VerifierResult verify(const SkIRect&, const SkBitmap&) final {
+        return makeError(SkString("Verifier does not accept standalone input"));
+    }
+};
+
+/** A list of GM verifiers. */
+class VerifierList {
+public:
+    /** Constructs a VerifierList with the given gm instance. */
+    explicit VerifierList(GM* gm);
+
+    /** Adds a verifier to the list of verifiers. */
+    void add(std::unique_ptr<GMVerifier> verifier);
+
+    /**
+     * Runs all verifiers against the given input. If any verifiers fail, returns the first error.
+     * Else, returns ok. This version can be used if no verifiers in the list require the gold
+     * image as input.
+     */
+    VerifierResult verifyAll(const SkColorInfo& colorInfo, const SkBitmap& actual);
+
+private:
+    /** The parent GM instance of this VerifierList. */
+    GM* fGM;
+
+    /** The list of verifiers. */
+    std::vector<std::unique_ptr<GMVerifier>> fVerifiers;
+
+    /** After running, set to the first verifier that failed, or nullptr if none failed. */
+    const GMVerifier* fFailedVerifier;
+
+    /** Returns true if any verifiers in the list need the gold image as input. */
+    bool needsGoldImage() const;
+};
+
+}
+}
+
+#endif

gn/gm.gni

@@ -389,6 +389,7 @@ gm_sources = [
   "$_gm/xfermodes3.cpp",
   "$_gm/yuvtorgbeffect.cpp",
   "$_gm/yuv420_odd_dim.cpp",
+  "$_gm/verifiers/gmverifier.cpp",
 ]
 
 gl_gm_sources = [