Add ES2 conformance test harness to dm.

This CL adds a few more exceptions to our ES2 test import, and adds the
dm code which actually runs the tests.

Change-Id: If6691dd35931f4f10262d3a1eff020c2c347ca59
Bug: skia:12484
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/459124
Auto-Submit: John Stiles <johnstiles@google.com>
Commit-Queue: Ethan Nicholas <ethannicholas@google.com>
Reviewed-by: Ethan Nicholas <ethannicholas@google.com>

gn/tests.gni

@@ -241,6 +241,7 @@ tests_sources = [
   "$_tests/SkSLDSLErrorLineNumbers.cpp",
   "$_tests/SkSLDSLOnlyTest.cpp",
   "$_tests/SkSLDSLTest.cpp",
+  "$_tests/SkSLES2ConformanceTest.cpp",
   "$_tests/SkSLGLSLTestbed.cpp",
   "$_tests/SkSLInterpreterTest.cpp",
   "$_tests/SkSLMemoryLayoutTest.cpp",

resources/sksl/es2_conformance/import_conformance_tests.py

@@ -12,6 +12,9 @@
 #     cat ${TEST_FILES}/*.test | ./import_conformance_tests.py
 #
 # This will generate two directories, "pass" and "fail", containing finished runtime shaders.
+# Note that some tests were originally designed to fail, because a conforming compiler should not
+# allow the program. A handful of others fail because they are incompatible with SkSL. This script
+# identifies SkSL-incompatible tests them and moves them from "pass" to "fail" automatically.
 #
 # Not all ES2 test files are meaningful in SkSL. These input files are not supported:
 # - linkage.test: Runtime Effects only handle fragment processing
@@ -103,6 +106,7 @@ for c in testCases:
     # Parse the case body
     skipTest = ''
     expectPass = True
+    allowMismatch = False
     testCode = ''
     inputs = []
     outputs = []
@@ -150,13 +154,25 @@ for c in testCases:
         print("skipped %s (%s)" % (testName, skipTest))
         continue
 
+    # The test is safe to run, but it might not get the same result.
+    # SkSL does not guarantee that function arguments will always be evaluated left-to-right.
+    if re.fullmatch('argument_eval_order_[12]', testName):
+        allowMismatch = True
+        print("allowing mismatch in %s" % testName)
+
+    # Switch tests to a "fail" expectation instead of "pass" when SkSL and GLSL disagree.
     # SkSL does not support casts which discard elements such as `float(myFloat4)`.
-    # Switch these tests to a "fail" expectation instead of "pass."
     if (re.fullmatch('(vec|bvec|ivec)[234]_to_(float|int|bool)', testName) or
         re.fullmatch('(vec|bvec|ivec)[34]_to_(vec|bvec|ivec)2', testName) or
         re.fullmatch('(vec|bvec|ivec)[4]_to_(vec|bvec|ivec)3', testName) or
+    # SkSL requires that function out-parameters match the precision of the variable passed in.
+        re.fullmatch('(out|inout)_lowp_(int|float)', testName) or
     # SkSL rejects code that fails to return a value; GLSL ES2 allows it.
-        testName == 'missing_returns'):
+        testName == 'missing_returns' or
+    # SkSL does not support a global `precision` directive.
+        testName == 'default_vs_explicit_precision' or
+    # SkSL does not allow variables to be created without an enclosing scope.
+        testName == 'variable_in_if_hides_global_variable'):
         assert expectPass
         expectPass = False
         print("moved %s to fail" % testName)
@@ -215,9 +231,10 @@ for c in testCases:
 
             # Verify output values inside ${OUTPUT}.
             outputChecks = "return true"
-            for v in outputs:
+            if not allowMismatch:
-                if len(v[2]) > varIndex:
+                for v in outputs:
-                    outputChecks += " && (%s == %s)" % (v[1], v[2][varIndex])
+                    if len(v[2]) > varIndex:
+                        outputChecks += " && (%s == %s)" % (v[1], v[2][varIndex])
 
             outputChecks += ";\n"
 

tests/SkSLES2ConformanceTest.cpp

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+/**
+ * This test relies on GLSL ES2 conformance test files, which are not included in Skia.
+ *
+ * To run the test suite, open `resources/sksl/es2_conformance/import_conformance_tests.py` and
+ * follow the instructions at the top to download and import the test suite.
+ */
+
+#include "gm/gm.h"
+#include "include/core/SkBitmap.h"
+#include "include/core/SkCanvas.h"
+#include "include/core/SkData.h"
+#include "include/core/SkFont.h"
+#include "include/core/SkPaint.h"
+#include "include/core/SkSize.h"
+#include "include/core/SkString.h"
+#include "include/core/SkStringView.h"
+#include "include/core/SkSurface.h"
+#include "include/effects/SkGradientShader.h"
+#include "include/effects/SkImageFilters.h"
+#include "include/effects/SkRuntimeEffect.h"
+#include "include/utils/SkRandom.h"
+#include "src/core/SkOSFile.h"
+#include "src/core/SkRuntimeEffectPriv.h"
+#include "src/gpu/GrCaps.h"
+#include "src/gpu/GrDirectContextPriv.h"
+#include "src/utils/SkOSPath.h"
+#include "tests/Test.h"
+#include "tools/Resources.h"
+#include "tools/ToolUtils.h"
+
+static void test_expect_fail(skiatest::Reporter* r, const char* testFile) {
+    SkRuntimeEffect::Options options{};
+    sk_sp<SkData> shaderData = GetResourceAsData(testFile);
+    if (!shaderData) {
+        ERRORF(r, "%s: Unable to load file", SkOSPath::Basename(testFile).c_str());
+        return;
+    }
+
+    SkString shaderString{reinterpret_cast<const char*>(shaderData->bytes()), shaderData->size()};
+    SkRuntimeEffect::Result result = SkRuntimeEffect::MakeForShader(shaderString, options);
+    if (result.effect) {
+        ERRORF(r, "%s: Expected failure, but compiled successfully",
+                  SkOSPath::Basename(testFile).c_str());
+        return;
+    }
+}
+
+static void test_expect_pass(skiatest::Reporter* r, SkSurface* surface, const char* testFile) {
+    SkRuntimeEffect::Options options{};
+    sk_sp<SkData> shaderData = GetResourceAsData(testFile);
+    if (!shaderData) {
+        ERRORF(r, "%s: Unable to load file", testFile);
+        return;
+    }
+
+    SkString shaderString{reinterpret_cast<const char*>(shaderData->bytes()), shaderData->size()};
+    SkRuntimeEffect::Result result = SkRuntimeEffect::MakeForShader(shaderString, options);
+    if (!result.effect) {
+        ERRORF(r, "%s: %s", testFile, result.errorText.c_str());
+        return;
+    }
+
+    SkRuntimeShaderBuilder builder(result.effect);
+    sk_sp<SkShader> shader = builder.makeShader(/*localMatrix=*/nullptr, /*isOpaque=*/true);
+    if (!shader) {
+        ERRORF(r, "%s: Unable to build shader", testFile);
+        return;
+    }
+
+    SkPaint paintShader;
+    paintShader.setShader(shader);
+    surface->getCanvas()->drawRect(SkRect::MakeWH(1, 1), paintShader);
+
+    SkBitmap bitmap;
+    REPORTER_ASSERT(r, bitmap.tryAllocPixels(surface->imageInfo()));
+    REPORTER_ASSERT(r, surface->readPixels(bitmap.info(), bitmap.getPixels(), bitmap.rowBytes(),
+                                           /*srcX=*/0, /*srcY=*/0));
+
+    SkColor color = bitmap.getColor(0, 0);
+    if (color != SkColorSetARGB(0xFF, 0x00, 0xFF, 0x00)) {
+        ERRORF(r, "%s: Expected solid green. Actual:\n"
+                  "RRGGBBAA\n"
+                  "%02X%02X%02X%02X",
+                  testFile,
+                  SkColorGetR(color), SkColorGetG(color), SkColorGetB(color), SkColorGetA(color));
+    }
+}
+
+static void iterate_dir(const char* directory, const std::function<void(const char*)>& run) {
+    SkString resourceDirectory = GetResourcePath(directory);
+    SkOSFile::Iter iter(resourceDirectory.c_str(), ".rts");
+    SkString name;
+
+    while (iter.next(&name, /*getDir=*/false)) {
+        SkString path(SkOSPath::Join(directory, name.c_str()));
+        run(path.c_str());
+    }
+}
+
+DEF_TEST(SkSL_ES2Conformance_Pass_CPU, r) {
+    const SkImageInfo info = SkImageInfo::MakeN32Premul(1, 1);
+    sk_sp<SkSurface> surface(SkSurface::MakeRaster(info));
+
+    iterate_dir("sksl/es2_conformance/pass/", [&](const char* path) {
+        test_expect_pass(r, surface.get(), path);
+    });
+}
+
+DEF_GPUTEST_FOR_RENDERING_CONTEXTS(SkSL_ES2Conformance_Pass_GPU, r, ctxInfo) {
+    const SkImageInfo info = SkImageInfo::MakeN32Premul(1, 1);
+    sk_sp<SkSurface> surface(SkSurface::MakeRenderTarget(ctxInfo.directContext(),
+                                                         SkBudgeted::kNo, info));
+    iterate_dir("sksl/es2_conformance/pass/", [&](const char* path) {
+        test_expect_pass(r, surface.get(), path);
+    });
+}
+
+DEF_TEST(SkSL_ES2Conformance_Fail, r) {
+    iterate_dir("sksl/es2_conformance/fail/", [&](const char* path) {
+        test_expect_fail(r, path);
+    });
+}