GM: allow caller to specify which result types trigger an error

Review URL: https://codereview.chromium.org/14187007

git-svn-id: http://skia.googlecode.com/svn/trunk@8652 2bbb7eff-a529-9590-31e7-b0007b416f81

gm/gm_error.h

@@ -57,6 +57,22 @@ namespace skiagm {
         return "Unknown";
     }
 
+    /**
+     * Fills in "type" with the ErrorType associated with name "name".
+     * Returns true if we found one, false if it is an unknown type name.
+     */
+    static bool getErrorTypeByName(const char name[], ErrorType *type) {
+        for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
+            ErrorType thisType = static_cast<ErrorType>(typeInt);
+            const char *thisTypeName = getErrorTypeName(thisType);
+            if (0 == strcmp(thisTypeName, name)) {
+                *type = thisType;
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * A combination of 0 or more ErrorTypes.
      */
@@ -93,6 +109,26 @@ namespace skiagm {
             return !(0 == (this->fBitfield & (1 << type)));
         }
 
+        /**
+         * Returns a string representation of all ErrorTypes in this
+         * ErrorCombination.
+         *
+         * @param separator text with which to separate ErrorType names
+         */
+        SkString asString(const char separator[]) const {
+            SkString s;
+            for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
+                ErrorType type = static_cast<ErrorType>(typeInt);
+                if (this->includes(type)) {
+                    if (!s.isEmpty()) {
+                        s.append(separator);
+                    }
+                    s.append(getErrorTypeName(type));
+                }
+            }
+            return s;
+        }
+
         /**
          * Returns a new ErrorCombination, which includes the union of all
          * ErrorTypes in two ErrorCombination objects (this and other).

gm/gmmain.cpp

@@ -174,13 +174,14 @@ static PipeFlagComboData gPipeWritingFlagCombos[] = {
         | SkGPipeWriter::kSharedAddressSpace_Flag }
 };
 
+const static ErrorCombination kDefaultIgnorableErrorTypes = ErrorCombination()
+    .plus(kMissingExpectations_ErrorType)
+    .plus(kIntentionallySkipped_ErrorType);
+
 class GMMain {
 public:
-    GMMain() : fUseFileHierarchy(false), fMismatchPath(NULL), fTestsRun(0),
+    GMMain() : fUseFileHierarchy(false), fIgnorableErrorTypes(kDefaultIgnorableErrorTypes),
-               fRenderModesEncountered(1) {
+               fMismatchPath(NULL), fTestsRun(0), fRenderModesEncountered(1) {}
-        fIgnorableErrorCombination.add(kMissingExpectations_ErrorType);
-        fIgnorableErrorCombination.add(kIntentionallySkipped_ErrorType);
-    }
 
     SkString make_name(const char shortName[], const char configName[]) {
         SkString name;
@@ -291,7 +292,7 @@ public:
         int significantErrors = 0;
         for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
             ErrorType type = static_cast<ErrorType>(typeInt);
-            if (!fIgnorableErrorCombination.includes(type)) {
+            if (!fIgnorableErrorTypes.includes(type)) {
                 significantErrors += fFailedTests[type].count();
             }
         }
@@ -305,7 +306,7 @@ public:
      * @param verbose whether to be all verbose about it
      */
     void DisplayResultTypeSummary(ErrorType type, bool verbose) {
-        bool isIgnorableType = fIgnorableErrorCombination.includes(type);
+        bool isIgnorableType = fIgnorableErrorTypes.includes(type);
 
         SkString line;
         if (isIgnorableType) {
@@ -1105,7 +1106,7 @@ public:
     //
 
     bool fUseFileHierarchy;
-    ErrorCombination fIgnorableErrorCombination;
+    ErrorCombination fIgnorableErrorTypes;
 
     const char* fMismatchPath;
 
@@ -1209,6 +1210,18 @@ DEFINE_string(gpuCacheSize, "", "<bytes> <count>: Limit the gpu cache to byte si
 #endif
 DEFINE_bool(hierarchy, false, "Whether to use multilevel directory structure "
             "when reading/writing files.");
+// TODO(epoger): Maybe should make SkCommandLineFlags handle default string
+// values differently, so that the first definition of ignoreErrorTypes worked?
+#if 0
+DEFINE_string(ignoreErrorTypes, kDefaultIgnorableErrorTypes.asString(" ").c_str(),
+              "Space-separated list of ErrorTypes that should be ignored. If any *other* error "
+              "types are encountered, the tool will exit with a nonzero return value.");
+#else
+DEFINE_string(ignoreErrorTypes, "", SkString(SkString(
+              "Space-separated list of ErrorTypes that should be ignored. If any *other* error "
+              "types are encountered, the tool will exit with a nonzero return value. "
+              "Defaults to: ") += kDefaultIgnorableErrorTypes.asString(" ")).c_str());
+#endif
 DEFINE_string(match, "",  "Only run tests whose name includes this substring/these substrings "
               "(more than one can be supplied, separated by spaces).");
 DEFINE_string(mismatchPath, "", "Write images for tests that failed due to "
@@ -1639,6 +1652,20 @@ int tool_main(int argc, char** argv) {
         }
     }
 
+    if (FLAGS_ignoreErrorTypes.count() > 0) {
+        gmmain.fIgnorableErrorTypes = ErrorCombination();
+        for (int i = 0; i < FLAGS_ignoreErrorTypes.count(); i++) {
+            ErrorType type;
+            const char *name = FLAGS_ignoreErrorTypes[i];
+            if (!getErrorTypeByName(name, &type)) {
+                gm_fprintf(stderr, "cannot find ErrorType with name '%s'\n", name);
+                return -1;
+            } else {
+                gmmain.fIgnorableErrorTypes.add(type);
+            }
+        }
+    }
+
 #if SK_SUPPORT_GPU
     if (FLAGS_gpuCacheSize.count() > 0) {
         if (FLAGS_gpuCacheSize.count() != 2) {

gm/tests/outputs/ignore-expectations-mismatch/output-expected/command_line

@@ -0,0 +1 @@
+out/Debug/gm --ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignore-expectations-mismatch/output-actual/json-summary.txt

gm/tests/outputs/ignore-expectations-mismatch/output-expected/json-summary.txt

@@ -0,0 +1,25 @@
+{
+   "actual-results" : {
+      "failed" : {
+         "565/selftest1" : {
+            "checksum" : 9512553915271796906
+         },
+         "8888/selftest1" : {
+            "checksum" : 14022967492765711532
+         }
+      },
+      "failure-ignored" : null,
+      "no-comparison" : null,
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "565/selftest1" : {
+         "checksums" : [ 11071285354315388429 ],
+         "ignore-failure" : false
+      },
+      "8888/selftest1" : {
+         "checksums" : [ 16527650414256125612 ],
+         "ignore-failure" : false
+      }
+   }
+}

gm/tests/outputs/ignore-expectations-mismatch/output-expected/return_value

@@ -0,0 +1 @@
+0

gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout

@@ -0,0 +1,14 @@
+GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels.json
+GM: drawing... selftest1 [300 200]
+GM: Ran 1 GMs
+GM: ... over  2 configs ["8888", "565"]
+GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
+GM: ... so there should be a total of 9 tests.
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: [ ] 0 NoGpuContext:
+GM: [*] 0 IntentionallySkipped:
+GM: [*] 0 RenderModeMismatch:
+GM: [ ] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
+GM: [*] 0 MissingExpectations:
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)

gm/tests/run.sh

@@ -165,4 +165,7 @@ gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $
 # Confirm that IntentionallySkipped tests are recorded as such.
 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
 
+# Ignore some error types (including ExpectationsMismatch)
+gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch"
+
 echo "All tests passed."