Add --writeJsonSummary option to gm

This writes out a JSON file containing expected and actual checksum values for every test.
Next step: allow the same checksum file to be compared against, instead of PNG files.
Review URL: https://codereview.appspot.com/6940053

git-svn-id: http://skia.googlecode.com/svn/trunk@6843 2bbb7eff-a529-9590-31e7-b0007b416f81

gm/gmmain.cpp

@@ -15,6 +15,7 @@
 
 #include "gm.h"
 #include "system_preferences.h"
+#include "SkBitmapChecksummer.h"
 #include "SkColorPriv.h"
 #include "SkData.h"
 #include "SkDeferredCanvas.h"
@@ -32,6 +33,8 @@
 #include "SkTileGridPicture.h"
 #include "SamplePipeControllers.h"
 
+#include "json/value.h"
+
 #if SK_SUPPORT_GPU
 #include "GrContextFactory.h"
 #include "GrRenderTarget.h"
@@ -74,6 +77,16 @@ const static ErrorBitfield ERROR_DIMENSION_MISMATCH      = 0x04;
 const static ErrorBitfield ERROR_READING_REFERENCE_IMAGE = 0x08;
 const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10;
 
+const static char kJsonKey_ActualResults[]   = "actual-results";
+const static char kJsonKey_ActualResults_Failed[]        = "failed";
+const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
+const static char kJsonKey_ActualResults_Succeeded[]     = "succeeded";
+const static char kJsonKey_ActualResults_AnyStatus_Checksum[]    = "checksum";
+
+const static char kJsonKey_ExpectedResults[] = "expected-results";
+const static char kJsonKey_ExpectedResults_Checksums[]     = "checksums";
+const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure";
+
 using namespace skiagm;
 
 /*
@@ -570,13 +583,20 @@ public:
     ErrorBitfield compare_to_reference_image_on_disk(
       const char readPath [], const SkString& name, SkBitmap &bitmap,
       const char diffPath [], const char renderModeDescriptor []) {
+        ErrorBitfield retval;
         SkString path = make_filename(readPath, "", name, "png");
         SkBitmap referenceBitmap;
-        if (SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap,
+        Json::Value expectedChecksumsArray;
+
+        bool decodedReferenceBitmap =
+            SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap,
                                        SkBitmap::kARGB_8888_Config,
                                        SkImageDecoder::kDecodePixels_Mode,
-                                       NULL)) {
-            return compare_to_reference_image_in_memory(name, bitmap,
+                                       NULL);
+        if (decodedReferenceBitmap) {
+            expectedChecksumsArray.append(Json::UInt64(
+                SkBitmapChecksummer::Compute64(referenceBitmap)));
+            retval = compare_to_reference_image_in_memory(name, bitmap,
                                                           referenceBitmap,
                                                           diffPath,
                                                           renderModeDescriptor);
@@ -586,8 +606,34 @@ public:
             }
             RecordError(ERROR_READING_REFERENCE_IMAGE, name,
                         renderModeDescriptor);
-            return ERROR_READING_REFERENCE_IMAGE;
+            retval = ERROR_READING_REFERENCE_IMAGE;
         }
+
+        // Add this result to the appropriate JSON collection of actual results,
+        // depending on status.
+        Json::Value actualResults;
+        actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] = Json::UInt64(
+            SkBitmapChecksummer::Compute64(bitmap));
+        if (decodedReferenceBitmap) {
+            if (ERROR_NONE == retval) {
+                fJsonActualResults_Succeeded[name.c_str()] = actualResults;
+            } else {
+                fJsonActualResults_Failed[name.c_str()] = actualResults;
+            }
+        } else {
+            fJsonActualResults_FailureIgnored[name.c_str()] = actualResults;
+        }
+
+        // Add this test to the JSON collection of expected results.
+        // For now, we assume that this collection starts out empty and we
+        // just fill it in as we go; once gm accepts a JSON file as input,
+        // we'll have to change that.
+        Json::Value expectedResults;
+        expectedResults[kJsonKey_ExpectedResults_Checksums] = expectedChecksumsArray;
+        expectedResults[kJsonKey_ExpectedResults_IgnoreFailure] = !decodedReferenceBitmap;
+        fJsonExpectedResults[name.c_str()] = expectedResults;
+
+        return retval;
     }
 
     // NOTE: As far as I can tell, this function is NEVER called with a
@@ -791,6 +837,11 @@ public:
     // information about all failed tests we have encountered so far
     SkTArray<FailRec> fFailedTests;
 
+    Json::Value fJsonExpectedResults;
+    Json::Value fJsonActualResults_Failed;
+    Json::Value fJsonActualResults_FailureIgnored;
+    Json::Value fJsonActualResults_Succeeded;
+
 }; // end of GMMain class definition
 
 #if SK_SUPPORT_GPU
@@ -874,6 +925,7 @@ static void usage(const char * argv0) {
 "    [--notexturecache]: disable the gpu texture cache\n"
 "    [--tiledPipe]: Exercise tiled SkGPipe replay\n"
 "    [--tileGrid]: use a tileGrid structure for SkPicture testing\n"
+"    [--writeJsonSummary <path>]: write a JSON-formatted result summary to this file\n"
 "    [--writePath|-w <path>]: write rendered images into this directory\n"
 "    [--writePicturePath|-wp <path>]: write .skp files into this directory\n"
              );
@@ -957,6 +1009,7 @@ int tool_main(int argc, char** argv) {
     setSystemPreferences();
     GMMain gmmain;
 
+    const char* writeJsonSummaryPath = NULL;// if non-null, where we write the JSON summary
     const char* writePath = NULL;   // if non-null, where we write the originals
     const char* writePicturePath = NULL;    // if non-null, where we write serialized pictures
     const char* readPath = NULL;    // if non-null, were we read from to compare
@@ -1080,6 +1133,11 @@ int tool_main(int argc, char** argv) {
             if (argv < stop && **argv) {
                 writePath = *argv;
             }
+        } else if (0 == strcmp(*argv, "--writeJsonSummary")) {
+            argv++;
+            if (argv < stop && **argv) {
+                writeJsonSummaryPath = *argv;
+            }
         } else if ((0 == strcmp(*argv, "--writePicturePath")) ||
                    (0 == strcmp(*argv, "-wp"))) {
             argv++;
@@ -1351,6 +1409,22 @@ int tool_main(int argc, char** argv) {
              testsRun, testsPassed, testsFailed, testsMissingReferenceImages);
     gmmain.ListErrors();
 
+    if (NULL != writeJsonSummaryPath) {
+        Json::Value actualResults;
+        actualResults[kJsonKey_ActualResults_Failed] =
+            gmmain.fJsonActualResults_Failed;
+        actualResults[kJsonKey_ActualResults_FailureIgnored] =
+            gmmain.fJsonActualResults_FailureIgnored;
+        actualResults[kJsonKey_ActualResults_Succeeded] =
+            gmmain.fJsonActualResults_Succeeded;
+        Json::Value root;
+        root[kJsonKey_ActualResults] = actualResults;
+        root[kJsonKey_ExpectedResults] = gmmain.fJsonExpectedResults;
+        std::string jsonStdString = root.toStyledString();
+        SkFILEWStream stream(writeJsonSummaryPath);
+        stream.write(jsonStdString.c_str(), jsonStdString.length());
+    }
+
 #if SK_SUPPORT_GPU
 
 #if GR_CACHE_STATS

gm/tests/inputs/empty-dir/README

@@ -0,0 +1 @@
+This directory intentionally left empty. Except for this file.

gm/tests/outputs/compared-against-different-pixels/output-expected/command_line

@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels -w gm/tests/outputs/compared-against-different-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-different-pixels/output-actual/images

gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt

@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      },
+      "failure-ignored" : null,
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 15161495552186645995 ],
+         "ignore-failure" : false
+      }
+   }
+}

gm/tests/outputs/compared-against-empty-dir/output-expected/command_line

@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-empty-dir/output-actual/images

gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt

@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      },
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : null,
+         "ignore-failure" : true
+      }
+   }
+}

gm/tests/outputs/compared-against-empty-dir/output-expected/return_value

@@ -0,0 +1 @@
+0

gm/tests/outputs/compared-against-empty-dir/output-expected/stdout

@@ -0,0 +1,5 @@
+reading from gm/tests/inputs/empty-dir
+writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images
+drawing... dashing2 [640 480]
+FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png
+Ran 1 tests: 0 passed, 0 failed, 1 missing reference images

gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line

@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images

gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt

@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "succeeded" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      }
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 2675870163990933333 ],
+         "ignore-failure" : false
+      }
+   }
+}

gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line

@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images

gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt

@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "succeeded" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      }
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 2675870163990933333 ],
+         "ignore-failure" : false
+      }
+   }
+}

gm/tests/run.sh

@@ -38,6 +38,7 @@ function compare_directories {
 # - with the arguments in $1
 # - writing resulting images into $2/output-actual/images
 # - writing stdout into $2/output-actual/stdout
+# - writing json summary into $2/output-actual/json-summary.txt
 # - writing return value into $2/output-actual/return_value
 # Then compare all of those against $2/output-expected .
 function gm_test {
@@ -51,7 +52,7 @@ function gm_test {
 
   rm -rf $ACTUAL_OUTPUT_DIR
   mkdir -p $ACTUAL_OUTPUT_DIR
-  COMMAND="$GM_BINARY $GM_ARGS -w $ACTUAL_OUTPUT_DIR/images"
+  COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images"
   echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
   $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
   echo $? >$ACTUAL_OUTPUT_DIR/return_value
@@ -72,4 +73,7 @@ gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/identical-pixe
 # Compare generated image against an input image file with different pixels.
 gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels"
 
+# Compare generated image against an empty "expected image" dir.
+gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+
 echo "All tests passed."

gyp/gm.gyp

@@ -10,6 +10,7 @@
       'include_dirs' : [
         '../src/core',
         '../src/pipe/utils/',
+        '../src/utils/',
       ],
       'includes': [
         'gmslides.gypi',
@@ -26,6 +27,8 @@
         'effects.gyp:effects',
         'images.gyp:images',
         'pdf.gyp:pdf',
+        'utils.gyp:utils',
+        '../third_party/externals/jsoncpp/jsoncpp.gyp:jsoncpp',
       ],
       'conditions': [
         ['skia_os == "mac"', {