From 34b73a2da18f9c1ee01ca716a2e5a4ea0fa3ace4 Mon Sep 17 00:00:00 2001
From: "epoger@google.com"
 <epoger@google.com@2bbb7eff-a529-9590-31e7-b0007b416f81>
Date: Wed, 3 Jul 2013 17:43:11 +0000
Subject: [PATCH] rebaseline.py self-tests: prepare to add tests of JSON-format
 rebaselining

For now, rebaseline.py still does not support JSON-format rebaselining, so
the one self-test reports failure.  But this change will allow us to test
the JSON-format rebaselining once rebaseline.py supports it.

R=scroggo@google.com

Review URL: https://codereview.chromium.org/18348015

git-svn-id: http://skia.googlecode.com/svn/trunk@9896 2bbb7eff-a529-9590-31e7-b0007b416f81
---
 .../output-expected/command_line              |  2 +-
 .../expected-results.json                     | 52 ++++++++++++++++++
 .../base-macmini/expected-results.json        | 26 +++++++++
 .../expected-results.json                     | 52 ++++++++++++++++++
 tools/tests/run.sh                            | 53 ++++++++++++++++---
 5 files changed, 177 insertions(+), 8 deletions(-)
 create mode 100644 tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
 create mode 100644 tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json
 create mode 100644 tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json

diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
index 9530f18c3e..76293588ef 100644
--- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line
@@ -1 +1 @@
-python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root tools/tests/rebaseline/input/json1
+python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
new file mode 100644
index 0000000000..d7608cd2a0
--- /dev/null
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json
@@ -0,0 +1,52 @@
+{
+   "expected-results" : {
+      "3x3bitmaprect_565.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_8888.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "aaclip_gpu.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 11899819492385205974 ]
+         ],
+         "ignore-failure" : false
+      },
+      "aarectmodes_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 14760033689012826769 ]
+         ],
+         "ignore-failure" : false
+      },
+      "imageblur_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 17796243856503591523 ]
+         ],
+         "ignore-failure" : false
+      },
+      "imageblur_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 7426416989687670152 ]
+         ],
+         "ignore-failure" : false
+      },
+      "shadertext3_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 10593797161686785561 ]
+         ],
+         "ignore-failure" : false
+      },
+      "xfermodeimagefilter_pdf.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "xfermodes_pdf.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 9151974350149210736 ]
+         ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json
new file mode 100644
index 0000000000..79af0cf5e7
--- /dev/null
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json
@@ -0,0 +1,26 @@
+{
+   "expected-results" : {
+      "aaclip_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 12345 ]
+         ],
+         "ignore-failure" : false
+      },
+      "aaclip_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 67890 ]
+         ],
+         "ignore-failure" : false
+      },
+      "aaclip_gpu.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 11899819492385205974 ]
+         ],
+         "ignore-failure" : false
+      },
+      "aaclip_pdf.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json
new file mode 100644
index 0000000000..b314ed68a4
--- /dev/null
+++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json
@@ -0,0 +1,52 @@
+{
+   "expected-results" : {
+      "3x3bitmaprect_565.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_8888.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "aaclip_gpu.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 11899819492385205974 ]
+         ],
+         "ignore-failure" : false
+      },
+      "aarectmodes_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 14760033689012826769 ]
+         ],
+         "ignore-failure" : false
+      },
+      "imageblur_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 3359963596899141322 ]
+         ],
+         "ignore-failure" : false
+      },
+      "imageblur_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 4217923806027861152 ]
+         ],
+         "ignore-failure" : false
+      },
+      "shadertext3_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 3713708307125704716 ]
+         ],
+         "ignore-failure" : false
+      },
+      "xfermodeimagefilter_pdf.png" : {
+         "allowed-digests" : null,
+         "ignore-failure" : false
+      },
+      "xfermodes_pdf.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 9151974350149210736 ]
+         ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/tools/tests/run.sh b/tools/tests/run.sh
index 3f76de1b58..1a3f40b2a6 100755
--- a/tools/tests/run.sh
+++ b/tools/tests/run.sh
@@ -39,7 +39,7 @@ function compare_directories {
     echo "compare_directories requires exactly 2 parameters, got $#"
     exit 1
   fi
-  diff --exclude=.* $1 $2
+  diff --recursive --exclude=.* $1 $2
   if [ $? != 0 ]; then
     echo "failed in: compare_directories $1 $2"
     exit 1
@@ -129,9 +129,11 @@ function benchgraph_test {
   compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
 }
 
+# Test rebaseline.py's soon-to-disappear image file rebaselining capability.
+#
 # Run rebaseline.py with arguments in $1, recording its dry-run output.
 # Then compare that dry-run output to the content of $2/output-expected.
-function rebaseline_test {
+function rebaseline_images_test {
   if [ $# != 2 ]; then
     echo "rebaseline_test requires exactly 2 parameters, got $#"
     exit 1
@@ -150,6 +152,39 @@ function rebaseline_test {
   compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
 }
 
+# Test rebaseline.py's new JSON-format expectations rebaselining capability.
+#
+# Copy expected-result.json files from $1 into a dir where they can be modified.
+# Run rebaseline.py with arguments in $2, recording its output.
+# Then compare the output (and modified expected-result.json files) to the
+# content of $2/output-expected.
+function rebaseline_test {
+  if [ $# != 3 ]; then
+    echo "rebaseline_test requires exactly 3 parameters, got $#"
+    exit 1
+  fi
+  COPY_EXPECTATIONS_FROM_DIR="$1"
+  ARGS="$2"
+  ACTUAL_OUTPUT_DIR="$3/output-actual"
+  EXPECTED_OUTPUT_DIR="$3/output-expected"
+
+  rm -rf $ACTUAL_OUTPUT_DIR
+  mkdir -p $ACTUAL_OUTPUT_DIR
+  EXPECTATIONS_TO_MODIFY_DIR="$ACTUAL_OUTPUT_DIR/gm-expectations"
+  SUBDIRS=$(ls $COPY_EXPECTATIONS_FROM_DIR)
+  for SUBDIR in $SUBDIRS; do
+    mkdir -p $EXPECTATIONS_TO_MODIFY_DIR/$SUBDIR
+    cp $COPY_EXPECTATIONS_FROM_DIR/$SUBDIR/expected-results.json \
+       $EXPECTATIONS_TO_MODIFY_DIR/$SUBDIR
+  done
+  COMMAND="python tools/rebaseline.py --expectations-root $EXPECTATIONS_TO_MODIFY_DIR $ARGS"
+  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
+  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
+  echo $? >$ACTUAL_OUTPUT_DIR/return_value
+
+  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
+}
+
 # Run jsondiff.py with arguments in $1, recording its output.
 # Then compare that output to the content of $2/output-expected.
 function jsondiff_test {
@@ -230,11 +265,15 @@ fi
 
 REBASELINE_INPUT=tools/tests/rebaseline/input
 REBASELINE_OUTPUT=tools/tests/rebaseline/output
-rebaseline_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
-rebaseline_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all"
-rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1"
-rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new"
-rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root $REBASELINE_INPUT/json1" "$REBASELINE_OUTPUT/using-json1-expectations"
+
+# These test the old image-file expectations.
+rebaseline_images_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
+rebaseline_images_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all"
+rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1"
+rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new"
+
+# These test the new JSON-format expectations.
+rebaseline_test "$REBASELINE_INPUT/json1" "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations"
 
 #
 # Test jsondiff.py ...