rebaseline_server: ignore builders we don't have any expectations for

(SkipBuildbotRuns)
BUG=skia:2486
NOTREECHECKS=True
NOTRY=True
R=rmistry@google.com
TBR=rmistry

Author: epoger@google.com

Review URL: https://codereview.chromium.org/269543002

git-svn-id: http://skia.googlecode.com/svn/trunk@14488 2bbb7eff-a529-9590-31e7-b0007b416f81

gm/rebaseline_server/compare_to_expectations.py

@@ -226,7 +226,14 @@ class ExpectationComparisons(results.BaseComparisons):
             results.KEY__RESULT_TYPE__NOCOMPARISON,
         ])
 
-    builders = sorted(actual_builder_dicts.keys())
+    # Only consider builders we have both expected and actual results for.
+    # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
+    # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
+    # builder')
+    actual_builder_set = set(actual_builder_dicts.keys())
+    expected_builder_set = set(expected_builder_dicts.keys())
+    builders = sorted(actual_builder_set.intersection(expected_builder_set))
+
     num_builders = len(builders)
     builder_num = 0
     for builder in builders:

gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Builder-We-Have-No-Expectations-File-For/actual-results.json

@@ -0,0 +1,88 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : {
+         "displacement_565.png" : [ "bitmap-64bitMD5", 4569468668668628191 ],
+         "displacement_8888.png" : [ "bitmap-64bitMD5", 11401048196735046263 ],
+         "displacement_gpu.png" : [ "bitmap-64bitMD5", 5698561127291561694 ],
+         "displacement_pdf-mac.png" : [ "bitmap-64bitMD5", 12901125495691049846 ],
+         "displacement_pdf-poppler.png" : [ "bitmap-64bitMD5", 16285974094717334658 ]
+      },
+      "no-comparison" : {
+         "bigblurs_565.png" : [ "bitmap-64bitMD5", 14704206703218007573 ],
+         "bigblurs_8888.png" : [ "bitmap-64bitMD5", 17309852422285247848 ],
+         "bigblurs_gpu.png" : [ "bitmap-64bitMD5", 1822195599289208664 ],
+         "bigblurs_pdf-mac.png" : [ "bitmap-64bitMD5", 16171608477794909861 ],
+         "bigblurs_pdf-poppler.png" : [ "bitmap-64bitMD5", 6539050160610613353 ]
+      },
+      "succeeded" : {
+         "3x3bitmaprect_565.png" : [ "bitmap-64bitMD5", 16998423976396106083 ],
+         "3x3bitmaprect_8888.png" : [ "bitmap-64bitMD5", 2054956815327187963 ],
+         "3x3bitmaprect_gpu.png" : [ "bitmap-64bitMD5", 2054956815327187963 ],
+         "3x3bitmaprect_pdf-mac.png" : [ "bitmap-64bitMD5", 8518347971308375604 ],
+         "3x3bitmaprect_pdf-poppler.png" : [ "bitmap-64bitMD5", 16723580409414313678 ]
+      }
+   },
+   "expected-results" : {
+      "3x3bitmaprect_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 16998423976396106083 ]
+         ],
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 2054956815327187963 ]
+         ],
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_gpu.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 2054956815327187963 ]
+         ],
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_pdf-mac.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 8518347971308375604 ]
+         ],
+         "ignore-failure" : false
+      },
+      "3x3bitmaprect_pdf-poppler.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 16723580409414313678 ]
+         ],
+         "ignore-failure" : false
+      },
+      "displacement_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 16249664097236120848 ]
+         ],
+         "ignore-failure" : true
+      },
+      "displacement_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 12120271618448785903 ]
+         ],
+         "ignore-failure" : true
+      },
+      "displacement_gpu.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 14033538044845358121 ]
+         ],
+         "ignore-failure" : true
+      },
+      "displacement_pdf-mac.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 10149713146651176710 ]
+         ],
+         "ignore-failure" : true
+      },
+      "displacement_pdf-poppler.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 4741823272437589681 ]
+         ],
+         "ignore-failure" : true
+      }
+   }
+}

gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json

@@ -6,6 +6,7 @@
       "isSortable": true, 
       "valuesAndCounts": {
         "Test-Android-GalaxyNexus-SGX540-Arm7-Release": 7, 
+        "Test-Builder-We-Have-No-Expectations-File-For": 3, 
         "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug": 3
       }
     }, 
@@ -14,7 +15,7 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "TODO": 10
+        "TODO": 13
       }
     }, 
     "resultType": {
@@ -22,9 +23,9 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "failed": 2, 
+        "failed": 4, 
         "no-comparison": 7, 
-        "succeeded": 1
+        "succeeded": 2
       }
     }, 
     "test": {
@@ -32,11 +33,11 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "3x3bitmaprect": 2, 
+        "3x3bitmaprect": 3, 
         "aaclip": 1, 
-        "bigblurs": 2, 
+        "bigblurs": 3, 
         "bitmapsource": 1, 
-        "displacement": 1, 
+        "displacement": 2, 
         "filterbitmap_checkerboard_192_192": 1, 
         "filterbitmap_checkerboard_32_2": 1, 
         "texdata": 1
@@ -44,7 +45,7 @@
     }
   }, 
   "header": {
-    "dataHash": "7627783634950094306", 
+    "dataHash": "-7043844904261310530", 
     "isEditable": false, 
     "isExported": true, 
     "schemaVersion": 2, 
@@ -130,6 +131,61 @@
       "imageBUrl": null, 
       "isDifferent": true
     }, 
+    {
+      "differenceData": {
+        "maxDiffPerChannel": [
+          136, 
+          68, 
+          34
+        ], 
+        "numDifferingPixels": 6081, 
+        "percentDifferingPixels": 2.4324, 
+        "perceptualDifference": 1.917199999999994, 
+        "weightedDiffMeasure": 0.06644571780084584
+      }, 
+      "extraColumns": {
+        "builder": "Test-Builder-We-Have-No-Expectations-File-For", 
+        "config": "TODO", 
+        "resultType": "failed", 
+        "test": "displacement"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/displacement/11401048196735046263.png", 
+      "imageBUrl": "bitmap-64bitMD5/displacement/5698561127291561694.png", 
+      "isDifferent": true
+    }, 
+    {
+      "differenceData": {
+        "maxDiffPerChannel": [
+          255, 
+          221, 
+          221
+        ], 
+        "numDifferingPixels": 50097, 
+        "percentDifferingPixels": 30.5767822265625, 
+        "perceptualDifference": 3.3917, 
+        "weightedDiffMeasure": 1.4826590790651433
+      }, 
+      "extraColumns": {
+        "builder": "Test-Builder-We-Have-No-Expectations-File-For", 
+        "config": "TODO", 
+        "resultType": "failed", 
+        "test": "bigblurs"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/bigblurs/17309852422285247848.png", 
+      "imageBUrl": "bitmap-64bitMD5/bigblurs/1822195599289208664.png", 
+      "isDifferent": true
+    }, 
+    {
+      "extraColumns": {
+        "builder": "Test-Builder-We-Have-No-Expectations-File-For", 
+        "config": "TODO", 
+        "resultType": "succeeded", 
+        "test": "3x3bitmaprect"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png", 
+      "imageBUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png", 
+      "isDifferent": false
+    }, 
     {
       "differenceData": {
         "maxDiffPerChannel": [