rebaseline_server: apply ignored-tests.txt within rebaseline_server, not just on the bots

(SkipBuildbotRuns) BUG=skia:2514 NOTREECHECKS=True NOTRY=True R=rmistry@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/287623006 git-svn-id: http://skia.googlecode.com/svn/trunk@14740 2bbb7eff-a529-9590-31e7-b0007b416f81
2014-05-14 18:04:46 +00:00 · 2014-05-14 18:04:46 +00:00 · 4cef1be821
commit 4cef1be821
parent 90b5a2a653
7 changed files with 142 additions and 15 deletions
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@ -48,6 +48,7 @@ EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
    results.KEY__EXPECTATIONS__REVIEWED,
 ]
 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'

 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')

@ -64,12 +65,15 @@ class ExpectationComparisons(results.BaseComparisons):

  def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
               expected_root=DEFAULT_EXPECTATIONS_DIR,
+               ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
               diff_base_url=None, builder_regex_list=None):
    """
    Args:
      actuals_root: root directory containing all actual-results.json files
      expected_root: root directory containing all expected-results.json files
+      ignore_failures_file: if a file with this name is found within
+          expected_root, ignore failures for any tests listed in the file
      generated_images_root: directory within which to create all pixel diffs;
          if this directory does not yet exist, it will be created
      diff_base_url: base URL within which the client should look for diff
@ -87,6 +91,11 @@ class ExpectationComparisons(results.BaseComparisons):
        download_actuals.create_filepath_url(generated_images_root))
    self._actuals_root = actuals_root
    self._expected_root = expected_root
+    self._ignore_failures_on_these_tests = []
+    if ignore_failures_file:
+      self._ignore_failures_on_these_tests = (
+          ExpectationComparisons._read_noncomment_lines(
+              os.path.join(expected_root, ignore_failures_file)))
    self._load_actual_and_expected()
    self._timestamp = int(time.time())
    logging.info('Results complete; took %d seconds.' %
@ -316,10 +325,11 @@ class ExpectationComparisons(results.BaseComparisons):
          # categories recorded within the gm_actuals AT ALL, and
          # instead evaluate the result_type ourselves based on what
          # we see in expectations vs actual checksum?
-          # See related http://skbug.com/2514 ('rebaseline_server: apply
-          # ignored-tests.txt within rebaseline_server, not just on the bots')
          if expected_image_relative_url == actual_image_relative_url:
            updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
+          elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
+                (test in self._ignore_failures_on_these_tests)):
+            updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
          else:
            updated_result_type = result_type
          extra_columns_dict = {
@ -361,6 +371,11 @@ def main():
      '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
      help='Directory containing all expected-result JSON files; defaults to '
      '\'%(default)s\' .')
+  parser.add_argument(
+      '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE,
+      help='If a file with this name is found within the EXPECTATIONS dir, '
+      'ignore failures for any tests listed in the file; defaults to '
+      '\'%(default)s\' .')
  parser.add_argument(
      '--outfile', required=True,
      help='File to write result summary into, in JSON format.')
@ -375,9 +390,11 @@ def main():
      help='Directory within which to download images and generate diffs; '
      'defaults to \'%(default)s\' .')
  args = parser.parse_args()
-  results_obj = ExpectationComparisons(actuals_root=args.actuals,
-                                       expected_root=args.expectations,
-                                       generated_images_root=args.workdir)
+  results_obj = ExpectationComparisons(
+      actuals_root=args.actuals,
+      expected_root=args.expectations,
+      ignore_failures_file=args.ignore_failures_file,
+      generated_images_root=args.workdir)
  gm_json.WriteToFile(
      results_obj.get_packaged_results_of_type(results_type=args.results),
      args.outfile)
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@ -245,6 +245,22 @@ class BaseComparisons(object):
        meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
    return meta_dict

+  @staticmethod
+  def _read_noncomment_lines(path):
+    """Return a list of all noncomment lines within a file.
+
+    (A "noncomment" line is one that does not start with a '#'.)
+
+    Args:
+      path: path to file
+    """
+    lines = []
+    with open(path, 'r') as fh:
+      for line in fh:
+        if not line.startswith('#'):
+          lines.append(line.strip())
+    return lines
+
  @staticmethod
  def _create_relative_url(hashtype_and_digest, test_name):
    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
--- a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json
+++ b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json
@ -1,6 +1,9 @@
 {
   "actual-results" : {
-      "failed" : null,
+      "failed" : {
+         "blanket-ignored_565.png" : [ "bitmap-64bitMD5", 22222222 ],
+         "blanket-ignored_8888.png" : [ "bitmap-64bitMD5", 22222222 ]
+      },
      "failure-ignored" : {
         "displacement_565.png" : [ "bitmap-64bitMD5", 4569468668668628191 ],
         "displacement_8888.png" : [ "bitmap-64bitMD5", 11401048196735046263 ],
@ -54,6 +57,18 @@
         ],
         "ignore-failure" : false
      },
+      "blanket-ignored_565.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 111111111 ]
+         ],
+         "ignore-failure" : false
+      },
+      "blanket-ignored_8888.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 111111111 ]
+         ],
+         "ignore-failure" : false
+      },
      "displacement_565.png" : {
         "allowed-digests" : [
            [ "bitmap-64bitMD5", 16249664097236120848 ]
--- a/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json
+++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json
@ -66,6 +66,24 @@
      ], 
      "reviewed-by-human": false
    }, 
+    "blanket-ignored_565.png": {
+      "allowed-digests": [
+        [
+          "bitmap-64bitMD5", 
+          111111111
+        ]
+      ], 
+      "ignore-failure": false
+    }, 
+    "blanket-ignored_8888.png": {
+      "allowed-digests": [
+        [
+          "bitmap-64bitMD5", 
+          111111111
+        ]
+      ], 
+      "ignore-failure": false
+    }, 
    "displacement_565.png": {
      "allowed-digests": [
        [
--- a/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt
+++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt
@ -0,0 +1,10 @@
+# Failures of any GM tests listed in this file will be ignored [1], as
+# if they had been marked "ignore-failure": true in the per-builder
+# expected-results.json files.
+#
+
+blanket-ignored
+
+# "texdata" tests should NOT be ignored, since the next lines is commented out:
+# texdata
+
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json
@ -15,7 +15,7 @@
        ], 
        [
          "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", 
-          3
+          4
        ]
      ]
    }, 
@ -26,7 +26,7 @@
      "valuesAndCounts": [
        [
          "TODO", 
-          13
+          14
        ]
      ]
    }, 
@ -41,7 +41,7 @@
        ], 
        [
          "no-comparison", 
-          7
+          8
        ], 
        [
          "succeeded", 
@ -70,6 +70,10 @@
          "bitmapsource", 
          1
        ], 
+        [
+          "blanket-ignored", 
+          1
+        ], 
        [
          "displacement", 
          2
@ -90,7 +94,7 @@
    }
  }, 
  "header": {
-    "dataHash": "-7043844904261310530", 
+    "dataHash": "-3232866296817265056", 
    "isEditable": false, 
    "isExported": true, 
    "schemaVersion": 3, 
@ -231,6 +235,17 @@
      "imageBUrl": "bitmap-64bitMD5/3x3bitmaprect/2054956815327187963.png", 
      "isDifferent": false
    }, 
+    {
+      "extraColumns": {
+        "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", 
+        "config": "TODO", 
+        "resultType": "no-comparison", 
+        "test": "blanket-ignored"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", 
+      "imageBUrl": null, 
+      "isDifferent": true
+    }, 
    {
      "differenceData": {
        "maxDiffPerChannel": [
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json
@ -11,7 +11,7 @@
        ], 
        [
          "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", 
-          15
+          17
        ]
      ]
    }, 
@ -22,11 +22,11 @@
      "valuesAndCounts": [
        [
          "565", 
-          9
+          10
        ], 
        [
          "8888", 
-          9
+          10
        ], 
        [
          "gpu", 
@ -53,7 +53,7 @@
        ], 
        [
          "failure-ignored", 
-          4
+          6
        ], 
        [
          "no-comparison", 
@ -86,6 +86,10 @@
          "bitmapsource", 
          2
        ], 
+        [
+          "blanket-ignored", 
+          2
+        ], 
        [
          "displacement", 
          5
@ -106,7 +110,7 @@
    }
  }, 
  "header": {
-    "dataHash": "2775016045957284034", 
+    "dataHash": "-7988419419043310401", 
    "isEditable": false, 
    "isExported": true, 
    "schemaVersion": 3, 
@ -370,6 +374,38 @@
      "imageBUrl": "bitmap-64bitMD5/aaclip/14456211900777561488.png", 
      "isDifferent": false
    }, 
+    {
+      "expectations": {
+        "bugs": null, 
+        "ignore-failure": false, 
+        "reviewed-by-human": null
+      }, 
+      "extraColumns": {
+        "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", 
+        "config": "565", 
+        "resultType": "failure-ignored", 
+        "test": "blanket-ignored"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png", 
+      "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", 
+      "isDifferent": true
+    }, 
+    {
+      "expectations": {
+        "bugs": null, 
+        "ignore-failure": false, 
+        "reviewed-by-human": null
+      }, 
+      "extraColumns": {
+        "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", 
+        "config": "8888", 
+        "resultType": "failure-ignored", 
+        "test": "blanket-ignored"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png", 
+      "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", 
+      "isDifferent": true
+    }, 
    {
      "expectations": {
        "bugs": null,