179 lines
7.1 KiB
Python
179 lines
7.1 KiB
Python
|
#!/usr/bin/python
|
||
|
|
||
|
'''
|
||
|
Copyright 2013 Google Inc.
|
||
|
|
||
|
Use of this source code is governed by a BSD-style license that can be
|
||
|
found in the LICENSE file.
|
||
|
'''
|
||
|
|
||
|
'''
|
||
|
Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
|
||
|
'''
|
||
|
|
||
|
# System-level imports
|
||
|
import fnmatch
|
||
|
import json
|
||
|
import os
|
||
|
import re
|
||
|
import sys
|
||
|
|
||
|
# Imports from within Skia
|
||
|
#
|
||
|
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
||
|
# that directory. That script allows us to parse the actual-results.json file
|
||
|
# written out by the GM tool.
|
||
|
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
|
||
|
# so any dirs that are already in the PYTHONPATH will be preferred.
|
||
|
GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||
|
if GM_DIRECTORY not in sys.path:
|
||
|
sys.path.append(GM_DIRECTORY)
|
||
|
import gm_json
|
||
|
|
||
|
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
|
||
|
|
||
|
class Results(object):
|
||
|
""" Loads actual and expected results from all builders, supplying combined
|
||
|
reports as requested. """
|
||
|
|
||
|
def __init__(self, actuals_root, expected_root):
|
||
|
"""
|
||
|
params:
|
||
|
actuals_root: root directory containing all actual-results.json files
|
||
|
expected_root: root directory containing all expected-results.json files
|
||
|
"""
|
||
|
self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
|
||
|
self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
|
||
|
self._all_results = self._Combine()
|
||
|
|
||
|
def GetAll(self):
|
||
|
"""Return results of all tests, as a list in this form:
|
||
|
|
||
|
[
|
||
|
{
|
||
|
"builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
|
||
|
"test": "bigmatrix",
|
||
|
"config": "8888",
|
||
|
"resultType": "failed",
|
||
|
"expectedHashType": "bitmap-64bitMD5",
|
||
|
"expectedHashDigest": "10894408024079689926",
|
||
|
"actualHashType": "bitmap-64bitMD5",
|
||
|
"actualHashDigest": "2409857384569",
|
||
|
},
|
||
|
...
|
||
|
]
|
||
|
"""
|
||
|
return self._all_results
|
||
|
|
||
|
@staticmethod
|
||
|
def _GetDictsFromRoot(root, pattern='*.json'):
|
||
|
"""Read all JSON dictionaries within a directory tree, returning them within
|
||
|
a meta-dictionary (keyed by the builder name for each dictionary).
|
||
|
|
||
|
params:
|
||
|
root: path to root of directory tree
|
||
|
pattern: which files to read within root (fnmatch-style pattern)
|
||
|
"""
|
||
|
meta_dict = {}
|
||
|
for dirpath, dirnames, filenames in os.walk(root):
|
||
|
for matching_filename in fnmatch.filter(filenames, pattern):
|
||
|
builder = os.path.basename(dirpath)
|
||
|
if builder.endswith('-Trybot'):
|
||
|
continue
|
||
|
fullpath = os.path.join(dirpath, matching_filename)
|
||
|
meta_dict[builder] = gm_json.LoadFromFile(fullpath)
|
||
|
return meta_dict
|
||
|
|
||
|
def _Combine(self):
|
||
|
"""Returns a list of all tests, across all builders, based on the
|
||
|
contents of self._actual_builder_dicts and self._expected_builder_dicts .
|
||
|
Returns the list in the same form needed for GetAllResults().
|
||
|
"""
|
||
|
all_tests = []
|
||
|
for builder in sorted(self._actual_builder_dicts.keys()):
|
||
|
actual_results_for_this_builder = (
|
||
|
self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
|
||
|
for result_type in sorted(actual_results_for_this_builder.keys()):
|
||
|
results_of_this_type = actual_results_for_this_builder[result_type]
|
||
|
if not results_of_this_type:
|
||
|
continue
|
||
|
for image_name in sorted(results_of_this_type.keys()):
|
||
|
actual_image = results_of_this_type[image_name]
|
||
|
try:
|
||
|
# TODO(epoger): assumes a single allowed digest per test
|
||
|
expected_image = (
|
||
|
self._expected_builder_dicts
|
||
|
[builder][gm_json.JSONKEY_EXPECTEDRESULTS]
|
||
|
[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
|
||
|
[0])
|
||
|
except (KeyError, TypeError):
|
||
|
# There are several cases in which we would expect to find
|
||
|
# no expectations for a given test:
|
||
|
#
|
||
|
# 1. result_type == NOCOMPARISON
|
||
|
# There are no expectations for this test yet!
|
||
|
#
|
||
|
# 2. ignore-tests.txt
|
||
|
# If a test has been listed in ignore-tests.txt, then its status
|
||
|
# may show as FAILUREIGNORED even if it doesn't have any
|
||
|
# expectations yet.
|
||
|
#
|
||
|
# 3. alternate rendering mode failures (e.g. serialized)
|
||
|
# In cases like
|
||
|
# https://code.google.com/p/skia/issues/detail?id=1684
|
||
|
# ('tileimagefilter GM test failing in serialized render mode'),
|
||
|
# the gm-actuals will list a failure for the alternate
|
||
|
# rendering mode even though we don't have explicit expectations
|
||
|
# for the test (the implicit expectation is that it must
|
||
|
# render the same in all rendering modes).
|
||
|
#
|
||
|
# Don't log types 1 or 2, because they are common.
|
||
|
# Log other types, because they are rare and we should know about
|
||
|
# them, but don't throw an exception, because we need to keep our
|
||
|
# tools working in the meanwhile!
|
||
|
if result_type not in [
|
||
|
gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
|
||
|
gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
|
||
|
print 'WARNING: No expectations found for test: %s' % {
|
||
|
'builder': builder,
|
||
|
'image_name': image_name,
|
||
|
'result_type': result_type,
|
||
|
}
|
||
|
expected_image = [None, None]
|
||
|
|
||
|
# If this test was recently rebaselined, it will remain in
|
||
|
# the "failed" set of actuals until all the bots have
|
||
|
# cycled (although the expectations have indeed been set
|
||
|
# from the most recent actuals). Treat these as successes
|
||
|
# instead of failures.
|
||
|
#
|
||
|
# TODO(epoger): Do we need to do something similar in
|
||
|
# other cases, such as when we have recently marked a test
|
||
|
# as ignoreFailure but it still shows up in the "failed"
|
||
|
# category? Maybe we should not rely on the result_type
|
||
|
# categories recorded within the gm_actuals AT ALL, and
|
||
|
# instead evaluate the result_type ourselves based on what
|
||
|
# we see in expectations vs actual checksum?
|
||
|
if expected_image == actual_image:
|
||
|
updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
|
||
|
else:
|
||
|
updated_result_type = result_type
|
||
|
|
||
|
# TODO(epoger): For now, don't include succeeded results.
|
||
|
# There are so many of them that they make the client too slow.
|
||
|
if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
|
||
|
continue
|
||
|
|
||
|
(test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
|
||
|
all_tests.append({
|
||
|
"builder": builder,
|
||
|
"test": test,
|
||
|
"config": config,
|
||
|
"resultType": updated_result_type,
|
||
|
"actualHashType": actual_image[0],
|
||
|
"actualHashDigest": str(actual_image[1]),
|
||
|
"expectedHashType": expected_image[0],
|
||
|
"expectedHashDigest": str(expected_image[1]),
|
||
|
})
|
||
|
return all_tests
|