97f0b089da
BUG=skia:1758 NOTRY=True R=borenet@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/270413002 git-svn-id: http://skia.googlecode.com/svn/trunk@14658 2bbb7eff-a529-9590-31e7-b0007b416f81
307 lines
11 KiB
Python
Executable File
307 lines
11 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
"""
|
|
Copyright 2013 Google Inc.
|
|
|
|
Use of this source code is governed by a BSD-style license that can be
|
|
found in the LICENSE file.
|
|
|
|
Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
|
|
"""
|
|
|
|
# System-level imports
|
|
import fnmatch
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
# Imports from within Skia
|
|
#
|
|
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
|
# that directory. That script allows us to parse the actual-results.json file
|
|
# written out by the GM tool.
|
|
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
|
|
# so any dirs that are already in the PYTHONPATH will be preferred.
|
|
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
|
|
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
|
|
if GM_DIRECTORY not in sys.path:
|
|
sys.path.append(GM_DIRECTORY)
|
|
import gm_json
|
|
import imagepairset
|
|
|
|
# Keys used to link an image to a particular GM test.
|
|
# NOTE: Keep these in sync with static/constants.js
|
|
REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 3
|
|
KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
|
|
KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
|
|
KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
|
|
KEY__EXTRACOLUMN__BUILDER = 'builder'
|
|
KEY__EXTRACOLUMN__CONFIG = 'config'
|
|
KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType'
|
|
KEY__EXTRACOLUMN__TEST = 'test'
|
|
KEY__HEADER = 'header'
|
|
KEY__HEADER__DATAHASH = 'dataHash'
|
|
KEY__HEADER__IS_EDITABLE = 'isEditable'
|
|
KEY__HEADER__IS_EXPORTED = 'isExported'
|
|
KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
|
|
KEY__HEADER__RESULTS_ALL = 'all'
|
|
KEY__HEADER__RESULTS_FAILURES = 'failures'
|
|
KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
|
|
KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
|
|
KEY__HEADER__TIME_UPDATED = 'timeUpdated'
|
|
KEY__HEADER__TYPE = 'type'
|
|
KEY__NEW_IMAGE_URL = 'newImageUrl'
|
|
KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
|
|
KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
|
|
KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
|
|
KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
|
|
|
|
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
|
|
IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config)
|
|
|
|
DEFAULT_ACTUALS_DIR = '.gm-actuals'
|
|
DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
|
|
PARENT_DIRECTORY, '.generated-images')
|
|
|
|
# Define the default set of builders we will process expectations/actuals for.
|
|
# This allows us to ignore builders for which we don't maintain expectations
|
|
# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
|
|
# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
|
|
# produces error when trying to add baselines for ASAN/TSAN builders')
|
|
DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
|
|
DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
|
|
'.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']
|
|
|
|
|
|
class BaseComparisons(object):
|
|
"""Base class for generating summary of comparisons between two image sets.
|
|
"""
|
|
|
|
def get_results_of_type(self, results_type):
|
|
"""Return results of some/all tests (depending on 'results_type' parameter).
|
|
|
|
Args:
|
|
results_type: string describing which types of results to include; must
|
|
be one of the RESULTS_* constants
|
|
|
|
Results are returned in a dictionary as output by ImagePairSet.as_dict().
|
|
"""
|
|
return self._results[results_type]
|
|
|
|
def get_packaged_results_of_type(self, results_type, reload_seconds=None,
|
|
is_editable=False, is_exported=True):
|
|
"""Package the results of some/all tests as a complete response_dict.
|
|
|
|
Args:
|
|
results_type: string indicating which set of results to return;
|
|
must be one of the RESULTS_* constants
|
|
reload_seconds: if specified, note that new results may be available once
|
|
these results are reload_seconds old
|
|
is_editable: whether clients are allowed to submit new baselines
|
|
is_exported: whether these results are being made available to other
|
|
network hosts
|
|
"""
|
|
response_dict = self._results[results_type]
|
|
time_updated = self.get_timestamp()
|
|
response_dict[KEY__HEADER] = {
|
|
KEY__HEADER__SCHEMA_VERSION: (
|
|
REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
|
|
|
|
# Timestamps:
|
|
# 1. when this data was last updated
|
|
# 2. when the caller should check back for new data (if ever)
|
|
KEY__HEADER__TIME_UPDATED: time_updated,
|
|
KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
|
|
(time_updated+reload_seconds) if reload_seconds else None),
|
|
|
|
# The type we passed to get_results_of_type()
|
|
KEY__HEADER__TYPE: results_type,
|
|
|
|
# Hash of dataset, which the client must return with any edits--
|
|
# this ensures that the edits were made to a particular dataset.
|
|
KEY__HEADER__DATAHASH: str(hash(repr(
|
|
response_dict[imagepairset.KEY__IMAGEPAIRS]))),
|
|
|
|
# Whether the server will accept edits back.
|
|
KEY__HEADER__IS_EDITABLE: is_editable,
|
|
|
|
# Whether the service is accessible from other hosts.
|
|
KEY__HEADER__IS_EXPORTED: is_exported,
|
|
}
|
|
return response_dict
|
|
|
|
def get_timestamp(self):
|
|
"""Return the time at which this object was created, in seconds past epoch
|
|
(UTC).
|
|
"""
|
|
return self._timestamp
|
|
|
|
_match_builders_pattern_list = [
|
|
re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
|
|
_skip_builders_pattern_list = [
|
|
re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]
|
|
|
|
def set_match_builders_pattern_list(self, pattern_list):
|
|
"""Override the default set of builders we should process.
|
|
|
|
The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .
|
|
|
|
Note that skip_builders_pattern_list overrides this; regardless of whether a
|
|
builder is in the "match" list, if it's in the "skip" list, we will skip it.
|
|
|
|
Args:
|
|
pattern_list: list of regex patterns; process builders that match any
|
|
entry within this list
|
|
"""
|
|
if pattern_list == None:
|
|
pattern_list = []
|
|
self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]
|
|
|
|
def set_skip_builders_pattern_list(self, pattern_list):
|
|
"""Override the default set of builders we should skip while processing.
|
|
|
|
The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .
|
|
|
|
This overrides match_builders_pattern_list; regardless of whether a
|
|
builder is in the "match" list, if it's in the "skip" list, we will skip it.
|
|
|
|
Args:
|
|
pattern_list: list of regex patterns; skip builders that match any
|
|
entry within this list
|
|
"""
|
|
if pattern_list == None:
|
|
pattern_list = []
|
|
self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]
|
|
|
|
def _ignore_builder(self, builder):
|
|
"""Returns True if we should skip processing this builder.
|
|
|
|
Args:
|
|
builder: name of this builder, as a string
|
|
|
|
Returns:
|
|
True if we should ignore expectations and actuals for this builder.
|
|
"""
|
|
for pattern in self._skip_builders_pattern_list:
|
|
if pattern.match(builder):
|
|
return True
|
|
for pattern in self._match_builders_pattern_list:
|
|
if pattern.match(builder):
|
|
return False
|
|
return True
|
|
|
|
def _read_builder_dicts_from_root(self, root, pattern='*.json'):
|
|
"""Read all JSON dictionaries within a directory tree.
|
|
|
|
Skips any dictionaries belonging to a builder we have chosen to ignore.
|
|
|
|
Args:
|
|
root: path to root of directory tree
|
|
pattern: which files to read within root (fnmatch-style pattern)
|
|
|
|
Returns:
|
|
A meta-dictionary containing all the JSON dictionaries found within
|
|
the directory tree, keyed by builder name (the basename of the directory
|
|
where each JSON dictionary was found).
|
|
|
|
Raises:
|
|
IOError if root does not refer to an existing directory
|
|
"""
|
|
# I considered making this call _read_dicts_from_root(), but I decided
|
|
# it was better to prune out the ignored builders within the os.walk().
|
|
if not os.path.isdir(root):
|
|
raise IOError('no directory found at path %s' % root)
|
|
meta_dict = {}
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
for matching_filename in fnmatch.filter(filenames, pattern):
|
|
builder = os.path.basename(dirpath)
|
|
if self._ignore_builder(builder):
|
|
continue
|
|
full_path = os.path.join(dirpath, matching_filename)
|
|
meta_dict[builder] = gm_json.LoadFromFile(full_path)
|
|
return meta_dict
|
|
|
|
def _read_dicts_from_root(self, root, pattern='*.json'):
|
|
"""Read all JSON dictionaries within a directory tree.
|
|
|
|
Args:
|
|
root: path to root of directory tree
|
|
pattern: which files to read within root (fnmatch-style pattern)
|
|
|
|
Returns:
|
|
A meta-dictionary containing all the JSON dictionaries found within
|
|
the directory tree, keyed by the pathname (relative to root) of each JSON
|
|
dictionary.
|
|
|
|
Raises:
|
|
IOError if root does not refer to an existing directory
|
|
"""
|
|
if not os.path.isdir(root):
|
|
raise IOError('no directory found at path %s' % root)
|
|
meta_dict = {}
|
|
for abs_dirpath, dirnames, filenames in os.walk(root):
|
|
rel_dirpath = os.path.relpath(abs_dirpath, root)
|
|
for matching_filename in fnmatch.filter(filenames, pattern):
|
|
abs_path = os.path.join(abs_dirpath, matching_filename)
|
|
rel_path = os.path.join(rel_dirpath, matching_filename)
|
|
meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
|
|
return meta_dict
|
|
|
|
@staticmethod
|
|
def _create_relative_url(hashtype_and_digest, test_name):
|
|
"""Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
|
|
|
|
If we don't have a record of this image, returns None.
|
|
|
|
Args:
|
|
hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
|
|
don't have a record of this image
|
|
test_name: string; name of the GM test that created this image
|
|
"""
|
|
if not hashtype_and_digest:
|
|
return None
|
|
return gm_json.CreateGmRelativeUrl(
|
|
test_name=test_name,
|
|
hash_type=hashtype_and_digest[0],
|
|
hash_digest=hashtype_and_digest[1])
|
|
|
|
@staticmethod
|
|
def combine_subdicts(input_dict):
|
|
""" Flatten out a dictionary structure by one level.
|
|
|
|
Input:
|
|
{
|
|
KEY_A1 : {
|
|
KEY_B1 : VALUE_B1,
|
|
},
|
|
KEY_A2 : {
|
|
KEY_B2 : VALUE_B2,
|
|
}
|
|
}
|
|
|
|
Output:
|
|
{
|
|
KEY_B1 : VALUE_B1,
|
|
KEY_B2 : VALUE_B2,
|
|
}
|
|
|
|
If this would result in any repeated keys, it will raise an Exception.
|
|
"""
|
|
output_dict = {}
|
|
for key, subdict in input_dict.iteritems():
|
|
for subdict_key, subdict_value in subdict.iteritems():
|
|
if subdict_key in output_dict:
|
|
raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
|
|
output_dict[subdict_key] = subdict_value
|
|
return output_dict
|
|
|
|
@staticmethod
|
|
def get_multilevel(input_dict, *keys):
|
|
""" Returns input_dict[key1][key2][...], or None if any key is not found.
|
|
"""
|
|
for key in keys:
|
|
if input_dict == None:
|
|
return None
|
|
input_dict = input_dict.get(key, None)
|
|
return input_dict
|