31d0b3d806
This allows us to compare GMs between configs across all builders, so we can see the largest deviations between raster and GPU renderings. BUG=skia:1919 NOTREECHECKS=True NOTRY=True R=rmistry@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/215503002 git-svn-id: http://skia.googlecode.com/svn/trunk@13991 2bbb7eff-a529-9590-31e7-b0007b416f81
220 lines
8.8 KiB
Python
Executable File
220 lines
8.8 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
"""
|
|
Copyright 2014 Google Inc.
|
|
|
|
Use of this source code is governed by a BSD-style license that can be
|
|
found in the LICENSE file.
|
|
|
|
Compare GM results for two configs, across all builders.
|
|
"""
|
|
|
|
# System-level imports
|
|
import argparse
|
|
import fnmatch
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
# Imports from within Skia
|
|
#
|
|
# TODO(epoger): Once we move the create_filepath_url() function out of
|
|
# download_actuals into a shared utility module, we won't need to import
|
|
# download_actuals anymore.
|
|
#
|
|
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
|
# that directory. That script allows us to parse the actual-results.json file
|
|
# written out by the GM tool.
|
|
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
|
|
# so any dirs that are already in the PYTHONPATH will be preferred.
|
|
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
|
|
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
|
|
TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
|
|
if GM_DIRECTORY not in sys.path:
|
|
sys.path.append(GM_DIRECTORY)
|
|
import download_actuals
|
|
import gm_json
|
|
import imagediffdb
|
|
import imagepair
|
|
import imagepairset
|
|
import results
|
|
|
|
|
|
class ConfigComparisons(results.BaseComparisons):
|
|
"""Loads results from two different configurations into an ImagePairSet.
|
|
|
|
Loads actual and expected results from all builders, except for those skipped
|
|
by BaseComparisons._ignore_builder().
|
|
"""
|
|
|
|
def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
|
|
generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
|
|
diff_base_url=None):
|
|
"""
|
|
Args:
|
|
configs: (string, string) tuple; pair of configs to compare
|
|
actuals_root: root directory containing all actual-results.json files
|
|
generated_images_root: directory within which to create all pixel diffs;
|
|
if this directory does not yet exist, it will be created
|
|
diff_base_url: base URL within which the client should look for diff
|
|
images; if not specified, defaults to a "file:///" URL representation
|
|
of generated_images_root
|
|
"""
|
|
time_start = int(time.time())
|
|
self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
|
|
self._diff_base_url = (
|
|
diff_base_url or
|
|
download_actuals.create_filepath_url(generated_images_root))
|
|
self._actuals_root = actuals_root
|
|
self._load_config_pairs(configs)
|
|
self._timestamp = int(time.time())
|
|
logging.info('Results complete; took %d seconds.' %
|
|
(self._timestamp - time_start))
|
|
|
|
def _load_config_pairs(self, configs):
|
|
"""Loads the results of all tests, across all builders (based on the
|
|
files within self._actuals_root), compares them across two configs,
|
|
and stores the summary in self._results.
|
|
|
|
Args:
|
|
configs: tuple of strings; pair of configs to compare
|
|
"""
|
|
logging.info('Reading actual-results JSON files from %s...' %
|
|
self._actuals_root)
|
|
actual_builder_dicts = ConfigComparisons._read_dicts_from_root(
|
|
self._actuals_root)
|
|
configA, configB = configs
|
|
logging.info('Comparing configs %s and %s...' % (configA, configB))
|
|
|
|
all_image_pairs = imagepairset.ImagePairSet(
|
|
descriptions=configs,
|
|
diff_base_url=self._diff_base_url)
|
|
failing_image_pairs = imagepairset.ImagePairSet(
|
|
descriptions=configs,
|
|
diff_base_url=self._diff_base_url)
|
|
|
|
all_image_pairs.ensure_extra_column_values_in_summary(
|
|
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
|
results.KEY__RESULT_TYPE__FAILED,
|
|
results.KEY__RESULT_TYPE__NOCOMPARISON,
|
|
results.KEY__RESULT_TYPE__SUCCEEDED,
|
|
])
|
|
failing_image_pairs.ensure_extra_column_values_in_summary(
|
|
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
|
results.KEY__RESULT_TYPE__FAILED,
|
|
results.KEY__RESULT_TYPE__NOCOMPARISON,
|
|
])
|
|
|
|
builders = sorted(actual_builder_dicts.keys())
|
|
num_builders = len(builders)
|
|
builder_num = 0
|
|
for builder in builders:
|
|
builder_num += 1
|
|
logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
|
|
(builder_num, num_builders, builder))
|
|
actual_results_for_this_builder = (
|
|
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
|
|
for result_type in sorted(actual_results_for_this_builder.keys()):
|
|
results_of_this_type = actual_results_for_this_builder[result_type]
|
|
if not results_of_this_type:
|
|
continue
|
|
|
|
tests_found = set()
|
|
for image_name in sorted(results_of_this_type.keys()):
|
|
(test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
|
|
tests_found.add(test)
|
|
|
|
for test in tests_found:
|
|
# Get image_relative_url (or None) for each of configA, configB
|
|
image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
|
|
configA_image_relative_url = ConfigComparisons._create_relative_url(
|
|
hashtype_and_digest=results_of_this_type.get(image_name_A),
|
|
test_name=test)
|
|
image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
|
|
configB_image_relative_url = ConfigComparisons._create_relative_url(
|
|
hashtype_and_digest=results_of_this_type.get(image_name_B),
|
|
test_name=test)
|
|
|
|
# If we have images for at least one of these two configs,
|
|
# add them to our list.
|
|
if configA_image_relative_url or configB_image_relative_url:
|
|
if configA_image_relative_url == configB_image_relative_url:
|
|
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
|
|
elif not configA_image_relative_url:
|
|
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
|
|
elif not configB_image_relative_url:
|
|
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
|
|
else:
|
|
result_type = results.KEY__RESULT_TYPE__FAILED
|
|
|
|
extra_columns_dict = {
|
|
results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
|
|
results.KEY__EXTRACOLUMN__BUILDER: builder,
|
|
results.KEY__EXTRACOLUMN__TEST: test,
|
|
# TODO(epoger): Right now, the client UI crashes if it receives
|
|
# results that do not include a 'config' column.
|
|
# Until we fix that, keep the client happy.
|
|
results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
|
|
}
|
|
|
|
try:
|
|
image_pair = imagepair.ImagePair(
|
|
image_diff_db=self._image_diff_db,
|
|
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
|
|
imageA_relative_url=configA_image_relative_url,
|
|
imageB_relative_url=configB_image_relative_url,
|
|
extra_columns=extra_columns_dict)
|
|
all_image_pairs.add_image_pair(image_pair)
|
|
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
|
|
failing_image_pairs.add_image_pair(image_pair)
|
|
except (KeyError, TypeError):
|
|
logging.exception(
|
|
'got exception while creating ImagePair for image_name '
|
|
'"%s", builder "%s"' % (image_name, builder))
|
|
|
|
self._results = {
|
|
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
|
|
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
|
|
}
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
|
|
datefmt='%m/%d/%Y %H:%M:%S',
|
|
level=logging.INFO)
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--actuals', default=results.DEFAULT_ACTUALS_DIR,
|
|
help='Directory containing all actual-result JSON files; defaults to '
|
|
'\'%(default)s\' .')
|
|
parser.add_argument(
|
|
'config', nargs=2,
|
|
help='Two configurations to compare (8888, gpu, etc.).')
|
|
parser.add_argument(
|
|
'--outfile', required=True,
|
|
help='File to write result summary into, in JSON format.')
|
|
parser.add_argument(
|
|
'--results', default=results.KEY__HEADER__RESULTS_FAILURES,
|
|
help='Which result types to include. Defaults to \'%(default)s\'; '
|
|
'must be one of ' +
|
|
str([results.KEY__HEADER__RESULTS_FAILURES,
|
|
results.KEY__HEADER__RESULTS_ALL]))
|
|
parser.add_argument(
|
|
'--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
|
|
help='Directory within which to download images and generate diffs; '
|
|
'defaults to \'%(default)s\' .')
|
|
args = parser.parse_args()
|
|
results_obj = ConfigComparisons(configs=args.config,
|
|
actuals_root=args.actuals,
|
|
generated_images_root=args.workdir)
|
|
gm_json.WriteToFile(
|
|
results_obj.get_packaged_results_of_type(results_type=args.results),
|
|
args.outfile)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|