5187c4313c
tools/skimage_main.cpp: Add the ability to write the results to checksum based filenames, much like GM uses. This will allow using the skpdiff server to rebaseline images. Write the keys in the JSON file as <original image>_<pref config>.png, so it matches gm_json.IMAGE_FILENAME_PATTERN. Also replace '_' with '-' in the original file name, to avoid confusing the pattern matcher. The '_' to '-' replacement also happens on the output filename. Read the keys in a similar manner. In make_outname, no longer remove a suffix. This fixes a bug where subset decoding writes multiple subsets to the same file. tools/rebaseline.py: Since the filenames written to json files now match gm_json.IMAGE_FILENAME_PATTERN, enable the option to match based on configs/tests when rebaselining skimage. test json files: Update to match the new format of output. gm/gm_expectations: Add a constructor that takes a BitmapAndDigest as input. tools/tests/skimage_self_test.py: Test that reading the expectations file just created by skimage with the same file actually compares to the original file (rather than just succeeding because expectations were missing). Change the expectations files to match the new format. Will require a buildbot change to use the new flag: https://codereview.chromium.org/27389002/ BUG=1466 R=epoger@google.com Review URL: https://codereview.chromium.org/26297004 git-svn-id: http://skia.googlecode.com/svn/trunk@11902 2bbb7eff-a529-9590-31e7-b0007b416f81
491 lines
23 KiB
Python
Executable File
491 lines
23 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
'''
|
|
Copyright 2012 Google Inc.
|
|
|
|
Use of this source code is governed by a BSD-style license that can be
|
|
found in the LICENSE file.
|
|
'''
|
|
|
|
'''
|
|
Rebaselines the given GM tests, on all bots and all configurations.
|
|
'''
|
|
|
|
# System-level imports
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import urllib2
|
|
|
|
# Imports from within Skia
|
|
#
|
|
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
|
# that directory. That script allows us to parse the actual-results.json file
|
|
# written out by the GM tool.
|
|
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
|
|
# so any dirs that are already in the PYTHONPATH will be preferred.
|
|
#
|
|
# This assumes that the 'gm' directory has been checked out as a sibling of
|
|
# the 'tools' directory containing this script, which will be the case if
|
|
# 'trunk' was checked out as a single unit.
|
|
GM_DIRECTORY = os.path.realpath(
|
|
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
|
|
if GM_DIRECTORY not in sys.path:
|
|
sys.path.append(GM_DIRECTORY)
|
|
import gm_json
|
|
|
|
# TODO(epoger): In the long run, we want to build this list automatically,
|
|
# but for now we hard-code it until we can properly address
|
|
# https://code.google.com/p/skia/issues/detail?id=1544
|
|
# ('live query of builder list makes rebaseline.py slow to start up')
|
|
TEST_BUILDERS = [
|
|
'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',
|
|
'Test-Android-GalaxyNexus-SGX540-Arm7-Release',
|
|
'Test-Android-IntelRhb-SGX544-x86-Debug',
|
|
'Test-Android-IntelRhb-SGX544-x86-Release',
|
|
'Test-Android-Nexus10-MaliT604-Arm7-Debug',
|
|
'Test-Android-Nexus10-MaliT604-Arm7-Release',
|
|
'Test-Android-Nexus4-Adreno320-Arm7-Debug',
|
|
'Test-Android-Nexus4-Adreno320-Arm7-Release',
|
|
'Test-Android-Nexus7-Tegra3-Arm7-Debug',
|
|
'Test-Android-Nexus7-Tegra3-Arm7-Release',
|
|
'Test-Android-NexusS-SGX540-Arm7-Debug',
|
|
'Test-Android-NexusS-SGX540-Arm7-Release',
|
|
'Test-Android-Xoom-Tegra2-Arm7-Debug',
|
|
'Test-Android-Xoom-Tegra2-Arm7-Release',
|
|
'Test-ChromeOS-Alex-GMA3150-x86-Debug',
|
|
'Test-ChromeOS-Alex-GMA3150-x86-Release',
|
|
'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug',
|
|
'Test-ChromeOS-Daisy-MaliT604-Arm7-Release',
|
|
'Test-ChromeOS-Link-HD4000-x86_64-Debug',
|
|
'Test-ChromeOS-Link-HD4000-x86_64-Release',
|
|
'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
|
|
'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',
|
|
'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug',
|
|
'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release',
|
|
'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug',
|
|
'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',
|
|
'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug',
|
|
'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release',
|
|
'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug',
|
|
'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release',
|
|
'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',
|
|
'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release',
|
|
'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug',
|
|
'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release',
|
|
'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug',
|
|
'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release',
|
|
'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind',
|
|
'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug',
|
|
'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Debug',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Release',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
|
|
'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite',
|
|
'Test-Win7-ShuttleA-HD2000-x86_64-Debug',
|
|
'Test-Win7-ShuttleA-HD2000-x86_64-Release',
|
|
]
|
|
|
|
# TODO: Get this from builder_name_schema in buildbot.
|
|
TRYBOT_SUFFIX = '-Trybot'
|
|
|
|
|
|
class _InternalException(Exception):
|
|
pass
|
|
|
|
class ExceptionHandler(object):
|
|
""" Object that handles exceptions, either raising them immediately or
|
|
collecting them to display later on."""
|
|
|
|
# params:
|
|
def __init__(self, keep_going_on_failure=False):
|
|
"""
|
|
params:
|
|
keep_going_on_failure: if False, report failures and quit right away;
|
|
if True, collect failures until
|
|
ReportAllFailures() is called
|
|
"""
|
|
self._keep_going_on_failure = keep_going_on_failure
|
|
self._failures_encountered = []
|
|
|
|
def RaiseExceptionOrContinue(self):
|
|
""" We have encountered an exception; either collect the info and keep
|
|
going, or exit the program right away."""
|
|
# Get traceback information about the most recently raised exception.
|
|
exc_info = sys.exc_info()
|
|
|
|
if self._keep_going_on_failure:
|
|
print >> sys.stderr, ('WARNING: swallowing exception %s' %
|
|
repr(exc_info[1]))
|
|
self._failures_encountered.append(exc_info)
|
|
else:
|
|
print >> sys.stderr, (
|
|
'\nHalting at first exception.\n' +
|
|
'Please file a bug to epoger@google.com at ' +
|
|
'https://code.google.com/p/skia/issues/entry, containing the ' +
|
|
'command you ran and the following stack trace.\n\n' +
|
|
'Afterwards, you can re-run with the --keep-going-on-failure ' +
|
|
'option set.\n')
|
|
raise exc_info[1], None, exc_info[2]
|
|
|
|
def ReportAllFailures(self):
|
|
if self._failures_encountered:
|
|
print >> sys.stderr, ('Encountered %d failures (see above).' %
|
|
len(self._failures_encountered))
|
|
sys.exit(1)
|
|
|
|
|
|
# Object that rebaselines a JSON expectations file (not individual image files).
|
|
class JsonRebaseliner(object):
|
|
|
|
# params:
|
|
# expectations_root: root directory of all expectations JSON files
|
|
# expectations_input_filename: filename (under expectations_root) of JSON
|
|
# expectations file to read; typically
|
|
# "expected-results.json"
|
|
# expectations_output_filename: filename (under expectations_root) to
|
|
# which updated expectations should be
|
|
# written; typically the same as
|
|
# expectations_input_filename, to overwrite
|
|
# the old content
|
|
# actuals_base_url: base URL from which to read actual-result JSON files
|
|
# actuals_filename: filename (under actuals_base_url) from which to read a
|
|
# summary of results; typically "actual-results.json"
|
|
# exception_handler: reference to rebaseline.ExceptionHandler object
|
|
# tests: list of tests to rebaseline, or None if we should rebaseline
|
|
# whatever files the JSON results summary file tells us to
|
|
# configs: which configs to run for each test, or None if we should
|
|
# rebaseline whatever configs the JSON results summary file tells
|
|
# us to
|
|
# add_new: if True, add expectations for tests which don't have any yet
|
|
# add_ignored: if True, add expectations for tests for which failures are
|
|
# currently ignored
|
|
# bugs: optional list of bug numbers which pertain to these expectations
|
|
# notes: free-form text notes to add to all updated expectations
|
|
# mark_unreviewed: if True, mark these expectations as NOT having been
|
|
# reviewed by a human; otherwise, leave that field blank.
|
|
# Currently, there is no way to make this script mark
|
|
# expectations as reviewed-by-human=True.
|
|
# TODO(epoger): Add that capability to a review tool.
|
|
# mark_ignore_failure: if True, mark failures of a given test as being
|
|
# ignored.
|
|
# from_trybot: if True, read actual-result JSON files generated from a
|
|
# trybot run rather than a waterfall run.
|
|
def __init__(self, expectations_root, expectations_input_filename,
|
|
expectations_output_filename, actuals_base_url,
|
|
actuals_filename, exception_handler,
|
|
tests=None, configs=None, add_new=False, add_ignored=False,
|
|
bugs=None, notes=None, mark_unreviewed=None,
|
|
mark_ignore_failure=False, from_trybot=False):
|
|
self._expectations_root = expectations_root
|
|
self._expectations_input_filename = expectations_input_filename
|
|
self._expectations_output_filename = expectations_output_filename
|
|
self._tests = tests
|
|
self._configs = configs
|
|
self._actuals_base_url = actuals_base_url
|
|
self._actuals_filename = actuals_filename
|
|
self._exception_handler = exception_handler
|
|
self._add_new = add_new
|
|
self._add_ignored = add_ignored
|
|
self._bugs = bugs
|
|
self._notes = notes
|
|
self._mark_unreviewed = mark_unreviewed
|
|
self._mark_ignore_failure = mark_ignore_failure;
|
|
if self._tests or self._configs:
|
|
self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
|
|
else:
|
|
self._image_filename_re = None
|
|
self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn'))
|
|
self._from_trybot = from_trybot
|
|
|
|
# Executes subprocess.call(cmd).
|
|
# Raises an Exception if the command fails.
|
|
def _Call(self, cmd):
|
|
if subprocess.call(cmd) != 0:
|
|
raise _InternalException('error running command: ' + ' '.join(cmd))
|
|
|
|
# Returns the full contents of filepath, as a single string.
|
|
# If filepath looks like a URL, try to read it that way instead of as
|
|
# a path on local storage.
|
|
#
|
|
# Raises _InternalException if there is a problem.
|
|
def _GetFileContents(self, filepath):
|
|
if filepath.startswith('http:') or filepath.startswith('https:'):
|
|
try:
|
|
return urllib2.urlopen(filepath).read()
|
|
except urllib2.HTTPError as e:
|
|
raise _InternalException('unable to read URL %s: %s' % (
|
|
filepath, e))
|
|
else:
|
|
return open(filepath, 'r').read()
|
|
|
|
# Returns a dictionary of actual results from actual-results.json file.
|
|
#
|
|
# The dictionary returned has this format:
|
|
# {
|
|
# u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322],
|
|
# u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152],
|
|
# u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]
|
|
# }
|
|
#
|
|
# If the JSON actual result summary file cannot be loaded, logs a warning
|
|
# message and returns None.
|
|
# If the JSON actual result summary file can be loaded, but we have
|
|
# trouble parsing it, raises an Exception.
|
|
#
|
|
# params:
|
|
# json_url: URL pointing to a JSON actual result summary file
|
|
# sections: a list of section names to include in the results, e.g.
|
|
# [gm_json.JSONKEY_ACTUALRESULTS_FAILED,
|
|
# gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;
|
|
# if None, then include ALL sections.
|
|
def _GetActualResults(self, json_url, sections=None):
|
|
try:
|
|
json_contents = self._GetFileContents(json_url)
|
|
except _InternalException:
|
|
print >> sys.stderr, (
|
|
'could not read json_url %s ; skipping this platform.' %
|
|
json_url)
|
|
return None
|
|
json_dict = gm_json.LoadFromString(json_contents)
|
|
results_to_return = {}
|
|
actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
|
|
if not sections:
|
|
sections = actual_results.keys()
|
|
for section in sections:
|
|
section_results = actual_results[section]
|
|
if section_results:
|
|
results_to_return.update(section_results)
|
|
return results_to_return
|
|
|
|
# Rebaseline all tests/types we specified in the constructor,
|
|
# within this builder's subdirectory in expectations/gm .
|
|
#
|
|
# params:
|
|
# builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
|
|
def RebaselineSubdir(self, builder):
|
|
# Read in the actual result summary, and extract all the tests whose
|
|
# results we need to update.
|
|
results_builder = str(builder)
|
|
if self._from_trybot:
|
|
results_builder = results_builder + TRYBOT_SUFFIX
|
|
actuals_url = '/'.join([self._actuals_base_url, results_builder,
|
|
self._actuals_filename])
|
|
# Only update results for tests that are currently failing.
|
|
# We don't want to rewrite results for tests that are already succeeding,
|
|
# because we don't want to add annotation fields (such as
|
|
# JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we
|
|
# are actually modifying.
|
|
sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
|
|
if self._add_new:
|
|
sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
|
|
if self._add_ignored:
|
|
sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)
|
|
results_to_update = self._GetActualResults(json_url=actuals_url,
|
|
sections=sections)
|
|
|
|
# Read in current expectations.
|
|
expectations_input_filepath = os.path.join(
|
|
self._expectations_root, builder, self._expectations_input_filename)
|
|
expectations_dict = gm_json.LoadFromFile(expectations_input_filepath)
|
|
expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
|
|
if not expected_results:
|
|
expected_results = {}
|
|
expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results
|
|
|
|
# Update the expectations in memory, skipping any tests/configs that
|
|
# the caller asked to exclude.
|
|
skipped_images = []
|
|
if results_to_update:
|
|
for (image_name, image_results) in results_to_update.iteritems():
|
|
if self._image_filename_re:
|
|
(test, config) = self._image_filename_re.match(image_name).groups()
|
|
if self._tests:
|
|
if test not in self._tests:
|
|
skipped_images.append(image_name)
|
|
continue
|
|
if self._configs:
|
|
if config not in self._configs:
|
|
skipped_images.append(image_name)
|
|
continue
|
|
if not expected_results.get(image_name):
|
|
expected_results[image_name] = {}
|
|
expected_results[image_name]\
|
|
[gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\
|
|
= [image_results]
|
|
if self._mark_unreviewed:
|
|
expected_results[image_name]\
|
|
[gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\
|
|
= False
|
|
if self._mark_ignore_failure:
|
|
expected_results[image_name]\
|
|
[gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\
|
|
= True
|
|
if self._bugs:
|
|
expected_results[image_name]\
|
|
[gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\
|
|
= self._bugs
|
|
if self._notes:
|
|
expected_results[image_name]\
|
|
[gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\
|
|
= self._notes
|
|
|
|
# Write out updated expectations.
|
|
expectations_output_filepath = os.path.join(
|
|
self._expectations_root, builder, self._expectations_output_filename)
|
|
gm_json.WriteToFile(expectations_dict, expectations_output_filepath)
|
|
|
|
# Mark the JSON file as plaintext, so text-style diffs can be applied.
|
|
# Fixes https://code.google.com/p/skia/issues/detail?id=1442
|
|
if self._using_svn:
|
|
self._Call(['svn', 'propset', '--quiet', 'svn:mime-type',
|
|
'text/x-json', expectations_output_filepath])
|
|
|
|
# main...
|
|
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog='Here is the full set of builders we know about:' +
|
|
'\n '.join([''] + sorted(TEST_BUILDERS)))
|
|
parser.add_argument('--actuals-base-url',
|
|
help=('base URL from which to read files containing JSON '
|
|
'summaries of actual GM results; defaults to '
|
|
'%(default)s. To get a specific revision (useful for '
|
|
'trybots) replace "svn" with "svn-history/r123". '
|
|
'If SKIMAGE is True, defaults to ' +
|
|
gm_json.SKIMAGE_ACTUALS_BASE_URL),
|
|
default='http://skia-autogen.googlecode.com/svn/gm-actual')
|
|
parser.add_argument('--actuals-filename',
|
|
help=('filename (within builder-specific subdirectories '
|
|
'of ACTUALS_BASE_URL) to read a summary of results '
|
|
'from; defaults to %(default)s'),
|
|
default='actual-results.json')
|
|
parser.add_argument('--add-new', action='store_true',
|
|
help=('in addition to the standard behavior of '
|
|
'updating expectations for failing tests, add '
|
|
'expectations for tests which don\'t have '
|
|
'expectations yet.'))
|
|
parser.add_argument('--add-ignored', action='store_true',
|
|
help=('in addition to the standard behavior of '
|
|
'updating expectations for failing tests, add '
|
|
'expectations for tests for which failures are '
|
|
'currently ignored.'))
|
|
parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+',
|
|
help=('Skia bug numbers (under '
|
|
'https://code.google.com/p/skia/issues/list ) which '
|
|
'pertain to this set of rebaselines.'))
|
|
parser.add_argument('--builders', metavar='BUILDER', nargs='+',
|
|
help=('which platforms to rebaseline; '
|
|
'if unspecified, rebaseline all known platforms '
|
|
'(see below for a list)'))
|
|
# TODO(epoger): Add test that exercises --configs argument.
|
|
parser.add_argument('--configs', metavar='CONFIG', nargs='+',
|
|
help=('which configurations to rebaseline, e.g. '
|
|
'"--configs 565 8888", as a filter over the full set '
|
|
'of results in ACTUALS_FILENAME; if unspecified, '
|
|
'rebaseline *all* configs that are available.'))
|
|
parser.add_argument('--expectations-filename',
|
|
help=('filename (under EXPECTATIONS_ROOT) to read '
|
|
'current expectations from, and to write new '
|
|
'expectations into (unless a separate '
|
|
'EXPECTATIONS_FILENAME_OUTPUT has been specified); '
|
|
'defaults to %(default)s'),
|
|
default='expected-results.json')
|
|
parser.add_argument('--expectations-filename-output',
|
|
help=('filename (under EXPECTATIONS_ROOT) to write '
|
|
'updated expectations into; by default, overwrites '
|
|
'the input file (EXPECTATIONS_FILENAME)'),
|
|
default='')
|
|
parser.add_argument('--expectations-root',
|
|
help=('root of expectations directory to update-- should '
|
|
'contain one or more builder subdirectories. '
|
|
'Defaults to %(default)s. If SKIMAGE is set, '
|
|
' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT),
|
|
default=os.path.join('expectations', 'gm'))
|
|
parser.add_argument('--keep-going-on-failure', action='store_true',
|
|
help=('instead of halting at the first error encountered, '
|
|
'keep going and rebaseline as many tests as '
|
|
'possible, and then report the full set of errors '
|
|
'at the end'))
|
|
parser.add_argument('--notes',
|
|
help=('free-form text notes to add to all updated '
|
|
'expectations'))
|
|
# TODO(epoger): Add test that exercises --tests argument.
|
|
parser.add_argument('--tests', metavar='TEST', nargs='+',
|
|
help=('which tests to rebaseline, e.g. '
|
|
'"--tests aaclip bigmatrix", as a filter over the '
|
|
'full set of results in ACTUALS_FILENAME; if '
|
|
'unspecified, rebaseline *all* tests that are '
|
|
'available.'))
|
|
parser.add_argument('--unreviewed', action='store_true',
|
|
help=('mark all expectations modified by this run as '
|
|
'"%s": False' %
|
|
gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED))
|
|
parser.add_argument('--ignore-failure', action='store_true',
|
|
help=('mark all expectations modified by this run as '
|
|
'"%s": True' %
|
|
gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED))
|
|
parser.add_argument('--from-trybot', action='store_true',
|
|
help=('pull the actual-results.json file from the '
|
|
'corresponding trybot, rather than the main builder'))
|
|
parser.add_argument('--skimage', action='store_true',
|
|
help=('Rebaseline skimage results instead of gm. Defaults '
|
|
'to False. If True, TESTS and CONFIGS are ignored, '
|
|
'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set '
|
|
'to alternate defaults, specific to skimage.'))
|
|
args = parser.parse_args()
|
|
exception_handler = ExceptionHandler(
|
|
keep_going_on_failure=args.keep_going_on_failure)
|
|
if args.builders:
|
|
builders = args.builders
|
|
missing_json_is_fatal = True
|
|
else:
|
|
builders = sorted(TEST_BUILDERS)
|
|
missing_json_is_fatal = False
|
|
if args.skimage:
|
|
# Use a different default if --skimage is specified.
|
|
if args.actuals_base_url == parser.get_default('actuals_base_url'):
|
|
args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL
|
|
if args.expectations_root == parser.get_default('expectations_root'):
|
|
args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT
|
|
for builder in builders:
|
|
if not builder in TEST_BUILDERS:
|
|
raise Exception(('unrecognized builder "%s"; ' +
|
|
'should be one of %s') % (
|
|
builder, TEST_BUILDERS))
|
|
|
|
expectations_json_file = os.path.join(args.expectations_root, builder,
|
|
args.expectations_filename)
|
|
if os.path.isfile(expectations_json_file):
|
|
rebaseliner = JsonRebaseliner(
|
|
expectations_root=args.expectations_root,
|
|
expectations_input_filename=args.expectations_filename,
|
|
expectations_output_filename=(args.expectations_filename_output or
|
|
args.expectations_filename),
|
|
tests=args.tests, configs=args.configs,
|
|
actuals_base_url=args.actuals_base_url,
|
|
actuals_filename=args.actuals_filename,
|
|
exception_handler=exception_handler,
|
|
add_new=args.add_new, add_ignored=args.add_ignored,
|
|
bugs=args.bugs, notes=args.notes,
|
|
mark_unreviewed=args.unreviewed,
|
|
mark_ignore_failure=args.ignore_failure,
|
|
from_trybot=args.from_trybot)
|
|
try:
|
|
rebaseliner.RebaselineSubdir(builder=builder)
|
|
except:
|
|
exception_handler.RaiseExceptionOrContinue()
|
|
else:
|
|
try:
|
|
raise _InternalException('expectations_json_file %s not found' %
|
|
expectations_json_file)
|
|
except:
|
|
exception_handler.RaiseExceptionOrContinue()
|
|
|
|
exception_handler.ReportAllFailures()
|