rebaseline.py: split image-based rebaselining, which will go away soon, into its own script
(part of step 3 in https://goto.google.com/ChecksumTransitionDetail ) also adds new --expectations-root option R=scroggo@google.com, senorblanco@chromium.org Review URL: https://codereview.chromium.org/17379004 git-svn-id: http://skia.googlecode.com/svn/trunk@9689 2bbb7eff-a529-9590-31e7-b0007b416f81
This commit is contained in:
epoger@google.com
parent
27d302581a
commit
99a8ec9705
@ -21,6 +21,9 @@ import subprocess
|
||||
import sys
|
||||
import urllib2
|
||||
|
||||
# Imports from local directory
|
||||
import rebaseline_imagefiles
|
||||
|
||||
# Imports from within Skia
|
||||
#
|
||||
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
||||
@ -38,6 +41,7 @@ if GM_DIRECTORY not in sys.path:
|
||||
sys.path.append(GM_DIRECTORY)
|
||||
import gm_json
|
||||
|
||||
JSON_EXPECTATIONS_FILENAME='expected-results.json'
|
||||
|
||||
# Mapping of gm-expectations subdir (under
|
||||
# https://skia.googlecode.com/svn/gm-expected/ )
|
||||
@ -71,14 +75,19 @@ SUBDIR_MAPPING = {
|
||||
class CommandFailedException(Exception):
|
||||
pass
|
||||
|
||||
class Rebaseliner(object):
|
||||
# Object that rebaselines a JSON expectations file (not individual image files).
|
||||
#
|
||||
# TODO(epoger): Most of this is just the code from the old ImageRebaseliner...
|
||||
# some of it will need to be updated in order to properly rebaseline JSON files.
|
||||
# There is a lot of code duplicated between here and ImageRebaseliner, but
|
||||
# that's fine because we will delete ImageRebaseliner soon.
|
||||
class JsonRebaseliner(object):
|
||||
|
||||
# params:
|
||||
# expectations_root: root directory of all expectations
|
||||
# json_base_url: base URL from which to read json_filename
|
||||
# json_filename: filename (under json_base_url) from which to read a
|
||||
# summary of results; typically "actual-results.json"
|
||||
# subdirs: which platform subdirectories to rebaseline; if not specified,
|
||||
# rebaseline all platform subdirectories
|
||||
# tests: list of tests to rebaseline, or None if we should rebaseline
|
||||
# whatever files the JSON results summary file tells us to
|
||||
# configs: which configs to run for each test; this should only be
|
||||
@ -88,24 +97,23 @@ class Rebaseliner(object):
|
||||
# files to checkout, display a list of operations that
|
||||
# we would normally perform
|
||||
# add_new: if True, add expectations for tests which don't have any yet
|
||||
def __init__(self, json_base_url, json_filename,
|
||||
subdirs=None, tests=None, configs=None, dry_run=False,
|
||||
add_new=False):
|
||||
# missing_json_is_fatal: whether to halt execution if we cannot read a
|
||||
# JSON actual result summary file
|
||||
def __init__(self, expectations_root, json_base_url, json_filename,
|
||||
tests=None, configs=None, dry_run=False,
|
||||
add_new=False, missing_json_is_fatal=False):
|
||||
raise ValueError('JsonRebaseliner not yet implemented') # TODO(epoger)
|
||||
if configs and not tests:
|
||||
raise ValueError('configs should only be specified if tests ' +
|
||||
'were specified also')
|
||||
self._expectations_root = expectations_root
|
||||
self._tests = tests
|
||||
self._configs = configs
|
||||
if not subdirs:
|
||||
self._subdirs = sorted(SUBDIR_MAPPING.keys())
|
||||
self._missing_json_is_fatal = False
|
||||
else:
|
||||
self._subdirs = subdirs
|
||||
self._missing_json_is_fatal = True
|
||||
self._json_base_url = json_base_url
|
||||
self._json_filename = json_filename
|
||||
self._dry_run = dry_run
|
||||
self._add_new = add_new
|
||||
self._missing_json_is_fatal = missing_json_is_fatal
|
||||
self._googlestorage_gm_actuals_root = (
|
||||
'http://chromium-skia-gm.commondatastorage.googleapis.com/gm')
|
||||
self._testname_pattern = re.compile('(\S+)_(\S+).png')
|
||||
@ -339,34 +347,33 @@ class Rebaseliner(object):
|
||||
outfilename=outfilename,
|
||||
all_results=all_results)
|
||||
|
||||
# Rebaseline all platforms/tests/types we specified in the constructor.
|
||||
def RebaselineAll(self):
|
||||
for subdir in self._subdirs:
|
||||
if not subdir in SUBDIR_MAPPING.keys():
|
||||
raise Exception(('unrecognized platform subdir "%s"; ' +
|
||||
'should be one of %s') % (
|
||||
subdir, SUBDIR_MAPPING.keys()))
|
||||
builder_name = SUBDIR_MAPPING[subdir]
|
||||
json_url = '/'.join([self._json_base_url,
|
||||
subdir, builder_name, subdir,
|
||||
self._json_filename])
|
||||
all_results = self._GetActualResults(json_url=json_url)
|
||||
# Rebaseline all tests/types we specified in the constructor,
|
||||
# within this gm-expectations subdir.
|
||||
#
|
||||
# params:
|
||||
# subdir : e.g. 'base-shuttle-win7-intel-float'
|
||||
# builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
|
||||
def RebaselineSubdir(self, subdir, builder):
|
||||
json_url = '/'.join([self._json_base_url,
|
||||
subdir, builder, subdir,
|
||||
self._json_filename])
|
||||
all_results = self._GetActualResults(json_url=json_url)
|
||||
|
||||
if self._tests:
|
||||
for test in self._tests:
|
||||
self._RebaselineOneTest(expectations_subdir=subdir,
|
||||
builder_name=builder_name,
|
||||
test=test, all_results=all_results)
|
||||
else: # get the raw list of files that need rebaselining from JSON
|
||||
filenames = self._GetFilesToRebaseline(json_url=json_url,
|
||||
add_new=self._add_new)
|
||||
for filename in filenames:
|
||||
outfilename = os.path.join(subdir, filename);
|
||||
self._RebaselineOneFile(expectations_subdir=subdir,
|
||||
builder_name=builder_name,
|
||||
infilename=filename,
|
||||
outfilename=outfilename,
|
||||
all_results=all_results)
|
||||
if self._tests:
|
||||
for test in self._tests:
|
||||
self._RebaselineOneTest(expectations_subdir=subdir,
|
||||
builder_name=builder,
|
||||
test=test, all_results=all_results)
|
||||
else: # get the raw list of files that need rebaselining from JSON
|
||||
filenames = self._GetFilesToRebaseline(json_url=json_url,
|
||||
add_new=self._add_new)
|
||||
for filename in filenames:
|
||||
outfilename = os.path.join(subdir, filename);
|
||||
self._RebaselineOneFile(expectations_subdir=subdir,
|
||||
builder_name=builder,
|
||||
infilename=filename,
|
||||
outfilename=outfilename,
|
||||
all_results=all_results)
|
||||
|
||||
# main...
|
||||
|
||||
@ -385,6 +392,11 @@ parser.add_argument('--dry-run', action='store_true',
|
||||
help='instead of actually downloading files or adding ' +
|
||||
'files to checkout, display a list of operations that ' +
|
||||
'we would normally perform')
|
||||
parser.add_argument('--expectations-root',
|
||||
help='root of expectations directory to update-- should ' +
|
||||
'contain one or more base-* subdirectories. Defaults to ' +
|
||||
'%(default)s',
|
||||
default='.')
|
||||
parser.add_argument('--json-base-url',
|
||||
help='base URL from which to read JSON_FILENAME ' +
|
||||
'files; defaults to %(default)s',
|
||||
@ -403,9 +415,46 @@ parser.add_argument('--tests', metavar='TEST', nargs='+',
|
||||
'failing tests (according to the actual-results.json ' +
|
||||
'file) will be rebaselined.')
|
||||
args = parser.parse_args()
|
||||
rebaseliner = Rebaseliner(tests=args.tests, configs=args.configs,
|
||||
subdirs=args.subdirs, dry_run=args.dry_run,
|
||||
json_base_url=args.json_base_url,
|
||||
json_filename=args.json_filename,
|
||||
add_new=args.add_new)
|
||||
rebaseliner.RebaselineAll()
|
||||
if args.subdirs:
|
||||
subdirs = args.subdirs
|
||||
missing_json_is_fatal = True
|
||||
else:
|
||||
subdirs = sorted(SUBDIR_MAPPING.keys())
|
||||
missing_json_is_fatal = False
|
||||
for subdir in subdirs:
|
||||
if not subdir in SUBDIR_MAPPING.keys():
|
||||
raise Exception(('unrecognized platform subdir "%s"; ' +
|
||||
'should be one of %s') % (
|
||||
subdir, SUBDIR_MAPPING.keys()))
|
||||
builder = SUBDIR_MAPPING[subdir]
|
||||
|
||||
# We instantiate different Rebaseliner objects depending
|
||||
# on whether we are rebaselining an expected-results.json file, or
|
||||
# individual image files. Different gm-expected subdirectories may move
|
||||
# from individual image files to JSON-format expectations at different
|
||||
# times, so we need to make this determination per subdirectory.
|
||||
#
|
||||
# See https://goto.google.com/ChecksumTransitionDetail
|
||||
expectations_json_file = os.path.join(args.expectations_root, subdir,
|
||||
JSON_EXPECTATIONS_FILENAME)
|
||||
if os.path.isfile(expectations_json_file):
|
||||
sys.stderr.write('ERROR: JsonRebaseliner is not implemented yet.\n')
|
||||
sys.exit(1)
|
||||
rebaseliner = JsonRebaseliner(
|
||||
expectations_root=args.expectations_root,
|
||||
tests=args.tests, configs=args.configs,
|
||||
dry_run=args.dry_run,
|
||||
json_base_url=args.json_base_url,
|
||||
json_filename=args.json_filename,
|
||||
add_new=args.add_new,
|
||||
missing_json_is_fatal=missing_json_is_fatal)
|
||||
else:
|
||||
rebaseliner = rebaseline_imagefiles.ImageRebaseliner(
|
||||
expectations_root=args.expectations_root,
|
||||
tests=args.tests, configs=args.configs,
|
||||
dry_run=args.dry_run,
|
||||
json_base_url=args.json_base_url,
|
||||
json_filename=args.json_filename,
|
||||
add_new=args.add_new,
|
||||
missing_json_is_fatal=missing_json_is_fatal)
|
||||
rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder)
|
||||
|
||||
341
tools/rebaseline_imagefiles.py
Executable file
341
tools/rebaseline_imagefiles.py
Executable file
@ -0,0 +1,341 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
'''
|
||||
Copyright 2013 Google Inc.
|
||||
|
||||
Use of this source code is governed by a BSD-style license that can be
|
||||
found in the LICENSE file.
|
||||
'''
|
||||
|
||||
'''
|
||||
Rebaselines GM test results as individual image files
|
||||
(the "old way", before https://goto.google.com/ChecksumTransitionDetail ).
|
||||
|
||||
Once we have switched our expectations to JSON form for all platforms,
|
||||
we can delete this file.
|
||||
|
||||
There is a lot of code duplicated between here and rebaseline.py, but
|
||||
that's fine because we will delete this file soon.
|
||||
'''
|
||||
|
||||
# System-level imports
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib2
|
||||
|
||||
# Imports from within Skia
|
||||
#
|
||||
# We need to add the 'gm' directory, so that we can import gm_json.py within
|
||||
# that directory. That script allows us to parse the actual-results.json file
|
||||
# written out by the GM tool.
|
||||
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
|
||||
# so any dirs that are already in the PYTHONPATH will be preferred.
|
||||
#
|
||||
# This assumes that the 'gm' directory has been checked out as a sibling of
|
||||
# the 'tools' directory containing this script, which will be the case if
|
||||
# 'trunk' was checked out as a single unit.
|
||||
GM_DIRECTORY = os.path.realpath(
|
||||
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
|
||||
if GM_DIRECTORY not in sys.path:
|
||||
sys.path.append(GM_DIRECTORY)
|
||||
import gm_json
|
||||
|
||||
|
||||
class CommandFailedException(Exception):
|
||||
pass
|
||||
|
||||
class ImageRebaseliner(object):
|
||||
|
||||
# params:
|
||||
# expectations_root: root directory of all expectations
|
||||
# json_base_url: base URL from which to read json_filename
|
||||
# json_filename: filename (under json_base_url) from which to read a
|
||||
# summary of results; typically "actual-results.json"
|
||||
# tests: list of tests to rebaseline, or None if we should rebaseline
|
||||
# whatever files the JSON results summary file tells us to
|
||||
# configs: which configs to run for each test; this should only be
|
||||
# specified if the list of tests was also specified (otherwise,
|
||||
# the JSON file will give us test names and configs)
|
||||
# dry_run: if True, instead of actually downloading files or adding
|
||||
# files to checkout, display a list of operations that
|
||||
# we would normally perform
|
||||
# add_new: if True, add expectations for tests which don't have any yet
|
||||
# missing_json_is_fatal: whether to halt execution if we cannot read a
|
||||
# JSON actual result summary file
|
||||
def __init__(self, expectations_root, json_base_url, json_filename,
|
||||
tests=None, configs=None, dry_run=False,
|
||||
add_new=False, missing_json_is_fatal=False):
|
||||
if configs and not tests:
|
||||
raise ValueError('configs should only be specified if tests ' +
|
||||
'were specified also')
|
||||
self._expectations_root = expectations_root
|
||||
self._tests = tests
|
||||
self._configs = configs
|
||||
self._json_base_url = json_base_url
|
||||
self._json_filename = json_filename
|
||||
self._dry_run = dry_run
|
||||
self._add_new = add_new
|
||||
self._missing_json_is_fatal = missing_json_is_fatal
|
||||
self._googlestorage_gm_actuals_root = (
|
||||
'http://chromium-skia-gm.commondatastorage.googleapis.com/gm')
|
||||
self._testname_pattern = re.compile('(\S+)_(\S+).png')
|
||||
self._is_svn_checkout = (
|
||||
os.path.exists('.svn') or
|
||||
os.path.exists(os.path.join(os.pardir, '.svn')))
|
||||
self._is_git_checkout = (
|
||||
os.path.exists('.git') or
|
||||
os.path.exists(os.path.join(os.pardir, '.git')))
|
||||
|
||||
# If dry_run is False, execute subprocess.call(cmd).
|
||||
# If dry_run is True, print the command we would have otherwise run.
|
||||
# Raises a CommandFailedException if the command fails.
|
||||
def _Call(self, cmd):
|
||||
if self._dry_run:
|
||||
print '%s' % ' '.join(cmd)
|
||||
return
|
||||
if subprocess.call(cmd) != 0:
|
||||
raise CommandFailedException('error running command: ' +
|
||||
' '.join(cmd))
|
||||
|
||||
# Download a single actual result from GoogleStorage, returning True if it
|
||||
# succeeded.
|
||||
def _DownloadFromGoogleStorage(self, infilename, outfilename, all_results):
|
||||
test_name = self._testname_pattern.match(infilename).group(1)
|
||||
if not test_name:
|
||||
print '# unable to find test_name for infilename %s' % infilename
|
||||
return False
|
||||
try:
|
||||
hash_type, hash_value = all_results[infilename]
|
||||
except KeyError:
|
||||
print ('# unable to find filename %s in all_results dict' %
|
||||
infilename)
|
||||
return False
|
||||
except ValueError as e:
|
||||
print '# ValueError reading filename %s from all_results dict: %s'%(
|
||||
infilename, e)
|
||||
return False
|
||||
url = '%s/%s/%s/%s.png' % (self._googlestorage_gm_actuals_root,
|
||||
hash_type, test_name, hash_value)
|
||||
try:
|
||||
self._DownloadFile(source_url=url, dest_filename=outfilename)
|
||||
return True
|
||||
except CommandFailedException:
|
||||
print '# Couldn\'t fetch gs_url %s' % url
|
||||
return False
|
||||
|
||||
# Download a single actual result from skia-autogen, returning True if it
|
||||
# succeeded.
|
||||
def _DownloadFromAutogen(self, infilename, outfilename,
|
||||
expectations_subdir, builder_name):
|
||||
url = ('http://skia-autogen.googlecode.com/svn/gm-actual/' +
|
||||
expectations_subdir + '/' + builder_name + '/' +
|
||||
expectations_subdir + '/' + infilename)
|
||||
try:
|
||||
self._DownloadFile(source_url=url, dest_filename=outfilename)
|
||||
return True
|
||||
except CommandFailedException:
|
||||
print '# Couldn\'t fetch autogen_url %s' % url
|
||||
return False
|
||||
|
||||
# Download a single file, raising a CommandFailedException if it fails.
|
||||
def _DownloadFile(self, source_url, dest_filename):
|
||||
# Download into a temporary file and then rename it afterwards,
|
||||
# so that we don't corrupt the existing file if it fails midway thru.
|
||||
temp_filename = os.path.join(os.path.dirname(dest_filename),
|
||||
'.temp-' + os.path.basename(dest_filename))
|
||||
|
||||
# TODO(epoger): Replace calls to "curl"/"mv" (which will only work on
|
||||
# Unix) with a Python HTTP library (which should work cross-platform)
|
||||
self._Call([ 'curl', '--fail', '--silent', source_url,
|
||||
'--output', temp_filename ])
|
||||
self._Call([ 'mv', temp_filename, dest_filename ])
|
||||
|
||||
# Returns the full contents of a URL, as a single string.
|
||||
#
|
||||
# Unlike standard URL handling, we allow relative "file:" URLs;
|
||||
# for example, "file:one/two" resolves to the file ./one/two
|
||||
# (relative to current working dir)
|
||||
def _GetContentsOfUrl(self, url):
|
||||
file_prefix = 'file:'
|
||||
if url.startswith(file_prefix):
|
||||
filename = url[len(file_prefix):]
|
||||
return open(filename, 'r').read()
|
||||
else:
|
||||
return urllib2.urlopen(url).read()
|
||||
|
||||
# Returns a dictionary of actual results from actual-results.json file.
|
||||
#
|
||||
# The dictionary returned has this format:
|
||||
# {
|
||||
# u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322],
|
||||
# u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152],
|
||||
# u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]
|
||||
# }
|
||||
#
|
||||
# If the JSON actual result summary file cannot be loaded, the behavior
|
||||
# depends on self._missing_json_is_fatal:
|
||||
# - if true: execution will halt with an exception
|
||||
# - if false: we will log an error message but return an empty dictionary
|
||||
#
|
||||
# params:
|
||||
# json_url: URL pointing to a JSON actual result summary file
|
||||
# sections: a list of section names to include in the results, e.g.
|
||||
# [gm_json.JSONKEY_ACTUALRESULTS_FAILED,
|
||||
# gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;
|
||||
# if None, then include ALL sections.
|
||||
def _GetActualResults(self, json_url, sections=None):
|
||||
try:
|
||||
json_contents = self._GetContentsOfUrl(json_url)
|
||||
except (urllib2.HTTPError, IOError):
|
||||
message = 'unable to load JSON summary URL %s' % json_url
|
||||
if self._missing_json_is_fatal:
|
||||
raise ValueError(message)
|
||||
else:
|
||||
print '# %s' % message
|
||||
return {}
|
||||
|
||||
json_dict = gm_json.LoadFromString(json_contents)
|
||||
results_to_return = {}
|
||||
actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
|
||||
if not sections:
|
||||
sections = actual_results.keys()
|
||||
for section in sections:
|
||||
section_results = actual_results[section]
|
||||
if section_results:
|
||||
results_to_return.update(section_results)
|
||||
return results_to_return
|
||||
|
||||
# Returns a list of files that require rebaselining.
|
||||
#
|
||||
# Note that this returns a list of FILES, like this:
|
||||
# ['imageblur_565.png', 'xfermodes_pdf.png']
|
||||
# rather than a list of TESTS, like this:
|
||||
# ['imageblur', 'xfermodes']
|
||||
#
|
||||
# params:
|
||||
# json_url: URL pointing to a JSON actual result summary file
|
||||
# add_new: if True, then return files listed in any of these sections:
|
||||
# - JSONKEY_ACTUALRESULTS_FAILED
|
||||
# - JSONKEY_ACTUALRESULTS_NOCOMPARISON
|
||||
# if False, then return files listed in these sections:
|
||||
# - JSONKEY_ACTUALRESULTS_FAILED
|
||||
#
|
||||
def _GetFilesToRebaseline(self, json_url, add_new):
|
||||
if self._dry_run:
|
||||
print ''
|
||||
print '#'
|
||||
print ('# Getting files to rebaseline from JSON summary URL %s ...'
|
||||
% json_url)
|
||||
sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
|
||||
if add_new:
|
||||
sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
|
||||
results_to_rebaseline = self._GetActualResults(json_url=json_url,
|
||||
sections=sections)
|
||||
files_to_rebaseline = results_to_rebaseline.keys()
|
||||
files_to_rebaseline.sort()
|
||||
print '# ... found files_to_rebaseline %s' % files_to_rebaseline
|
||||
if self._dry_run:
|
||||
print '#'
|
||||
return files_to_rebaseline
|
||||
|
||||
# Rebaseline a single file.
|
||||
def _RebaselineOneFile(self, expectations_subdir, builder_name,
|
||||
infilename, outfilename, all_results):
|
||||
if self._dry_run:
|
||||
print ''
|
||||
print '# ' + infilename
|
||||
|
||||
# First try to download this result image from Google Storage.
|
||||
# If that fails, try skia-autogen.
|
||||
# If that fails too, just go on to the next file.
|
||||
#
|
||||
# This not treated as a fatal failure because not all
|
||||
# platforms generate all configs (e.g., Android does not
|
||||
# generate PDF).
|
||||
#
|
||||
# TODO(epoger): Once we are downloading only files that the
|
||||
# actual-results.json file told us to, this should become a
|
||||
# fatal error. (If the actual-results.json file told us that
|
||||
# the test failed with XXX results, we should be able to download
|
||||
# those results every time.)
|
||||
if not self._DownloadFromGoogleStorage(infilename=infilename,
|
||||
outfilename=outfilename,
|
||||
all_results=all_results):
|
||||
if not self._DownloadFromAutogen(infilename=infilename,
|
||||
outfilename=outfilename,
|
||||
expectations_subdir=expectations_subdir,
|
||||
builder_name=builder_name):
|
||||
print '# Couldn\'t fetch infilename ' + infilename
|
||||
return
|
||||
|
||||
# Add this file to version control (if appropriate).
|
||||
if self._add_new:
|
||||
if self._is_svn_checkout:
|
||||
cmd = [ 'svn', 'add', '--quiet', outfilename ]
|
||||
self._Call(cmd)
|
||||
cmd = [ 'svn', 'propset', '--quiet', 'svn:mime-type',
|
||||
'image/png', outfilename ];
|
||||
self._Call(cmd)
|
||||
elif self._is_git_checkout:
|
||||
cmd = [ 'git', 'add', outfilename ]
|
||||
self._Call(cmd)
|
||||
|
||||
# Rebaseline the given configs for a single test.
|
||||
#
|
||||
# params:
|
||||
# expectations_subdir
|
||||
# builder_name
|
||||
# test: a single test to rebaseline
|
||||
# all_results: a dictionary of all actual results
|
||||
def _RebaselineOneTest(self, expectations_subdir, builder_name, test,
|
||||
all_results):
|
||||
if self._configs:
|
||||
configs = self._configs
|
||||
else:
|
||||
if (expectations_subdir == 'base-shuttle-win7-intel-angle'):
|
||||
configs = [ 'angle', 'anglemsaa16' ]
|
||||
else:
|
||||
configs = [ '565', '8888', 'gpu', 'pdf', 'mesa', 'msaa16',
|
||||
'msaa4' ]
|
||||
if self._dry_run:
|
||||
print ''
|
||||
print '# ' + expectations_subdir + ':'
|
||||
for config in configs:
|
||||
infilename = test + '_' + config + '.png'
|
||||
outfilename = os.path.join(self._expectations_root,
|
||||
expectations_subdir, infilename);
|
||||
self._RebaselineOneFile(expectations_subdir=expectations_subdir,
|
||||
builder_name=builder_name,
|
||||
infilename=infilename,
|
||||
outfilename=outfilename,
|
||||
all_results=all_results)
|
||||
|
||||
# Rebaseline all tests/types we specified in the constructor,
|
||||
# within this gm-expectations subdir.
|
||||
#
|
||||
# params:
|
||||
# subdir : e.g. 'base-shuttle-win7-intel-float'
|
||||
# builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
|
||||
def RebaselineSubdir(self, subdir, builder):
|
||||
json_url = '/'.join([self._json_base_url,
|
||||
subdir, builder, subdir,
|
||||
self._json_filename])
|
||||
all_results = self._GetActualResults(json_url=json_url)
|
||||
|
||||
if self._tests:
|
||||
for test in self._tests:
|
||||
self._RebaselineOneTest(expectations_subdir=subdir,
|
||||
builder_name=builder,
|
||||
test=test, all_results=all_results)
|
||||
else: # get the raw list of files that need rebaselining from JSON
|
||||
filenames = self._GetFilesToRebaseline(json_url=json_url,
|
||||
add_new=self._add_new)
|
||||
for filename in filenames:
|
||||
outfilename = os.path.join(subdir, filename);
|
||||
self._RebaselineOneFile(expectations_subdir=subdir,
|
||||
builder_name=builder,
|
||||
infilename=filename,
|
||||
outfilename=outfilename,
|
||||
all_results=all_results)
|
||||
@ -0,0 +1,52 @@
|
||||
{
|
||||
"expected-results" : {
|
||||
"3x3bitmaprect_565.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"3x3bitmaprect_8888.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aaclip_gpu.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 11899819492385205974 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aarectmodes_565.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 14760033689012826769 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"imageblur_565.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 17796243856503591523 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"imageblur_8888.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 7426416989687670152 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"shadertext3_8888.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 10593797161686785561 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"xfermodeimagefilter_pdf.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"xfermodes_pdf.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 9151974350149210736 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
{
|
||||
"expected-results" : {
|
||||
"aaclip_565.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 12345 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aaclip_8888.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 67890 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aaclip_gpu.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 11899819492385205974 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aaclip_pdf.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
{
|
||||
"expected-results" : {
|
||||
"3x3bitmaprect_565.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"3x3bitmaprect_8888.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aaclip_gpu.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 11899819492385205974 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"aarectmodes_565.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 14760033689012826769 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"imageblur_565.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 3359963596899141322 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"imageblur_8888.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 4217923806027861152 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"shadertext3_8888.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 3713708307125704716 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"xfermodeimagefilter_pdf.png" : {
|
||||
"allowed-digests" : null,
|
||||
"ignore-failure" : false
|
||||
},
|
||||
"xfermodes_pdf.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 9151974350149210736 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1 +1 @@
|
||||
python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
|
||||
python tools/rebaseline.py --dry-run --expectations-root fake/expectations/path --json-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float
|
||||
|
||||
@ -3,46 +3,46 @@
|
||||
|
||||
# test1_565.png
|
||||
# unable to find filename test1_565.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test1_565.png --output base-android-galaxy-nexus/.temp-test1_565.png
|
||||
mv base-android-galaxy-nexus/.temp-test1_565.png base-android-galaxy-nexus/test1_565.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test1_565.png --output fake/expectations/path/base-android-galaxy-nexus/.temp-test1_565.png
|
||||
mv fake/expectations/path/base-android-galaxy-nexus/.temp-test1_565.png fake/expectations/path/base-android-galaxy-nexus/test1_565.png
|
||||
|
||||
# test1_8888.png
|
||||
# unable to find filename test1_8888.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test1_8888.png --output base-android-galaxy-nexus/.temp-test1_8888.png
|
||||
mv base-android-galaxy-nexus/.temp-test1_8888.png base-android-galaxy-nexus/test1_8888.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test1_8888.png --output fake/expectations/path/base-android-galaxy-nexus/.temp-test1_8888.png
|
||||
mv fake/expectations/path/base-android-galaxy-nexus/.temp-test1_8888.png fake/expectations/path/base-android-galaxy-nexus/test1_8888.png
|
||||
|
||||
# base-android-galaxy-nexus:
|
||||
|
||||
# test2_565.png
|
||||
# unable to find filename test2_565.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test2_565.png --output base-android-galaxy-nexus/.temp-test2_565.png
|
||||
mv base-android-galaxy-nexus/.temp-test2_565.png base-android-galaxy-nexus/test2_565.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test2_565.png --output fake/expectations/path/base-android-galaxy-nexus/.temp-test2_565.png
|
||||
mv fake/expectations/path/base-android-galaxy-nexus/.temp-test2_565.png fake/expectations/path/base-android-galaxy-nexus/test2_565.png
|
||||
|
||||
# test2_8888.png
|
||||
# unable to find filename test2_8888.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test2_8888.png --output base-android-galaxy-nexus/.temp-test2_8888.png
|
||||
mv base-android-galaxy-nexus/.temp-test2_8888.png base-android-galaxy-nexus/test2_8888.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/test2_8888.png --output fake/expectations/path/base-android-galaxy-nexus/.temp-test2_8888.png
|
||||
mv fake/expectations/path/base-android-galaxy-nexus/.temp-test2_8888.png fake/expectations/path/base-android-galaxy-nexus/test2_8888.png
|
||||
|
||||
# base-shuttle-win7-intel-float:
|
||||
|
||||
# test1_565.png
|
||||
# unable to find filename test1_565.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test1_565.png --output base-shuttle-win7-intel-float/.temp-test1_565.png
|
||||
mv base-shuttle-win7-intel-float/.temp-test1_565.png base-shuttle-win7-intel-float/test1_565.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test1_565.png --output fake/expectations/path/base-shuttle-win7-intel-float/.temp-test1_565.png
|
||||
mv fake/expectations/path/base-shuttle-win7-intel-float/.temp-test1_565.png fake/expectations/path/base-shuttle-win7-intel-float/test1_565.png
|
||||
|
||||
# test1_8888.png
|
||||
# unable to find filename test1_8888.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test1_8888.png --output base-shuttle-win7-intel-float/.temp-test1_8888.png
|
||||
mv base-shuttle-win7-intel-float/.temp-test1_8888.png base-shuttle-win7-intel-float/test1_8888.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test1_8888.png --output fake/expectations/path/base-shuttle-win7-intel-float/.temp-test1_8888.png
|
||||
mv fake/expectations/path/base-shuttle-win7-intel-float/.temp-test1_8888.png fake/expectations/path/base-shuttle-win7-intel-float/test1_8888.png
|
||||
|
||||
# base-shuttle-win7-intel-float:
|
||||
|
||||
# test2_565.png
|
||||
# unable to find filename test2_565.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test2_565.png --output base-shuttle-win7-intel-float/.temp-test2_565.png
|
||||
mv base-shuttle-win7-intel-float/.temp-test2_565.png base-shuttle-win7-intel-float/test2_565.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test2_565.png --output fake/expectations/path/base-shuttle-win7-intel-float/.temp-test2_565.png
|
||||
mv fake/expectations/path/base-shuttle-win7-intel-float/.temp-test2_565.png fake/expectations/path/base-shuttle-win7-intel-float/test2_565.png
|
||||
|
||||
# test2_8888.png
|
||||
# unable to find filename test2_8888.png in all_results dict
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test2_8888.png --output base-shuttle-win7-intel-float/.temp-test2_8888.png
|
||||
mv base-shuttle-win7-intel-float/.temp-test2_8888.png base-shuttle-win7-intel-float/test2_8888.png
|
||||
curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/test2_8888.png --output fake/expectations/path/base-shuttle-win7-intel-float/.temp-test2_8888.png
|
||||
mv fake/expectations/path/base-shuttle-win7-intel-float/.temp-test2_8888.png fake/expectations/path/base-shuttle-win7-intel-float/test2_8888.png
|
||||
|
||||
@ -0,0 +1 @@
|
||||
python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root tools/tests/rebaseline/input/json1
|
||||
@ -0,0 +1 @@
|
||||
1
|
||||
@ -0,0 +1 @@
|
||||
ERROR: JsonRebaseliner is not implemented yet.
|
||||
@ -209,9 +209,10 @@ fi
|
||||
|
||||
REBASELINE_INPUT=tools/tests/rebaseline/input
|
||||
REBASELINE_OUTPUT=tools/tests/rebaseline/output
|
||||
rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
|
||||
rebaseline_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset"
|
||||
rebaseline_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all"
|
||||
rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1"
|
||||
rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new"
|
||||
rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root $REBASELINE_INPUT/json1" "$REBASELINE_OUTPUT/using-json1-expectations"
|
||||
|
||||
echo "All tests passed."
|
||||
|
||||
Loading…
Reference in New Issue
Block a user