f5ad077741
BUG=806 Review URL: https://codereview.appspot.com/6492091 git-svn-id: http://skia.googlecode.com/svn/trunk@5435 2bbb7eff-a529-9590-31e7-b0007b416f81
234 lines
9.7 KiB
Python
234 lines
9.7 KiB
Python
'''
|
|
Downloads the actual gm results most recently generated by the Skia buildbots,
|
|
and adds any new ones to SVN control.
|
|
|
|
Launch with --help to see more information.
|
|
|
|
|
|
Copyright 2011 Google Inc.
|
|
|
|
Use of this source code is governed by a BSD-style license that can be
|
|
found in the LICENSE file.
|
|
'''
|
|
|
|
# common Python modules
|
|
import fnmatch
|
|
import optparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
# modules declared within this same directory
|
|
import compare_baselines
|
|
import svn
|
|
|
|
USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...'
|
|
HELP_STRING = '''
|
|
|
|
Downloads the actual gm results most recently generated by the Skia buildbots,
|
|
and adds any new ones to SVN control.
|
|
|
|
If no baseline_subdir is given, then this tool will download the most-recently
|
|
generated actual gm results for ALL platforms.
|
|
|
|
''' + compare_baselines.HOWTO_STRING
|
|
|
|
# Base URL of SVN repository where buildbots store actual gm image results.
|
|
GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
|
|
|
|
# GM baseline image URL in regular Skia SVN repository
|
|
GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected'
|
|
|
|
GM_EXPECTED_DIR = 'gm-expected'
|
|
|
|
OPTION_ADD_NEW_FILES = '--add-new-files'
|
|
OPTION_BUILDER_SUFFIX = '--builder-suffix'
|
|
DEFAULT_BUILDER_SUFFIX = '32'
|
|
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
|
|
|
|
def GetLatestResultsSvnUrl(svn, baseline_subdir, builder_suffix):
|
|
"""Return SVN URL from which we can check out the MOST RECENTLY generated images for this
|
|
baseline type.
|
|
|
|
@param svn an Svn object we can use to call ListSubdirs()
|
|
@param baseline_subdir indicates which platform we want images for
|
|
@param builder_suffix if multiple builders uploaded actual GM images for this baseline type,
|
|
choose the one whose builder_name matches this suffix
|
|
"""
|
|
root_url = '%s/%s' % (GM_ACTUAL_URL, baseline_subdir)
|
|
subdirs = sorted(svn.ListSubdirs(root_url))
|
|
num_subdirs = len(subdirs)
|
|
print('Within actual-results root URL %s, found these %d subdirs (presumably builder_names): %s'
|
|
% (root_url, num_subdirs, subdirs))
|
|
|
|
selected_subdir = None
|
|
if num_subdirs == 0:
|
|
print 'Found no builder_name subdirs, so reading actual images from the root_url itself.'
|
|
return root_url
|
|
elif num_subdirs == 1:
|
|
selected_subdir = subdirs[0]
|
|
print 'Found exactly one subdir in actual-results root_url: %s' % selected_subdir
|
|
else:
|
|
for possible_subdir in subdirs:
|
|
if possible_subdir.endswith(builder_suffix):
|
|
selected_subdir = possible_subdir
|
|
print 'Selected the first subdir ending in "%s": %s' % (
|
|
builder_suffix, selected_subdir)
|
|
break
|
|
|
|
if selected_subdir:
|
|
return '%s/%s/%s' % (root_url, selected_subdir, baseline_subdir)
|
|
else:
|
|
raise Exception('none of these subdirs of %s ended in "%s": %s' % (
|
|
root_url, builder_suffix, subdirs))
|
|
|
|
def GetBaselineSvnUrl(baseline_subdir):
|
|
"""Return SVN URL from which we can check out the baseline images for this
|
|
baseline type.
|
|
|
|
@param baseline_subdir indicates which platform we want baselines for
|
|
"""
|
|
return '%s/%s' % (GM_BASELINE_URL, baseline_subdir)
|
|
|
|
def CopyMatchingFiles(source_dir, dest_dir, filename_pattern, only_copy_updates=False):
|
|
"""Copy all files from source_dir that match filename_pattern, and save them (with their
|
|
original filenames) in dest_dir.
|
|
|
|
@param source_dir
|
|
@param dest_dir where to save the copied files
|
|
@param filename_pattern only copy files that match this Unix-style filename
|
|
pattern (e.g., '*.jpg')
|
|
@param only_copy_updates if True, only copy files that are already present in dest_dir
|
|
"""
|
|
all_filenames = os.listdir(source_dir)
|
|
matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
|
|
for filename in matching_filenames:
|
|
source_path = os.path.join(source_dir, filename)
|
|
dest_path = os.path.join(dest_dir, filename)
|
|
if only_copy_updates and not os.path.isfile(dest_path):
|
|
continue
|
|
shutil.copyfile(source_path, dest_path)
|
|
|
|
def DownloadBaselinesForOnePlatform(baseline_subdir):
|
|
"""Download most recently generated baseline images for a single platform,
|
|
and add any new ones to SVN control.
|
|
|
|
@param baseline_subdir
|
|
"""
|
|
# Create repo_to_modify to handle the SVN repository we will add files to.
|
|
gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk...
|
|
try:
|
|
os.makedirs(gm_dir)
|
|
except:
|
|
pass
|
|
repo_to_modify = svn.Svn(gm_dir)
|
|
repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir)
|
|
|
|
# If there are any locally modified files in that directory, exit
|
|
# (so that we don't risk overwriting the user's previous work).
|
|
new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
|
|
if not options.ignore_local_mods:
|
|
if new_and_modified_files:
|
|
raise Exception('Exiting because there are already new and/or '
|
|
'modified files in %s. To continue in spite of '
|
|
'that, run with %s option.' % (
|
|
baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
|
|
|
|
# Download actual gm images into a separate repo in a temporary directory.
|
|
actual_dir = tempfile.mkdtemp()
|
|
actual_repo = svn.Svn(actual_dir)
|
|
print 'Using %s as a temp dir' % actual_dir
|
|
actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baseline_subdir,
|
|
builder_suffix=options.builder_suffix)
|
|
print 'Reading actual buildbot GM results from %s' % actual_url
|
|
actual_repo.Checkout(actual_url, '.')
|
|
|
|
# Copy any of those files we are interested in into repo_to_modify,
|
|
# and then delete the temporary directory.
|
|
CopyMatchingFiles(source_dir=actual_dir,
|
|
dest_dir=os.path.join(gm_dir, baseline_subdir),
|
|
filename_pattern='*.png',
|
|
only_copy_updates=(not options.add_new_files))
|
|
shutil.rmtree(actual_dir)
|
|
actual_repo = None
|
|
|
|
# Add any new files to SVN control (if we are running with add_new_files).
|
|
if options.add_new_files:
|
|
new_files = repo_to_modify.GetNewFiles()
|
|
if new_files:
|
|
repo_to_modify.AddFiles(sorted(new_files))
|
|
|
|
# Set the mimetype property on any new/modified image files in
|
|
# baseline_subdir. (We used to set the mimetype property on *all* image
|
|
# files in the directory, even those whose content wasn't changing,
|
|
# but that caused confusion. See
|
|
# http://code.google.com/p/skia/issues/detail?id=618 .)
|
|
modified_files = repo_to_modify.GetNewAndModifiedFiles()
|
|
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
|
|
svn.PROPERTY_MIMETYPE, 'image/png')
|
|
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
|
|
svn.PROPERTY_MIMETYPE, 'application/pdf')
|
|
|
|
def RaiseUsageException():
|
|
raise Exception('%s\nRun with --help for more detail.' % (
|
|
USAGE_STRING % __file__))
|
|
|
|
def Main(options, args):
|
|
"""Allow other scripts to call this script with fake command-line args.
|
|
"""
|
|
# If no platforms are specified, do 'em all.
|
|
num_args = len(args)
|
|
if num_args == 0:
|
|
# TODO(epoger): automate the default set of platforms. We want to ensure
|
|
# that the user gets all of the platforms that the bots are running,
|
|
# not just whatever subdirectories he happens to have checked out...
|
|
# See http://code.google.com/p/skia/issues/detail?id=678
|
|
# Now that we have added Svn.ListSubdirs(), we should be able to do this
|
|
# pretty easily...
|
|
#
|
|
# For now, I generate this list using these Unix commands:
|
|
# svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort >/tmp/baselines
|
|
# svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base | sort >/tmp/actual
|
|
# comm -1 -2 /tmp/baselines /tmp/actual
|
|
args = [
|
|
'base-android-galaxy-nexus',
|
|
'base-android-nexus-7',
|
|
'base-android-nexus-s',
|
|
'base-android-xoom',
|
|
'base-macmini',
|
|
'base-macmini-lion-float',
|
|
'base-shuttle-win7-intel-float',
|
|
'base-shuttle_ubuntu12_ati5770',
|
|
]
|
|
|
|
# Trim all subdir names.
|
|
baseline_subdirs = []
|
|
for arg in args:
|
|
baseline_subdirs.append(arg.rstrip(os.sep))
|
|
|
|
# Process the subdirs, one at a time.
|
|
for baseline_subdir in baseline_subdirs:
|
|
DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir)
|
|
|
|
if __name__ == '__main__':
|
|
parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
|
|
parser.add_option(OPTION_ADD_NEW_FILES,
|
|
action='store_true', default=False,
|
|
help='in addition to downloading new versions of '
|
|
'existing baselines, also download baselines that are '
|
|
'not under SVN control yet')
|
|
parser.add_option(OPTION_BUILDER_SUFFIX,
|
|
action='store', type='string', default=DEFAULT_BUILDER_SUFFIX,
|
|
help='if multiple builders have uploaded actual GM images '
|
|
'for this platform, download the images uploaded by the '
|
|
'builder whose name ends in this suffix; defaults to '
|
|
'"%s".' % DEFAULT_BUILDER_SUFFIX)
|
|
parser.add_option(OPTION_IGNORE_LOCAL_MODS,
|
|
action='store_true', default=False,
|
|
help='allow tool to run even if there are already '
|
|
'local modifications in the baseline_subdir')
|
|
(options, args) = parser.parse_args()
|
|
Main(options, args)
|