skia2/tools/download_baselines.py
epoger@google.com 0f645b68a3 download_baselines.py : only set svn properties of files whose content has changed
and while I was at it, sort the filenames before acting on them, to make the output easier to read

BUG=http://code.google.com/p/skia/issues/detail?id=618
Review URL: https://codereview.appspot.com/6223061

git-svn-id: http://skia.googlecode.com/svn/trunk@4031 2bbb7eff-a529-9590-31e7-b0007b416f81
2012-05-22 19:14:01 +00:00

179 lines
6.9 KiB
Python

'''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.
Launch with --help to see more information.
Copyright 2011 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''
# common Python modules
import fnmatch
import optparse
import os
import re
import shutil
import sys
import tempfile
# modules declared within this same directory
import compare_baselines
import svn
USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...'
HELP_STRING = '''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.
If no baseline_subdir is given, then this tool will download the most-recently
generated actual gm results for ALL platforms.
''' + compare_baselines.HOWTO_STRING
# Base URL of SVN repository where buildbots store actual gm image results.
SVN_BASE_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
OPTION_ADD_NEW_FILES = '--add-new-files'
def GetLatestResultsSvnUrl(baseline_subdir):
"""Return SVN URL from which we can check out the MOST RECENTLY generated
images for this baseline type.
@param baseline_subdir indicates which platform we want images for
"""
# trim off 'gm/' prefix
gm_prefix = 'gm%s' % os.sep
if not baseline_subdir.startswith(gm_prefix):
raise Exception('baseline_subdir "%s" should start with "%s"' % (
baseline_subdir, gm_prefix))
return '%s/%s' % (SVN_BASE_URL, baseline_subdir[len(gm_prefix):])
def CopyMatchingFiles(source_dir, dest_dir, filename_pattern,
only_copy_updates=False):
"""Copy all files from source_dir that match filename_pattern, and
save them (with their original filenames) in dest_dir.
@param source_dir
@param dest_dir where to save the copied files
@param filename_pattern only copy files that match this Unix-style filename
pattern (e.g., '*.jpg')
@param only_copy_updates if True, only copy files that are already
present in dest_dir
"""
all_filenames = os.listdir(source_dir)
matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
for filename in matching_filenames:
source_path = os.path.join(source_dir, filename)
dest_path = os.path.join(dest_dir, filename)
if only_copy_updates and not os.path.isfile(dest_path):
continue
shutil.copyfile(source_path, dest_path)
def DownloadBaselinesForOnePlatform(baseline_subdir):
"""Download most recently generated baseline images for a single platform,
and add any new ones to SVN control.
@param baseline_subdir
"""
# Create repo_to_modify to handle the SVN repository we will add files to.
repo_to_modify = svn.Svn(baseline_subdir)
# If there are any locally modified files in that directory, exit
# (so that we don't risk overwriting the user's previous work).
new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
if not options.ignore_local_mods:
if new_and_modified_files:
raise Exception('Exiting because there are already new and/or '
'modified files in %s. To continue in spite of '
'that, run with %s option.' % (
baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
# Download actual gm images into a separate repo in a temporary directory.
tempdir = tempfile.mkdtemp()
download_repo = svn.Svn(tempdir)
download_repo.Checkout(GetLatestResultsSvnUrl(baseline_subdir), '.')
# Copy any of those files we are interested in into repo_to_modify,
# and then delete the temporary directory.
CopyMatchingFiles(source_dir=tempdir, dest_dir=baseline_subdir,
filename_pattern='*.png',
only_copy_updates=(not options.add_new_files))
shutil.rmtree(tempdir)
download_repo = None
# Add any new files to SVN control (if we are running with add_new_files).
if options.add_new_files:
new_files = repo_to_modify.GetNewFiles()
if new_files:
repo_to_modify.AddFiles(sorted(new_files))
# Set the mimetype property on any new/modified image files in
# baseline_subdir. (We used to set the mimetype property on *all* image
# files in the directory, even those whose content wasn't changing,
# but that caused confusion. See
# http://code.google.com/p/skia/issues/detail?id=618 .)
modified_files = repo_to_modify.GetNewAndModifiedFiles()
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
svn.PROPERTY_MIMETYPE, 'image/png')
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
svn.PROPERTY_MIMETYPE, 'application/pdf')
def RaiseUsageException():
raise Exception('%s\nRun with --help for more detail.' % (
USAGE_STRING % __file__))
def Main(options, args):
"""Allow other scripts to call this script with fake command-line args.
"""
# If no platforms are specified, do 'em all.
num_args = len(args)
if num_args == 0:
# TODO(epoger): automate the default set of platforms. We want to ensure
# that the user gets all of the platforms that the bots are running,
# not just whatever subdirectories he happens to have checked out...
args = [
'gm/base-linux',
'gm/base-linux-fixed',
'gm/base-macmini',
'gm/base-macmini-fixed',
'gm/base-macmini-lion-fixed',
'gm/base-macmini-lion-float',
'gm/base-win',
'gm/base-win-fixed',
]
# Trim all subdir names.
baseline_subdirs = []
for arg in args:
baseline_subdirs.append(arg.rstrip(os.sep))
# Make sure all those subdirectories exist.
for baseline_subdir in baseline_subdirs:
if not os.path.isdir(baseline_subdir):
raise Exception('could not find baseline_subdir "%s"' %
baseline_subdir)
# Process the subdirs, one at a time.
for baseline_subdir in baseline_subdirs:
DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir)
if __name__ == '__main__':
parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
parser.add_option(OPTION_IGNORE_LOCAL_MODS,
action='store_true', default=False,
help='allow tool to run even if there are already '
'local modifications in the baseline_subdir')
parser.add_option(OPTION_ADD_NEW_FILES,
action='store_true', default=False,
help='in addition to downloading new versions of '
'existing baselines, also download baselines that are '
'not under SVN control yet')
(options, args) = parser.parse_args()
Main(options, args)