AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add --add-new-files option to download-baselines.py script

Browse Source 
This will allow the user to determine whether the script downloads just
updates for baselines that are already checked in, or all available baselines.

BUG=436
addresses http://code.google.com/p/skia/issues/detail?id=436 ('add "update" flag to download-baselines script')
Review URL: http://codereview.appspot.com/5523050

git-svn-id: http://skia.googlecode.com/svn/trunk@2993 2bbb7eff-a529-9590-31e7-b0007b416f81
This commit is contained in:
epoger@google.com 2012-01-10 14:10:34 +00:00
parent 70ccffe0e7
commit d6256557cd
2 changed files with 70 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
tools/download-baselines.py
View File

@ -8,11 +8,13 @@ steps:
cd .../trunk
svn update
# make sure there are no files awaiting svn commit
tools/download-baselines.py gm/base-macmini-lion-fixed # or other gm/ subdir
python tools/download-baselines.py gm/base-macmini-lion-fixed # or other gm/ subdir
# upload CL for review
# validate that the new images look right
# commit CL
Launch with --help to see more options.
Copyright 2011 Google Inc.
@ -21,6 +23,8 @@ found in the LICENSE file.
'''
# common Python modules
import optparse
import os
import re
import sys
import urllib2
@ -46,6 +50,10 @@ ACTUALS_BY_BASELINE_SUBDIR = {
'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Float_NoDebug/gm/actual',
}
USAGE_STRING = 'usage: %s [options] <baseline_subdir>'
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
OPTION_ADD_NEW_FILES = '--add-new-files'
IMAGE_REGEX = '.+\.png'
IMAGE_MIMETYPE = 'image/png'
@ -83,13 +91,16 @@ def GetLatestResultsUrl(baseline_subdir):
print 'most_recent_result_url is %s' % most_recent_result_url
return most_recent_result_url
def DownloadMatchingFiles(source_url, filename_regex, dest_dir):
def DownloadMatchingFiles(source_url, filename_regex, dest_dir,
only_download_updates=False):
"""Download all files from source_url that match filename_regex, and save
them (with their original filenames) in dest_dir.
@param source_url
@param filename_regex
@param dest_dir
@param filename_regex only download files that match this regex
@param dest_dir where to save the downloaded files
@param only_download_updates if True, only download files that are already
present in dest_dir (download updated versions of those files)
"""
while source_url.endswith('/'):
source_url = source_url[:-1]
@ -98,8 +109,10 @@ def DownloadMatchingFiles(source_url, filename_regex, dest_dir):
link_regex = re.compile('<a href="(%s)">' % filename_regex)
links = link_regex.findall(html)
for link in links:
DownloadBinaryFile('%s/%s' % (source_url, link),
'%s/%s' % (dest_dir, link))
dest_path = os.path.join(dest_dir, link)
if only_download_updates and not os.path.isfile(dest_path):
continue
DownloadBinaryFile('%s/%s' % (source_url, link), dest_path)
def DownloadBinaryFile(source_url, dest_path):
"""Download a single file from its source_url and save it to local disk
@ -114,28 +127,58 @@ def DownloadBinaryFile(source_url, dest_path):
local_fh.write(url_fh.read())
local_fh.close()
def Main(arglist):
def Main(options, args):
"""Download most recently generated baseline images for a given platform,
and add any new ones to SVN control.
@param arglist sys.argv or equivalent
@param options
@param args
"""
num_args = len(arglist)
if num_args != 2:
raise Exception('usage: %s <baseline_subdir>' % __file__)
num_args = len(args)
if num_args != 1:
RaiseUsageException()
baseline_subdir = arglist[1]
baseline_subdir = args[0]
while baseline_subdir.endswith('/'):
baseline_subdir = baseline_subdir[:-1]
results_url = GetLatestResultsUrl(baseline_subdir)
DownloadMatchingFiles(results_url, IMAGE_REGEX, baseline_subdir)
svn_handler = svn.Svn(baseline_subdir)
# If there are any locally modified files in that directory, exit
# (so that we don't risk overwriting the user's previous work).
new_and_modified_files = svn_handler.GetNewAndModifiedFiles()
if not options.ignore_local_mods:
if new_and_modified_files:
raise Exception('Exiting because there are already new and/or '
'modified files in %s. To continue in spite of '
'that, run with %s option.' % (
baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
# Download the actual results from the appropriate buildbot.
results_url = GetLatestResultsUrl(baseline_subdir)
DownloadMatchingFiles(source_url=results_url, filename_regex=IMAGE_REGEX,
dest_dir=baseline_subdir,
only_download_updates=(not options.add_new_files))
# Add any new files to SVN control (if we are running with add_new_files).
new_files = svn_handler.GetNewFiles()
if new_files:
if new_files and options.add_new_files:
svn_handler.AddFiles(new_files)
svn_handler.SetProperty(new_files, svn.PROPERTY_MIMETYPE,
IMAGE_MIMETYPE)
def RaiseUsageException():
raise Exception(USAGE_STRING % __file__)
if __name__ == '__main__':
Main(sys.argv)
parser = optparse.OptionParser(USAGE_STRING % '%prog')
parser.add_option(OPTION_IGNORE_LOCAL_MODS,
action='store_true', default=False,
help='allow tool to run even if there are already '
'local modifications in the baseline_subdir')
parser.add_option(OPTION_ADD_NEW_FILES,
action='store_true', default=False,
help='in addition to downloading new versions of '
'existing baselines, also download baselines that are '
'not under SVN control yet')
(options, args) = parser.parse_args()
Main(options, args)

11
tools/svn.py
View File

@ -39,7 +39,16 @@ class Svn:
SVN control.
"""
stdout = self._RunCommand(['svn', 'status'])
new_regex = re.compile('\? +(.+)')
new_regex = re.compile('^\?.....\s+(.+)$', re.MULTILINE)
files = new_regex.findall(stdout)
return files
def GetNewAndModifiedFiles(self):
"""Return a list of files in this dir which are newly added or modified,
including those that are not (yet) under SVN control.
"""
stdout = self._RunCommand(['svn', 'status'])
new_regex = re.compile('^[AM\?].....\s+(.+)$', re.MULTILINE)
files = new_regex.findall(stdout)
return files