From a7a6f2e01d07a0e01fca948dfca5c5f4df24b5f9 Mon Sep 17 00:00:00 2001 From: borenet Date: Mon, 29 Feb 2016 05:57:31 -0800 Subject: [PATCH] Add test_skia.py, isolates for test_skia, images, skps This enables running DM through Swarming. NOTRY=true BUG=skia:4763 GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1743113003 Review URL: https://codereview.chromium.org/1743113003 --- infra/bots/common.py | 226 +++++++++++++++++++++++++++- infra/bots/compile_skia.isolate | 2 +- infra/bots/compile_skia.py | 14 +- infra/bots/download_images.py | 28 ++++ infra/bots/download_skps.py | 28 ++++ infra/bots/flavor/default_flavor.py | 11 +- infra/bots/images.isolate | 7 + infra/bots/skps.isolate | 7 + infra/bots/test_skia.isolate | 12 ++ infra/bots/test_skia.py | 30 ++++ 10 files changed, 349 insertions(+), 16 deletions(-) create mode 100644 infra/bots/download_images.py create mode 100644 infra/bots/download_skps.py create mode 100644 infra/bots/images.isolate create mode 100644 infra/bots/skps.isolate create mode 100644 infra/bots/test_skia.isolate create mode 100644 infra/bots/test_skia.py diff --git a/infra/bots/common.py b/infra/bots/common.py index 9b96440c4f..6f606d9a73 100644 --- a/infra/bots/common.py +++ b/infra/bots/common.py @@ -6,9 +6,15 @@ # found in the LICENSE file. +import contextlib +import math import os +import shutil +import socket import subprocess import sys +import time +import urllib2 from flavor import android_flavor from flavor import chromeos_flavor @@ -29,15 +35,23 @@ GM_ACTUAL_FILENAME = 'actual-results.json' GM_EXPECTATIONS_FILENAME = 'expected-results.json' GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt' +GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes' + GS_GM_BUCKET = 'chromium-skia-gm' GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries' +GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s' +GS_SUBDIR_TMPL_SKP = 'playback_%s/skps' + SKIA_REPO = 'https://skia.googlesource.com/skia.git' INFRA_REPO = 'https://skia.googlesource.com/buildbot.git' SERVICE_ACCOUNT_FILE = 'service-account-skia.json' SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json' +VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' +VERSION_FILE_SKP = 'SKP_VERSION' + def is_android(bot_cfg): """Determine whether the given bot is an Android bot.""" @@ -66,21 +80,84 @@ def is_xsan(bot_cfg): bot_cfg.get('extra_config') == 'TSAN') +def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir): + # Ensure that the tmp_dir exists. + if not os.path.isdir(tmp_dir): + os.makedirs(tmp_dir) + + # Get the expected version. + with open(os.path.join(skia_dir, version_file)) as f: + expected_version = f.read().rstrip() + + print 'Expected %s = %s' % (version_file, expected_version) + + # Get the actually-downloaded version, if we have one. + actual_version_file = os.path.join(tmp_dir, version_file) + try: + with open(actual_version_file) as f: + actual_version = f.read().rstrip() + except IOError: + actual_version = -1 + + print 'Actual %s = %s' % (version_file, actual_version) + + # If we don't have the desired version, download it. + if actual_version != expected_version: + if actual_version != -1: + os.remove(actual_version_file) + if os.path.isdir(dst_dir): + shutil.rmtree(dst_dir) + os.makedirs(dst_dir) + gs_path = 'gs://%s/%s/*' % (GS_GM_BUCKET, gs_path_tmpl % expected_version) + print 'Downloading from %s' % gs_path + subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir]) + with open(actual_version_file, 'w') as f: + f.write(expected_version) + + +def get_uninteresting_hashes(hashes_file): + retries = 5 + timeout = 60 + wait_base = 15 + + socket.setdefaulttimeout(timeout) + for retry in range(retries): + try: + with contextlib.closing( + urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w: + hashes = w.read() + with open(hashes_file, 'w') as f: + f.write(hashes) + break + except Exception as e: + print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % ( + GOLD_UNINTERESTING_HASHES_URL, e) + if retry == retries: + raise + waittime = wait_base * math.pow(2, retry) + print 'Retry in %d seconds.' % waittime + time.sleep(waittime) + + class BotInfo(object): - def __init__(self, bot_name, slave_name, out_dir): + def __init__(self, bot_name, swarm_out_dir): """Initialize the bot, given its name. Assumes that CWD is the directory containing this file. """ self.name = bot_name - self.slave_name = slave_name self.skia_dir = os.path.abspath(os.path.join( os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir)) + self.swarm_out_dir = swarm_out_dir os.chdir(self.skia_dir) self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir)) - self.out_dir = out_dir self.spec = self.get_bot_spec(bot_name) + self.bot_cfg = self.spec['builder_cfg'] + if self.bot_cfg['role'] == 'Build': + self.out_dir = os.path.join(swarm_out_dir, 'out') + else: + self.out_dir = 'out' self.configuration = self.spec['configuration'] self.default_env = { 'SKIA_OUT': self.out_dir, @@ -89,16 +166,29 @@ class BotInfo(object): } self.default_env.update(self.spec['env']) self.build_targets = [str(t) for t in self.spec['build_targets']] - self.bot_cfg = self.spec['builder_cfg'] self.is_trybot = self.bot_cfg['is_trybot'] self.upload_dm_results = self.spec['upload_dm_results'] self.upload_perf_results = self.spec['upload_perf_results'] + self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata', + self.name, 'data') + self.resource_dir = os.path.join(self.build_dir, 'resources') + self.images_dir = os.path.join(self.build_dir, 'images') + self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps') self.dm_flags = self.spec['dm_flags'] self.nanobench_flags = self.spec['nanobench_flags'] self._ccache = None self._checked_for_ccache = False + self._already_ran = {} + self.tmp_dir = os.path.join(self.build_dir, 'tmp') self.flavor = self.get_flavor(self.bot_cfg) + # These get filled in during subsequent steps. + self.device_dirs = None + self.build_number = None + self.got_revision = None + self.master_name = None + self.slave_name = None + @property def ccache(self): if not self._checked_for_ccache: @@ -148,3 +238,131 @@ class BotInfo(object): print 'ENV: %s' % _env print '============' subprocess.check_call(cmd, env=_env, cwd=cwd) + + def compile_steps(self): + for t in self.build_targets: + self.flavor.compile(t) + + def _run_once(self, fn, *args, **kwargs): + if not fn.__name__ in self._already_ran: + self._already_ran[fn.__name__] = True + fn(*args, **kwargs) + + def install(self): + """Copy the required executables and files to the device.""" + self.device_dirs = self.flavor.get_device_dirs() + + # Run any device-specific installation. + self.flavor.install() + + # TODO(borenet): Only copy files which have changed. + # Resources + self.flavor.copy_directory_contents_to_device(self.resource_dir, + self.device_dirs.resource_dir) + + def _key_params(self): + """Build a unique key from the builder name (as a list). + + E.g. arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6 + """ + # Don't bother to include role, which is always Test. + # TryBots are uploaded elsewhere so they can use the same key. + blacklist = ['role', 'is_trybot'] + + flat = [] + for k in sorted(self.bot_cfg.keys()): + if k not in blacklist: + flat.append(k) + flat.append(self.bot_cfg[k]) + return flat + + def test_steps(self, got_revision, master_name, slave_name, build_number): + """Run the DM test.""" + self.build_number = build_number + self.got_revision = got_revision + self.master_name = master_name + self.slave_name = slave_name + self._run_once(self.install) + + use_hash_file = False + if self.upload_dm_results: + # This must run before we write anything into self.device_dirs.dm_dir + # or we may end up deleting our output on machines where they're the same. + host_dm_dir = os.path.join(self.swarm_out_dir, 'dm') + print 'host dm dir: %s' % host_dm_dir + self.flavor.create_clean_host_dir(host_dm_dir) + if str(host_dm_dir) != str(self.device_dirs.dm_dir): + self.flavor.create_clean_device_dir(self.device_dirs.dm_dir) + + # Obtain the list of already-generated hashes. + hash_filename = 'uninteresting_hashes.txt' + host_hashes_file = self.tmp_dir.join(hash_filename) + hashes_file = self.flavor.device_path_join( + self.device_dirs.tmp_dir, hash_filename) + + try: + get_uninteresting_hashes(host_hashes_file) + except Exception: + pass + + if os.path.exists(host_hashes_file): + self.flavor.copy_file_to_device(host_hashes_file, hashes_file) + use_hash_file = True + + # Run DM. + properties = [ + 'gitHash', self.got_revision, + 'master', self.master_name, + 'builder', self.name, + 'build_number', self.build_number, + ] + if self.is_trybot: + properties.extend([ + 'issue', self.m.properties['issue'], + 'patchset', self.m.properties['patchset'], + ]) + + args = [ + 'dm', + '--undefok', # This helps branches that may not know new flags. + '--verbose', + '--resourcePath', self.device_dirs.resource_dir, + '--skps', self.device_dirs.skp_dir, + '--images', self.flavor.device_path_join( + self.device_dirs.images_dir, 'dm'), + '--nameByHash', + '--properties' + ] + properties + + args.append('--key') + args.extend(self._key_params()) + if use_hash_file: + args.extend(['--uninterestingHashesFile', hashes_file]) + if self.upload_dm_results: + args.extend(['--writePath', self.device_dirs.dm_dir]) + + skip_flag = None + if self.bot_cfg.get('cpu_or_gpu') == 'CPU': + skip_flag = '--nogpu' + elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': + skip_flag = '--nocpu' + if skip_flag: + args.append(skip_flag) + args.extend(self.dm_flags) + + self.flavor.run(args, env=self.default_env) + + if self.upload_dm_results: + # Copy images and JSON to host machine if needed. + self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir, + host_dm_dir) + + # See skia:2789. + if ('Valgrind' in self.name and + self.builder_cfg.get('cpu_or_gpu') == 'GPU'): + abandonGpuContext = list(args) + abandonGpuContext.append('--abandonGpuContext') + self.flavor.run(abandonGpuContext) + preAbandonGpuContext = list(args) + preAbandonGpuContext.append('--preAbandonGpuContext') + self.flavor.run(preAbandonGpuContext) diff --git a/infra/bots/compile_skia.isolate b/infra/bots/compile_skia.isolate index 51168e0119..5866f4c797 100644 --- a/infra/bots/compile_skia.isolate +++ b/infra/bots/compile_skia.isolate @@ -4,7 +4,7 @@ ], 'variables': { 'command': [ - 'python', 'compile_skia.py', '<(BUILDER_NAME)', '${ISOLATED_OUTDIR}/out', + 'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}/out', ], }, } diff --git a/infra/bots/compile_skia.py b/infra/bots/compile_skia.py index b3b625121f..ca2c7db600 100644 --- a/infra/bots/compile_skia.py +++ b/infra/bots/compile_skia.py @@ -6,17 +6,19 @@ # found in the LICENSE file. +import argparse import common +import os import sys def main(): - if len(sys.argv) != 3: - print >> sys.stderr, 'Usage: compile_skia.py ' - sys.exit(1) - bot = common.BotInfo(sys.argv[1], 'fake-slave', sys.argv[2]) - for t in bot.build_targets: - bot.flavor.compile(t) + parser = argparse.ArgumentParser() + parser.add_argument('--builder_name', required=True) + parser.add_argument('--swarm_out_dir', required=True) + args = parser.parse_args() + bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir)) + bot.compile_steps() if __name__ == '__main__': diff --git a/infra/bots/download_images.py b/infra/bots/download_images.py new file mode 100644 index 0000000000..4342a9f1a6 --- /dev/null +++ b/infra/bots/download_images.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import common +import os +import sys + + +def main(): + if len(sys.argv) != 1: + print >> sys.stderr, 'Usage: download_images.py' + sys.exit(1) + skia_dir = os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir, os.pardir)) + dst_dir = os.path.join(skia_dir, os.pardir, 'images') + tmp_dir = os.path.join(skia_dir, os.pardir, 'tmp') + common.download_dir(skia_dir, tmp_dir, common.VERSION_FILE_SK_IMAGE, + common.GS_SUBDIR_TMPL_SK_IMAGE, dst_dir) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/download_skps.py b/infra/bots/download_skps.py new file mode 100644 index 0000000000..45b5de8831 --- /dev/null +++ b/infra/bots/download_skps.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import common +import os +import sys + + +def main(): + if len(sys.argv) != 1: + print >> sys.stderr, 'Usage: download_skps.py' + sys.exit(1) + skia_dir = os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir, os.pardir)) + dst_dir = os.path.join(skia_dir, os.pardir, 'skps') + tmp_dir = os.path.join(skia_dir, os.pardir, 'tmp') + common.download_dir(skia_dir, tmp_dir, common.VERSION_FILE_SKP, + common.GS_SUBDIR_TMPL_SKP, dst_dir) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/flavor/default_flavor.py b/infra/bots/flavor/default_flavor.py index 5263073744..3603f043aa 100644 --- a/infra/bots/flavor/default_flavor.py +++ b/infra/bots/flavor/default_flavor.py @@ -71,10 +71,10 @@ class DefaultFlavorUtils(object): self._bot_info = bot_info self.chrome_path = os.path.join(os.path.expanduser('~'), 'src') - def step(self, cmd, **kwargs): + def run(self, cmd, **kwargs): """Runs a step as appropriate for this flavor.""" - path_to_app = self._bot_info.out_dir.join( - self._bot_info.configuration, cmd[0]) + path_to_app = os.path.join(self._bot_info.out_dir, + self._bot_info.configuration, cmd[0]) if (sys.platform == 'linux' and 'x86_64' in self._bot_info.bot_name and not 'TSAN' in self._bot_info.bot_name): @@ -141,7 +141,8 @@ class DefaultFlavorUtils(object): def create_clean_host_dir(self, path): """Convenience function for creating a clean directory.""" - shutil.rmtree(path) + if os.path.exists(path): + shutil.rmtree(path) os.makedirs(path) def install(self): @@ -161,7 +162,7 @@ class DefaultFlavorUtils(object): """ join = lambda p: os.path.join(self._bot_info.build_dir, p) return DeviceDirs( - dm_dir=join('dm'), + dm_dir=os.path.join(self._bot_info.swarm_out_dir, 'dm'), perf_data_dir=self._bot_info.perf_data_dir, resource_dir=self._bot_info.resource_dir, images_dir=join('images'), diff --git a/infra/bots/images.isolate b/infra/bots/images.isolate new file mode 100644 index 0000000000..b93d95d76a --- /dev/null +++ b/infra/bots/images.isolate @@ -0,0 +1,7 @@ +{ + 'variables': { + 'files': [ + '../../../images/', + ], + }, +} diff --git a/infra/bots/skps.isolate b/infra/bots/skps.isolate new file mode 100644 index 0000000000..15b2b1a5e8 --- /dev/null +++ b/infra/bots/skps.isolate @@ -0,0 +1,7 @@ +{ + 'variables': { + 'files': [ + '../../../skps/', + ], + }, +} diff --git a/infra/bots/test_skia.isolate b/infra/bots/test_skia.isolate new file mode 100644 index 0000000000..d9eab8cfd9 --- /dev/null +++ b/infra/bots/test_skia.isolate @@ -0,0 +1,12 @@ +{ + 'includes': [ + 'images.isolate', + 'skia_repo.isolate', + 'skps.isolate', + ], + 'variables': { + 'command': [ + 'python', 'test_skia.py', '--master_name', '<(MASTER_NAME)', '--builder_name', '<(BUILDER_NAME)', '--build_number', '<(BUILD_NUMBER)', '--slave_name', '<(SLAVE_NAME)', '--revision', '<(REVISION)', '--swarm_out_dir', '${ISOLATED_OUTDIR}', + ], + }, +} diff --git a/infra/bots/test_skia.py b/infra/bots/test_skia.py new file mode 100644 index 0000000000..59992edd0e --- /dev/null +++ b/infra/bots/test_skia.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import argparse +import common +import os +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--master_name', required=True) + parser.add_argument('--builder_name', required=True) + parser.add_argument('--build_number', required=True) + parser.add_argument('--slave_name', required=True) + parser.add_argument('--revision', required=True) + parser.add_argument('--swarm_out_dir', required=True) + args = parser.parse_args() + bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir)) + bot.test_steps(args.revision, args.master_name, args.slave_name, + args.build_number) + + +if __name__ == '__main__': + main()