AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Port Skia recipe to normal Python scripts, move to Skia repo

Browse Source 
BUG=skia:4763
GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1703663002

Review URL: https://codereview.chromium.org/1703663002
This commit is contained in:
borenet 2016-02-18 08:05:48 -08:00 committed by Commit bot
parent e1fce93f36
commit d9fa758292
18 changed files with 1275 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
PRESUBMIT.py
View File

@ -85,6 +85,7 @@ def _PythonChecks(input_api, output_api):
'R0201', # Method could be a function.
'E1003', # Using class name in super.
'W0613', # Unused argument.
'W0105', # String statement has no effect.
)
# Run Pylint on only the modified python files. Unfortunately it still runs
# Pylint on the whole file instead of just the modified lines.

18
infra/bots/README.md Normal file
View File

@ -0,0 +1,18 @@
Skia Buildbot Scripts
=====================
The scripts in this directory are ported from Skia's buildbot recipes and are
intended to run as standalone Python scripts either locally or via Swarming.
How to Run
----------
The scripts can be run by hand, eg:
$ cd infra/bots
$ python compile_skia.py Build-Ubuntu-GCC-x86_64-Debug ../../out
Or, you can run the scripts via Swarming:
$ isolate archive --isolate-server https://isolateserver.appspot.com/ -i infra/bots/compile_skia.isolate -s ../compile-skia.isolated --verbose --config-variable BUILDER_NAME=Build-Ubuntu-GCC-x86_64-Debug
$ swarming.py run --swarming https://chromium-swarm.appspot.com --isolate-server https://isolateserver.appspot.com --dimension os Ubuntu --dimension pool Skia --task-name compile-skia --io-timeout=3600 --hard-timeout=3600 ../compile-skia.isolated

150
infra/bots/common.py Normal file
View File

@ -0,0 +1,150 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import subprocess
import sys
from flavor import android_flavor
from flavor import chromeos_flavor
from flavor import cmake_flavor
from flavor import coverage_flavor
from flavor import default_flavor
from flavor import ios_flavor
from flavor import valgrind_flavor
from flavor import xsan_flavor
CONFIG_COVERAGE = 'Coverage'
CONFIG_DEBUG = 'Debug'
CONFIG_RELEASE = 'Release'
VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE)
GM_ACTUAL_FILENAME = 'actual-results.json'
GM_EXPECTATIONS_FILENAME = 'expected-results.json'
GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt'
GS_GM_BUCKET = 'chromium-skia-gm'
GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries'
SKIA_REPO = 'https://skia.googlesource.com/skia.git'
INFRA_REPO = 'https://skia.googlesource.com/buildbot.git'
SERVICE_ACCOUNT_FILE = 'service-account-skia.json'
SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json'
def is_android(bot_cfg):
"""Determine whether the given bot is an Android bot."""
return ('Android' in bot_cfg.get('extra_config', '') or
bot_cfg.get('os') == 'Android')
def is_chromeos(bot_cfg):
return ('CrOS' in bot_cfg.get('extra_config', '') or
bot_cfg.get('os') == 'ChromeOS')
def is_cmake(bot_cfg):
return 'CMake' in bot_cfg.get('extra_config', '')
def is_ios(bot_cfg):
return ('iOS' in bot_cfg.get('extra_config', '') or
bot_cfg.get('os') == 'iOS')
def is_valgrind(bot_cfg):
return 'Valgrind' in bot_cfg.get('extra_config', '')
def is_xsan(bot_cfg):
return (bot_cfg.get('extra_config') == 'ASAN' or
bot_cfg.get('extra_config') == 'MSAN' or
bot_cfg.get('extra_config') == 'TSAN')
class BotInfo(object):
def __init__(self, bot_name, slave_name, out_dir):
"""Initialize the bot, given its name.
Assumes that CWD is the directory containing this file.
"""
self.name = bot_name
self.slave_name = slave_name
self.skia_dir = os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)),
os.pardir, os.pardir))
os.chdir(self.skia_dir)
self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir))
self.out_dir = out_dir
self.spec = self.get_bot_spec(bot_name)
self.configuration = self.spec['configuration']
self.default_env = {
'SKIA_OUT': self.out_dir,
'BUILDTYPE': self.configuration,
'PATH': os.environ['PATH'],
}
self.default_env.update(self.spec['env'])
self.build_targets = [str(t) for t in self.spec['build_targets']]
self.bot_cfg = self.spec['builder_cfg']
self.is_trybot = self.bot_cfg['is_trybot']
self.upload_dm_results = self.spec['upload_dm_results']
self.upload_perf_results = self.spec['upload_perf_results']
self.dm_flags = self.spec['dm_flags']
self.nanobench_flags = self.spec['nanobench_flags']
self._ccache = None
self._checked_for_ccache = False
self.flavor = self.get_flavor(self.bot_cfg)
@property
def ccache(self):
if not self._checked_for_ccache:
self._checked_for_ccache = True
if sys.platform != 'win32':
try:
result = subprocess.check_output(['which', 'ccache'])
self._ccache = result.rstrip()
except subprocess.CalledProcessError:
pass
return self._ccache
def get_bot_spec(self, bot_name):
"""Retrieve the bot spec for this bot."""
sys.path.append(self.skia_dir)
from tools import buildbot_spec
return buildbot_spec.get_builder_spec(bot_name)
def get_flavor(self, bot_cfg):
"""Return a flavor utils object specific to the given bot."""
if is_android(bot_cfg):
return android_flavor.AndroidFlavorUtils(self)
elif is_chromeos(bot_cfg):
return chromeos_flavor.ChromeOSFlavorUtils(self)
elif is_cmake(bot_cfg):
return cmake_flavor.CMakeFlavorUtils(self)
elif is_ios(bot_cfg):
return ios_flavor.iOSFlavorUtils(self)
elif is_valgrind(bot_cfg):
return valgrind_flavor.ValgrindFlavorUtils(self)
elif is_xsan(bot_cfg):
return xsan_flavor.XSanFlavorUtils(self)
elif bot_cfg.get('configuration') == CONFIG_COVERAGE:
return coverage_flavor.CoverageFlavorUtils(self)
else:
return default_flavor.DefaultFlavorUtils(self)
def run(self, cmd, env=None, cwd=None):
_env = {}
_env.update(self.default_env)
_env.update(env or {})
cwd = cwd or self.skia_dir
print '============'
print 'CMD: %s' % cmd
print 'CWD: %s' % cwd
print 'ENV: %s' % _env
print '============'
subprocess.check_call(cmd, env=_env, cwd=cwd)

10
infra/bots/compile_skia.isolate Normal file
View File

@ -0,0 +1,10 @@
{
'includes': [
'skia_repo.isolate',
],
'variables': {
'command': [
'python', 'compile_skia.py', '<(BUILDER_NAME)', '${ISOLATED_OUTDIR}/out',
],
},
}

23
infra/bots/compile_skia.py Normal file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import common
import sys
def main():
if len(sys.argv) != 3:
print >> sys.stderr, 'Usage: compile_skia.py <builder name> <out-dir>'
sys.exit(1)
bot = common.BotInfo(sys.argv[1], 'fake-slave', sys.argv[2])
for t in bot.build_targets:
bot.flavor.compile(t)
if __name__ == '__main__':
main()

6
infra/bots/flavor/__init__.py Normal file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

100
infra/bots/flavor/android_devices.py Normal file
View File

@ -0,0 +1,100 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import json
DEFAULT_SDK_ROOT = '/home/chrome-bot/android-sdk-linux'
MAC_SDK_ROOT = '/Users/chrome-bot/adt-bundle-mac-x86_64-20140702/sdk'
MACMINI_SDK_ROOT = '/Users/chrome-bot/android-sdk-macosx'
SlaveInfo = collections.namedtuple('SlaveInfo',
'serial android_sdk_root has_root')
SLAVE_INFO = {
'skiabot-mac-10_8-compile-000':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-001':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-002':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-003':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-004':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-005':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-006':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-007':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-008':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-mac-10_8-compile-009':
SlaveInfo('noserial', MAC_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-androidone-001':
SlaveInfo('AG86044202A04GC', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-androidone-002':
SlaveInfo('AG8404EC06G02GC', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-androidone-003':
SlaveInfo('AG8404EC0688EGC', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-galaxys3-001':
SlaveInfo('4df713b8244a21cf', DEFAULT_SDK_ROOT, False),
'skiabot-shuttle-ubuntu12-galaxys3-002':
SlaveInfo('32309a56e9b3a09f', DEFAULT_SDK_ROOT, False),
'skiabot-shuttle-ubuntu12-galaxys4-001':
SlaveInfo('4d0032a5d8cb6125', MACMINI_SDK_ROOT, False),
'skiabot-shuttle-ubuntu12-galaxys4-002':
SlaveInfo('4d00353cd8ed61c3', MACMINI_SDK_ROOT, False),
'skiabot-shuttle-ubuntu12-nexus5-001':
SlaveInfo('03f61449437cc47b', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus5-002':
SlaveInfo('018dff3520c970f6', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-nexus6-001':
SlaveInfo('ZX1G22JJWS', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-nexus6-002':
SlaveInfo('ZX1G22JN35', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-nexus6-003':
SlaveInfo('ZX1G22JXXM', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus7-001':
SlaveInfo('015d210a13480604', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus7-002':
SlaveInfo('015d18848c280217', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus7-003':
SlaveInfo('015d16897c401e17', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus9-001':
SlaveInfo('HT43RJT00022', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus9-002':
SlaveInfo('HT4AEJT03112', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus9-003':
SlaveInfo('HT4ADJT03339', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus10-001':
SlaveInfo('R32C801B5LH', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexus10-003':
SlaveInfo('R32CB017X2L', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexusplayer-001':
SlaveInfo('D76C708B', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu12-nexusplayer-002':
SlaveInfo('8AB5139A', DEFAULT_SDK_ROOT, True),
'skiabot-shuttle-ubuntu15-nvidia-shield-001':
SlaveInfo('04217150066510000078', MACMINI_SDK_ROOT, False),
'skiabot-linux-housekeeper-003':
SlaveInfo('noserial', DEFAULT_SDK_ROOT, False),
'vm690-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
'vm691-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
'vm692-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
'vm693-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
'default':
SlaveInfo('noserial', DEFAULT_SDK_ROOT, False),
}
if __name__ == '__main__':
print json.dumps(SLAVE_INFO) # pragma: no cover

241
infra/bots/flavor/android_flavor.py Normal file
View File

@ -0,0 +1,241 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import android_devices
import default_flavor
import os
"""Android flavor utils, used for building for and running tests on Android."""
class _ADBWrapper(object):
"""Wrapper for ADB."""
def __init__(self, path_to_adb, serial, android_flavor):
self._adb = path_to_adb
self._serial = serial
self._wait_count = 0
self._android_flavor = android_flavor
def wait_for_device(self):
"""Run 'adb wait-for-device'."""
self._wait_count += 1
cmd = [
os.path.join(self._android_flavor.android_bin, 'adb_wait_for_device'),
'-s', self._serial,
]
self._android_flavor._bot_info.run(
cmd, env=self._android_flavor._default_env)
def maybe_wait_for_device(self):
"""Run 'adb wait-for-device' if it hasn't already been run."""
if self._wait_count == 0:
self.wait_for_device()
def __call__(self, *args, **kwargs):
self.maybe_wait_for_device()
return self._android_flavor._bot_info.run(self._adb + args, **kwargs)
class AndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
def __init__(self, skia_api):
super(AndroidFlavorUtils, self).__init__(skia_api)
self.device = self._bot_info.spec['device_cfg']
slave_info = android_devices.SLAVE_INFO.get(
self._bot_info.slave_name,
android_devices.SLAVE_INFO['default'])
self.serial = slave_info.serial
self.android_bin = os.path.join(
self._bot_info.skia_dir, 'platform_tools', 'android', 'bin')
self._android_sdk_root = slave_info.android_sdk_root
self._adb = _ADBWrapper(
os.path.join(self._android_sdk_root, 'platform-tools', 'adb'),
self.serial,
self)
self._has_root = slave_info.has_root
self._default_env = {'ANDROID_SDK_ROOT': self._android_sdk_root,
'ANDROID_HOME': self._android_sdk_root,
'SKIA_ANDROID_VERBOSE_SETUP': '1'}
def step(self, name, cmd, env=None, **kwargs):
self._adb.maybe_wait_for_device()
args = [self.android_bin.join('android_run_skia'),
'--verbose',
'--logcat',
'-d', self.device,
'-s', self.serial,
'-t', self._bot_info.configuration,
]
env = dict(env or {})
env.update(self._default_env)
return self._bot_info.run(self._bot_info.m.step, name=name, cmd=args + cmd,
env=env, **kwargs)
def compile(self, target):
"""Build the given target."""
env = dict(self._default_env)
ccache = self._bot_info.ccache
if ccache:
env['ANDROID_MAKE_CCACHE'] = ccache
cmd = [os.path.join(self.android_bin, 'android_ninja'), target,
'-d', self.device]
if 'Clang' in self._bot_info.name:
cmd.append('--clang')
self._bot_info.run(cmd, env=env)
def device_path_join(self, *args):
"""Like os.path.join(), but for paths on a connected Android device."""
return '/'.join(args)
def device_path_exists(self, path):
"""Like os.path.exists(), but for paths on a connected device."""
exists_str = 'FILE_EXISTS'
return exists_str in self._adb(
name='exists %s' % self._bot_info.m.path.basename(path),
serial=self.serial,
cmd=['shell', 'if', '[', '-e', path, '];',
'then', 'echo', exists_str + ';', 'fi'],
stdout=self._bot_info.m.raw_io.output(),
infra_step=True
).stdout
def _remove_device_dir(self, path):
"""Remove the directory on the device."""
self._adb(name='rmdir %s' % self._bot_info.m.path.basename(path),
serial=self.serial,
cmd=['shell', 'rm', '-r', path],
infra_step=True)
# Sometimes the removal fails silently. Verify that it worked.
if self.device_path_exists(path):
raise Exception('Failed to remove %s!' % path) # pragma: no cover
def _create_device_dir(self, path):
"""Create the directory on the device."""
self._adb(name='mkdir %s' % self._bot_info.m.path.basename(path),
serial=self.serial,
cmd=['shell', 'mkdir', '-p', path],
infra_step=True)
def copy_directory_contents_to_device(self, host_dir, device_dir):
"""Like shutil.copytree(), but for copying to a connected device."""
self._bot_info.run(
self._bot_info.m.step,
name='push %s' % self._bot_info.m.path.basename(host_dir),
cmd=[self.android_bin.join('adb_push_if_needed'), '--verbose',
'-s', self.serial, host_dir, device_dir],
env=self._default_env,
infra_step=True)
def copy_directory_contents_to_host(self, device_dir, host_dir):
"""Like shutil.copytree(), but for copying from a connected device."""
self._bot_info.run(
self._bot_info.m.step,
name='pull %s' % self._bot_info.m.path.basename(device_dir),
cmd=[self.android_bin.join('adb_pull_if_needed'), '--verbose',
'-s', self.serial, device_dir, host_dir],
env=self._default_env,
infra_step=True)
def copy_file_to_device(self, host_path, device_path):
"""Like shutil.copyfile, but for copying to a connected device."""
self._adb(name='push %s' % self._bot_info.m.path.basename(host_path),
serial=self.serial,
cmd=['push', host_path, device_path],
infra_step=True)
def create_clean_device_dir(self, path):
"""Like shutil.rmtree() + os.makedirs(), but on a connected device."""
self._remove_device_dir(path)
self._create_device_dir(path)
def install(self):
"""Run device-specific installation steps."""
if self._has_root:
self._adb(name='adb root',
serial=self.serial,
cmd=['root'],
infra_step=True)
# Wait for the device to reconnect.
self._bot_info.run(
self._bot_info.m.step,
name='wait',
cmd=['sleep', '10'],
infra_step=True)
self._adb.wait_for_device()
# TODO(borenet): Set CPU scaling mode to 'performance'.
self._bot_info.run(self._bot_info.m.step,
name='kill skia',
cmd=[self.android_bin.join('android_kill_skia'),
'--verbose', '-s', self.serial],
env=self._default_env,
infra_step=True)
if self._has_root:
self._adb(name='stop shell',
serial=self.serial,
cmd=['shell', 'stop'],
infra_step=True)
# Print out battery stats.
self._adb(name='starting battery stats',
serial=self.serial,
cmd=['shell', 'dumpsys', 'batteryproperties'],
infra_step=True)
def cleanup_steps(self):
"""Run any device-specific cleanup steps."""
self._adb(name='final battery stats',
serial=self.serial,
cmd=['shell', 'dumpsys', 'batteryproperties'],
infra_step=True)
self._adb(name='reboot',
serial=self.serial,
cmd=['reboot'],
infra_step=True)
self._bot_info.run(
self._bot_info.m.step,
name='wait for reboot',
cmd=['sleep', '10'],
infra_step=True)
self._adb.wait_for_device()
def read_file_on_device(self, path, *args, **kwargs):
"""Read the given file."""
return self._adb(name='read %s' % self._bot_info.m.path.basename(path),
serial=self.serial,
cmd=['shell', 'cat', path],
stdout=self._bot_info.m.raw_io.output(),
infra_step=True).stdout.rstrip()
def remove_file_on_device(self, path, *args, **kwargs):
"""Delete the given file."""
return self._adb(name='rm %s' % self._bot_info.m.path.basename(path),
serial=self.serial,
cmd=['shell', 'rm', '-f', path],
infra_step=True,
*args,
**kwargs)
def get_device_dirs(self):
""" Set the directories which will be used by the build steps."""
device_scratch_dir = self._adb(
name='get EXTERNAL_STORAGE dir',
serial=self.serial,
cmd=['shell', 'echo', '$EXTERNAL_STORAGE'],
)
prefix = self.device_path_join(device_scratch_dir, 'skiabot', 'skia_')
return default_flavor.DeviceDirs(
dm_dir=prefix + 'dm',
perf_data_dir=prefix + 'perf',
resource_dir=prefix + 'resources',
images_dir=prefix + 'images',
skp_dir=prefix + 'skp/skps',
tmp_dir=prefix + 'tmp_dir')

59
infra/bots/flavor/chromeos_flavor.py Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
import ssh_flavor
"""Utils for building for and running tests on ChromeOS."""
class ChromeOSFlavorUtils(ssh_flavor.SSHFlavorUtils):
def __init__(self, bot_info):
super(ChromeOSFlavorUtils, self).__init__(bot_info)
self.board = self._bot_info.spec['device_cfg']
self.device_root_dir = '/usr/local/skiabot'
self.device_bin_dir = self.device_path_join(self.device_root_dir, 'bin')
def step(self, name, cmd, **kwargs):
"""Wrapper for the Step API; runs a step as appropriate for this flavor."""
local_path = self._bot_info.out_dir.join(
'config', 'chromeos-%s' % self.board,
self._bot_info.configuration, cmd[0])
remote_path = self.device_path_join(self.device_bin_dir, cmd[0])
self.copy_file_to_device(local_path, remote_path)
super(ChromeOSFlavorUtils, self).step(name=name,
cmd=[remote_path]+cmd[1:],
**kwargs)
def compile(self, target):
"""Build the given target."""
cmd = [os.path.join(self._bot_info.skia_dir, 'platform_tools', 'chromeos',
'bin', 'chromeos_make'),
'-d', self.board,
target]
self._bot_info.run(cmd)
def install(self):
"""Run any device-specific installation steps."""
self.create_clean_device_dir(self.device_bin_dir)
def get_device_dirs(self):
""" Set the directories which will be used by the build steps."""
prefix = self.device_path_join(self.device_root_dir, 'skia_')
def join(suffix):
return ''.join((prefix, suffix))
return default_flavor.DeviceDirs(
dm_dir=join('dm_out'), # 'dm' conflicts with the binary
perf_data_dir=join('perf'),
resource_dir=join('resources'),
images_dir=join('images'),
skp_dir=self.device_path_join(join('skp'), 'skps'),
tmp_dir=join('tmp_dir'))

20
infra/bots/flavor/cmake_flavor.py Normal file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
"""CMake flavor utils, used for building Skia with CMake."""
class CMakeFlavorUtils(default_flavor.DefaultFlavorUtils):
def compile(self, target):
"""Build Skia with CMake. Ignores `target`."""
cmake_build = os.path.join(self._bot_info.skia_dir, 'cmake', 'cmake_build')
self._bot_info.run([cmake_build, target])

114
infra/bots/flavor/coverage_flavor.py Normal file
View File

@ -0,0 +1,114 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
import subprocess
import time
"""Utils for running coverage tests."""
class CoverageFlavorUtils(default_flavor.DefaultFlavorUtils):
def compile(self, target):
"""Build the given target."""
cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
'llvm_coverage_build'),
target]
self._bot_info.run(cmd)
def step(self, cmd, **kwargs):
"""Run the given step through coverage."""
# Slice out the 'key' and 'properties' arguments to be reused.
key = []
properties = []
current = None
for i in xrange(0, len(cmd)):
if isinstance(cmd[i], basestring) and cmd[i] == '--key':
current = key
elif isinstance(cmd[i], basestring) and cmd[i] == '--properties':
current = properties
elif isinstance(cmd[i], basestring) and cmd[i].startswith('--'):
current = None
if current is not None:
current.append(cmd[i])
results_dir = self._bot_info.out_dir.join('coverage_results')
self.create_clean_host_dir(results_dir)
# Run DM under coverage.
report_file_basename = '%s.cov' % self._bot_info.got_revision
report_file = os.path.join(results_dir, report_file_basename)
args = [
'python',
os.path.join(self._bot_info.skia_dir, 'tools', 'llvm_coverage_run.py'),
] + cmd + ['--outResultsFile', report_file]
self._bot_info.run(args, **kwargs)
# Generate nanobench-style JSON output from the coverage report.
git_timestamp = subprocess.check_output(['git', 'log', '-n1',
self._bot_info.got_revision, '--format=%%ci']).rstrip()
nanobench_json = results_dir.join('nanobench_%s_%s.json' % (
self._bot_info.got_revision, git_timestamp))
line_by_line_basename = ('coverage_by_line_%s_%s.json' % (
self._bot_info.got_revision, git_timestamp))
line_by_line = results_dir.join(line_by_line_basename)
args = [
'python',
os.path.join(self._bot_info.skia_dir, 'tools',
'parse_llvm_coverage.py'),
'--report', report_file, '--nanobench', nanobench_json,
'--linebyline', line_by_line]
args.extend(key)
args.extend(properties)
self._bot_info.run(args)
# Upload raw coverage data.
now = time.utcnow()
gs_json_path = '/'.join((
str(now.year).zfill(4), str(now.month).zfill(2),
str(now.day).zfill(2), str(now.hour).zfill(2),
self._bot_info.name,
str(self._bot_info.build_number)))
if self._bot_info.is_trybot:
gs_json_path = '/'.join(('trybot', gs_json_path,
str(self._bot_info.issue)))
self._bot_info.gsutil_upload(
'upload raw coverage data',
source=report_file,
bucket='skia-infra',
dest='/'.join(('coverage-raw-v1', gs_json_path, report_file_basename)))
# Upload nanobench JSON data.
gsutil_path = self._bot_info.m.path['depot_tools'].join(
'third_party', 'gsutil', 'gsutil')
upload_args = [self._bot_info.name,
self._bot_info.m.properties['buildnumber'],
results_dir,
self._bot_info.got_revision, gsutil_path]
if self._bot_info.is_trybot:
upload_args.append(self._bot_info.m.properties['issue'])
self._bot_info.run(
self._bot_info.m.python,
'upload nanobench coverage results',
script=self._bot_info.resource('upload_bench_results.py'),
args=upload_args,
cwd=self._bot_info.m.path['checkout'],
abort_on_failure=False,
infra_step=True)
# Upload line-by-line coverage data.
self._bot_info.gsutil_upload(
'upload line-by-line coverage data',
source=line_by_line,
bucket='skia-infra',
dest='/'.join(('coverage-json-v1', gs_json_path,
line_by_line_basename)))

172
infra/bots/flavor/default_flavor.py Normal file
View File

@ -0,0 +1,172 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Default flavor utils class, used for desktop bots."""
import os
import shutil
import sys
class DeviceDirs(object):
def __init__(self,
dm_dir,
perf_data_dir,
resource_dir,
images_dir,
skp_dir,
tmp_dir):
self._dm_dir = dm_dir
self._perf_data_dir = perf_data_dir
self._resource_dir = resource_dir
self._images_dir = images_dir
self._skp_dir = skp_dir
self._tmp_dir = tmp_dir
@property
def dm_dir(self):
"""Where DM writes."""
return self._dm_dir
@property
def perf_data_dir(self):
return self._perf_data_dir
@property
def resource_dir(self):
return self._resource_dir
@property
def images_dir(self):
return self._images_dir
@property
def skp_dir(self):
return self._skp_dir
@property
def tmp_dir(self):
return self._tmp_dir
class DefaultFlavorUtils(object):
"""Utilities to be used by build steps.
The methods in this class define how certain high-level functions should
work. Each build step flavor should correspond to a subclass of
DefaultFlavorUtils which may override any of these functions as appropriate
for that flavor.
For example, the AndroidFlavorUtils will override the functions for
copying files between the host and Android device, as well as the
'step' function, so that commands may be run through ADB.
"""
def __init__(self, bot_info, *args, **kwargs):
self._bot_info = bot_info
self.chrome_path = os.path.join(os.path.expanduser('~'), 'src')
def step(self, cmd, **kwargs):
"""Runs a step as appropriate for this flavor."""
path_to_app = self._bot_info.out_dir.join(
self._bot_info.configuration, cmd[0])
if (sys.platform == 'linux' and
'x86_64' in self._bot_info.bot_name and
not 'TSAN' in self._bot_info.bot_name):
new_cmd = ['catchsegv', path_to_app]
else:
new_cmd = [path_to_app]
new_cmd.extend(cmd[1:])
return self._bot_info.run(new_cmd, **kwargs)
def compile(self, target):
"""Build the given target."""
# The CHROME_PATH environment variable is needed for bots that use
# toolchains downloaded by Chrome.
env = {'CHROME_PATH': self.chrome_path}
if sys.platform == 'win32':
make_cmd = ['python', 'make.py']
else:
make_cmd = ['make']
cmd = make_cmd + [target]
self._bot_info.run(cmd, env=env)
def device_path_join(self, *args):
"""Like os.path.join(), but for paths on a connected device."""
return os.path.join(*args)
def device_path_exists(self, path):
"""Like os.path.exists(), but for paths on a connected device."""
return os.path.exists(path, infra_step=True) # pragma: no cover
def copy_directory_contents_to_device(self, host_dir, device_dir):
"""Like shutil.copytree(), but for copying to a connected device."""
# For "normal" bots who don't have an attached device, we expect
# host_dir and device_dir to be the same.
if str(host_dir) != str(device_dir):
raise ValueError('For bots who do not have attached devices, copying '
'from host to device is undefined and only allowed if '
'host_path and device_path are the same (%s vs %s).' % (
str(host_dir), str(device_dir))) # pragma: no cover
def copy_directory_contents_to_host(self, device_dir, host_dir):
"""Like shutil.copytree(), but for copying from a connected device."""
# For "normal" bots who don't have an attached device, we expect
# host_dir and device_dir to be the same.
if str(host_dir) != str(device_dir):
raise ValueError('For bots who do not have attached devices, copying '
'from device to host is undefined and only allowed if '
'host_path and device_path are the same (%s vs %s).' % (
str(host_dir), str(device_dir))) # pragma: no cover
def copy_file_to_device(self, host_path, device_path):
"""Like shutil.copyfile, but for copying to a connected device."""
# For "normal" bots who don't have an attached device, we expect
# host_dir and device_dir to be the same.
if str(host_path) != str(device_path): # pragma: no cover
raise ValueError('For bots who do not have attached devices, copying '
'from host to device is undefined and only allowed if '
'host_path and device_path are the same (%s vs %s).' % (
str(host_path), str(device_path)))
def create_clean_device_dir(self, path):
"""Like shutil.rmtree() + os.makedirs(), but on a connected device."""
self.create_clean_host_dir(path)
def create_clean_host_dir(self, path):
"""Convenience function for creating a clean directory."""
shutil.rmtree(path)
os.makedirs(path)
def install(self):
"""Run device-specific installation steps."""
pass
def cleanup_steps(self):
"""Run any device-specific cleanup steps."""
pass
def get_device_dirs(self):
""" Set the directories which will be used by the build steps.
These refer to paths on the same device where the test executables will
run, for example, for Android bots these are paths on the Android device
itself. For desktop bots, these are just local paths.
"""
join = lambda p: os.path.join(self._bot_info.build_dir, p)
return DeviceDirs(
dm_dir=join('dm'),
perf_data_dir=self._bot_info.perf_data_dir,
resource_dir=self._bot_info.resource_dir,
images_dir=join('images'),
skp_dir=self._bot_info.local_skp_dir,
tmp_dir=join('tmp'))
def __repr__(self):
return '<%s object>' % self.__class__.__name__ # pragma: no cover

114
infra/bots/flavor/ios_flavor.py Normal file
View File

@ -0,0 +1,114 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
import subprocess
"""iOS flavor utils, used for building for and running tests on iOS."""
class iOSFlavorUtils(default_flavor.DefaultFlavorUtils):
def __init__(self, bot_info):
super(iOSFlavorUtils, self).__init__(bot_info)
self.ios_bin = os.path.join(self._bot_info.skia_dir, 'platform_tools',
'ios', 'bin')
def step(self, cmd, **kwargs):
args = [os.path.join(self.ios_bin, 'ios_run_skia')]
# Convert 'dm' and 'nanobench' from positional arguments
# to flags, which is what iOSShell expects to select which
# one is being run.
cmd = ["--" + c if c in ['dm', 'nanobench'] else c
for c in cmd]
return self._bot_info.run(args + cmd, **kwargs)
def compile(self, target):
"""Build the given target."""
cmd = [os.path.join(self.ios_bin, 'ios_ninja')]
self._bot_info.run(cmd)
def device_path_join(self, *args):
"""Like os.path.join(), but for paths on a connected iOS device."""
return '/'.join(args)
def device_path_exists(self, path):
"""Like os.path.exists(), but for paths on a connected device."""
return self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_path_exists'), path],
) # pragma: no cover
def _remove_device_dir(self, path):
"""Remove the directory on the device."""
return self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_rm'), path],
)
def _create_device_dir(self, path):
"""Create the directory on the device."""
return self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_mkdir'), path],
)
def copy_directory_contents_to_device(self, host_dir, device_dir):
"""Like shutil.copytree(), but for copying to a connected device."""
return self._bot_info.run([
os.path.join(self.ios_bin, 'ios_push_if_needed'),
host_dir, device_dir
])
def copy_directory_contents_to_host(self, device_dir, host_dir):
"""Like shutil.copytree(), but for copying from a connected device."""
return self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_pull_if_needed'),
device_dir, host_dir],
)
def copy_file_to_device(self, host_path, device_path):
"""Like shutil.copyfile, but for copying to a connected device."""
self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_push_file'), host_path, device_path],
) # pragma: no cover
def create_clean_device_dir(self, path):
"""Like shutil.rmtree() + os.makedirs(), but on a connected device."""
self._remove_device_dir(path)
self._create_device_dir(path)
def install(self):
"""Run device-specific installation steps."""
self._bot_info.run([os.path.join(self.ios_bin, 'ios_install')])
def cleanup_steps(self):
"""Run any device-specific cleanup steps."""
self._bot_info.run([os.path.join(self.ios_bin, 'ios_restart')])
self._bot_info.run(['sleep', '20'])
def read_file_on_device(self, path):
"""Read the given file."""
return subprocess.check_output(
[os.path.join(self.ios_bin, 'ios_cat_file'), path]).rstrip()
def remove_file_on_device(self, path):
"""Remove the file on the device."""
return self._bot_info.run(
[os.path.join(self.ios_bin, 'ios_rm'), path],
)
def get_device_dirs(self):
""" Set the directories which will be used by the build steps."""
prefix = self.device_path_join('skiabot', 'skia_')
return default_flavor.DeviceDirs(
dm_dir=prefix + 'dm',
perf_data_dir=prefix + 'perf',
resource_dir=prefix + 'resources',
images_dir=prefix + 'images',
skp_dir=prefix + 'skp/skps',
tmp_dir=prefix + 'tmp_dir')

32
infra/bots/flavor/ssh_devices.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import json
DEFAULT_PORT = '22'
DEFAULT_USER = 'chrome-bot'
SlaveInfo = collections.namedtuple('SlaveInfo',
'ssh_user ssh_host ssh_port')
SLAVE_INFO = {
'skiabot-shuttle-ubuntu12-003':
SlaveInfo('root', '192.168.1.123', DEFAULT_PORT),
'skiabot-shuttle-ubuntu12-004':
SlaveInfo('root', '192.168.1.134', DEFAULT_PORT),
'default':
SlaveInfo('nouser', 'noip', 'noport'),
}
if __name__ == '__main__':
print json.dumps(SLAVE_INFO) # pragma: no cover

123
infra/bots/flavor/ssh_flavor.py Normal file
View File

@ -0,0 +1,123 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
import posixpath
import subprocess
import ssh_devices
"""Utils for running tests remotely over SSH."""
class SSHFlavorUtils(default_flavor.DefaultFlavorUtils):
def __init__(self, *args, **kwargs):
super(SSHFlavorUtils, self).__init__(*args, **kwargs)
slave_info = ssh_devices.SLAVE_INFO.get(self._bot_info.slave_name,
ssh_devices.SLAVE_INFO['default'])
self._host = slave_info.ssh_host
self._port = slave_info.ssh_port
self._user = slave_info.ssh_user
@property
def host(self):
return self._host
@property
def port(self):
return self._port
@property
def user(self):
return self._user
def ssh(self, cmd, **kwargs):
"""Run the given SSH command."""
ssh_cmd = ['ssh']
if self.port:
ssh_cmd.extend(['-p', self.port])
dest = self.host
if self.user:
dest = self.user + '@' + dest
ssh_cmd.append(dest)
ssh_cmd.extend(cmd)
return self._bot_info.run(ssh_cmd, **kwargs)
def step(self, *args, **kwargs):
"""Run the given step over SSH."""
self.ssh(*args, **kwargs)
def device_path_join(self, *args):
"""Like os.path.join(), but for paths on a remote machine."""
return posixpath.join(*args)
def device_path_exists(self, path): # pragma: no cover
"""Like os.path.exists(), but for paths on a remote device."""
try:
self.ssh(['test', '-e', path])
return True
except subprocess.CalledProcessError:
return False
def _remove_device_dir(self, path):
"""Remove the directory on the device."""
self.ssh(['rm', '-rf', path])
def _create_device_dir(self, path):
"""Create the directory on the device."""
self.ssh(['mkdir', '-p', path])
def create_clean_device_dir(self, path):
"""Like shutil.rmtree() + os.makedirs(), but on a remote device."""
self._remove_device_dir(path)
self._create_device_dir(path)
def _make_scp_cmd(self, remote_path, recurse=True):
"""Prepare an SCP command.
Returns a partial SCP command and an adjusted remote path.
"""
cmd = ['scp']
if recurse:
cmd.append('-r')
if self.port:
cmd.extend(['-P', self.port])
adj_remote_path = self.host + ':' + remote_path
if self.user:
adj_remote_path = self.user + '@' + adj_remote_path
return cmd, adj_remote_path
def copy_directory_contents_to_device(self, host_dir, device_dir):
"""Like shutil.copytree(), but for copying to a remote device."""
_, remote_path = self._make_scp_cmd(device_dir)
cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
'scp_dir_contents.sh'),
host_dir, remote_path]
self._bot_info.run(cmd)
def copy_directory_contents_to_host(self, device_dir, host_dir):
"""Like shutil.copytree(), but for copying from a remote device."""
_, remote_path = self._make_scp_cmd(device_dir)
cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
'scp_dir_contents.sh'),
remote_path, host_dir]
self._bot_info.run(cmd)
def copy_file_to_device(self, host_path, device_path):
"""Like shutil.copyfile, but for copying to a connected device."""
cmd, remote_path = self._make_scp_cmd(device_path, recurse=False)
cmd.extend([host_path, remote_path])
self._bot_info.run(cmd)
def read_file_on_device(self, path):
return self.ssh(['cat', path]).rstrip()
def remove_file_on_device(self, path):
"""Delete the given file."""
return self.ssh(['rm', '-f', path])

31
infra/bots/flavor/valgrind_flavor.py Normal file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import default_flavor
import os
"""Utils for running under Valgrind."""
class ValgrindFlavorUtils(default_flavor.DefaultFlavorUtils):
def __init__(self, *args, **kwargs):
super(ValgrindFlavorUtils, self).__init__(*args, **kwargs)
self._suppressions_file = os.path.join(self._bot_info.skia_dir,
'tools', 'valgrind.supp')
def step(self, name, cmd, **kwargs):
new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full',
'--track-origins=yes', '--error-exitcode=1', '--num-callers=40',
'--suppressions=%s' % self._suppressions_file]
path_to_app = os.path.join(self._bot_info.out_dir,
self._bot_info.configuration, cmd[0])
new_cmd.append(path_to_app)
new_cmd.extend(cmd[1:])
return self._bot_info.run(new_cmd, **kwargs)

54
infra/bots/flavor/xsan_flavor.py Normal file
View File

@ -0,0 +1,54 @@
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utils for running under *SAN"""
import default_flavor
import os
class XSanFlavorUtils(default_flavor.DefaultFlavorUtils):
def __init__(self, *args, **kwargs):
super(XSanFlavorUtils, self).__init__(*args, **kwargs)
self._sanitizer = {
# We'd love to just pass 'address,undefined' and get all the checks, but
# we're not anywhere close to being able to do that. Instead we start
# with a set of checks that we know pass or nearly pass. See here for
# more information:
# http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation
'ASAN': ('address,bool,function,integer-divide-by-zero,nonnull-attribute,'
'null,object-size,return,returns-nonnull-attribute,shift,'
'signed-integer-overflow,unreachable,vla-bound,vptr'),
# MSAN and TSAN can't run together with ASAN, so they're their own bots.
'MSAN': 'memory',
'TSAN': 'thread',
}[self._bot_info.bot_cfg['extra_config']]
def compile(self, target):
cmd = [os.path.join(self._bot_info.skia_dir, 'tools', 'xsan_build'),
self._sanitizer, target]
self._bot_info.run(cmd)
def step(self, cmd, env=None, **kwargs):
"""Wrapper for the Step API; runs a step as appropriate for this flavor."""
lsan_suppressions = self._bot_info.skia_dir.join('tools', 'lsan.supp')
tsan_suppressions = self._bot_info.skia_dir.join('tools', 'tsan.supp')
ubsan_suppressions = self._bot_info.skia_dir.join('tools', 'ubsan.supp')
env = dict(env or {})
env['ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'
env['LSAN_OPTIONS'] = ('symbolize=1 print_suppressions=1 suppressions=%s' %
lsan_suppressions)
env['TSAN_OPTIONS'] = 'suppressions=%s' % tsan_suppressions
env['UBSAN_OPTIONS'] = 'suppressions=%s' % ubsan_suppressions
path_to_app = os.path.join(self._bot_info.out_dir,
self._bot_info.configuration, cmd[0])
new_cmd = [path_to_app]
new_cmd.extend(cmd[1:])
return self._bot_info.run(new_cmd, env=env, **kwargs)

7
infra/bots/skia_repo.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'../../',
],
},
}