AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[recipes] Move a lot of logic out of vars module

Browse Source 
In general, vars should only contain variables which are the same for
all tasks.  Variables specific to compilation belong in the build
module (or compile recipe), and those specific to running tests belong
in the flavor module, or the individual recipe which uses them.

Bug: skia:6473
Change-Id: Ifd55a57118c5801e6f4934a6b5de9d1567415b9a
Reviewed-on: https://skia-review.googlesource.com/128545
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
This commit is contained in:
Eric Boren 2018-05-18 07:36:55 -04:00 committed by Skia Commit-Bot
parent 50edafacc8
commit 72f6668eb7
46 changed files with 1154 additions and 671 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
infra/bots/recipe_modules/build/android.py
View File

@ -6,7 +6,8 @@
import re
def compile_fn(api, out_dir):
def compile_fn(api, checkout_root, out_dir):
skia_dir = checkout_root.join('skia')
compiler = api.vars.builder_cfg.get('compiler')
configuration = api.vars.builder_cfg.get('configuration')
extra_tokens = api.vars.extra_tokens
@ -34,7 +35,7 @@ def compile_fn(api, out_dir):
'target_cpu': quote(target_arch),
}
extra_cflags.append('-DDUMMY_ndk_version=%s' %
api.run.asset_version(ndk_asset))
api.run.asset_version(ndk_asset, skia_dir))
if configuration != 'Debug':
args['is_debug'] = 'false'
@ -59,11 +60,11 @@ def compile_fn(api, out_dir):
gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems()))
gn = 'gn.exe' if 'Win' in os else 'gn'
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
gn = api.vars.skia_dir.join('bin', gn)
gn = skia_dir.join('bin', gn)
with api.context(cwd=api.vars.skia_dir):
with api.context(cwd=skia_dir):
api.run(api.python, 'fetch-gn',
script=api.vars.skia_dir.join('bin', 'fetch-gn'),
script=skia_dir.join('bin', 'fetch-gn'),
infra_step=True)
# If this is the SkQP build, set up the environment and run the script
@ -72,7 +73,7 @@ def compile_fn(api, out_dir):
api.infra.update_go_deps()
output_binary = out_dir.join('run_testlab')
build_target = api.vars.skia_dir.join('infra', 'cts', 'run_testlab.go')
build_target = skia_dir.join('infra', 'cts', 'run_testlab.go')
build_cmd = ['go', 'build', '-o', output_binary, build_target]
with api.context(env=api.infra.go_env):
api.run(api.step, 'build firebase runner', cmd=build_cmd)
@ -88,8 +89,7 @@ def compile_fn(api, out_dir):
'APK_OUTPUT_DIR': out_dir,
}
mk_universal = api.vars.skia_dir.join('tools', 'skqp',
'make_universal_apk')
mk_universal = skia_dir.join('tools', 'skqp', 'make_universal_apk')
with api.context(env=env):
api.run(api.step, 'make_universal', cmd=[mk_universal])
else:
@ -99,6 +99,4 @@ def compile_fn(api, out_dir):
def copy_extra_build_products(api, src, dst):
if 'SKQP' in api.vars.extra_tokens:
wlist = api.vars.skia_dir.join('infra','cts', 'whitelist_devices.json')
api.file.copy('copy whitelist', wlist, dst)
pass

12
infra/bots/recipe_modules/build/api.py
View File

@ -36,13 +36,11 @@ class BuildApi(recipe_api.RecipeApi):
self.copy_fn = default.copy_extra_build_products
super(BuildApi, self).__init__(*args, **kwargs)
def __call__(self):
def __call__(self, checkout_root, out_dir):
"""Compile the code."""
out_dir = self.m.vars.skia_out.join(self.m.vars.configuration)
self.compile_fn(self.m, out_dir)
self.compile_fn(self.m, checkout_root, out_dir)
def copy_build_products(self, dst):
def copy_build_products(self, out_dir, dst):
"""Copy whitelisted build products to dst."""
src = self.m.vars.skia_out.join(self.m.vars.configuration)
util.copy_whitelisted_build_products(self.m, src, dst)
self.copy_fn(self.m, src, dst)
util.copy_whitelisted_build_products(self.m, out_dir, dst)
self.copy_fn(self.m, out_dir, dst)

11
infra/bots/recipe_modules/build/chromebook.py
View File

@ -3,7 +3,8 @@
# found in the LICENSE file.
def compile_fn(api, out_dir):
def compile_fn(api, checkout_root, out_dir):
skia_dir = checkout_root.join('skia')
configuration = api.vars.builder_cfg.get('configuration')
os = api.vars.builder_cfg.get('os')
target_arch = api.vars.builder_cfg.get('target_arch')
@ -74,7 +75,7 @@ def compile_fn(api, out_dir):
'skia_use_egl': 'true',
}
extra_cflags.append('-DDUMMY_clang_linux_version=%s' %
api.run.asset_version('clang_linux'))
api.run.asset_version('clang_linux', skia_dir))
if configuration != 'Debug':
args['is_debug'] = 'false'
@ -86,11 +87,11 @@ def compile_fn(api, out_dir):
gn = 'gn.exe' if 'Win' in os else 'gn'
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
gn = api.vars.skia_dir.join('bin', gn)
gn = skia_dir.join('bin', gn)
with api.context(cwd=api.vars.skia_dir, env=env):
with api.context(cwd=skia_dir, env=env):
api.run(api.python, 'fetch-gn',
script=api.vars.skia_dir.join('bin', 'fetch-gn'),
script=skia_dir.join('bin', 'fetch-gn'),
infra_step=True)
api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args])
api.run(api.step, 'ninja',

11
infra/bots/recipe_modules/build/chromecast.py
View File

@ -3,7 +3,8 @@
# found in the LICENSE file.
def compile_fn(api, out_dir):
def compile_fn(api, checkout_root, out_dir):
skia_dir = checkout_root.join('skia')
configuration = api.vars.builder_cfg.get('configuration')
os = api.vars.builder_cfg.get('os')
target_arch = api.vars.builder_cfg.get('target_arch')
@ -21,7 +22,7 @@ def compile_fn(api, out_dir):
# Makes the binary small enough to fit on the small disk.
'-g0',
('-DDUMMY_cast_toolchain_version=%s' %
api.run.asset_version('cast_toolchain')),
api.run.asset_version('cast_toolchain', skia_dir)),
]
extra_ldflags = [
@ -55,11 +56,11 @@ def compile_fn(api, out_dir):
gn = 'gn.exe' if 'Win' in os else 'gn'
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
gn = api.vars.skia_dir.join('bin', gn)
gn = skia_dir.join('bin', gn)
with api.context(cwd=api.vars.skia_dir):
with api.context(cwd=skia_dir):
api.run(api.python, 'fetch-gn',
script=api.vars.skia_dir.join('bin', 'fetch-gn'),
script=skia_dir.join('bin', 'fetch-gn'),
infra_step=True)
api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args])
api.run(api.step, 'ninja',

35
infra/bots/recipe_modules/build/default.py
View File

@ -6,12 +6,12 @@
from . import util
def build_command_buffer(api):
def build_command_buffer(api, chrome_dir, skia_dir, out):
api.run(api.python, 'build command_buffer',
script=api.vars.skia_dir.join('tools', 'build_command_buffer.py'),
script=skia_dir.join('tools', 'build_command_buffer.py'),
args=[
'--chrome-dir', api.vars.checkout_root,
'--output-dir', api.vars.skia_out.join(api.vars.configuration),
'--chrome-dir', chrome_dir,
'--output-dir', out,
'--no-sync', '--no-hooks', '--make-output-dir'])
@ -40,7 +40,8 @@ def compile_swiftshader(api, swiftshader_root, cc, cxx, out):
cmd=['ninja', '-C', out, 'libEGL.so', 'libGLESv2.so'])
def compile_fn(api, out_dir):
def compile_fn(api, checkout_root, out_dir):
skia_dir = checkout_root.join('skia')
compiler = api.vars.builder_cfg.get('compiler', '')
configuration = api.vars.builder_cfg.get('configuration', '')
extra_tokens = api.vars.extra_tokens
@ -69,7 +70,7 @@ def compile_fn(api, out_dir):
extra_ldflags.append('-B%s/bin' % clang_linux)
extra_ldflags.append('-fuse-ld=lld')
extra_cflags.append('-DDUMMY_clang_linux_version=%s' %
api.run.asset_version('clang_linux'))
api.run.asset_version('clang_linux', skia_dir))
if os == 'Ubuntu14':
extra_ldflags.extend(['-static-libstdc++', '-static-libgcc'])
@ -89,7 +90,7 @@ def compile_fn(api, out_dir):
'-Wno-format-truncation',
'-Wno-uninitialized',
('-DDUMMY_mips64el_toolchain_linux_version=%s' %
api.run.asset_version('mips64el_toolchain_linux'))
api.run.asset_version('mips64el_toolchain_linux', skia_dir))
])
if configuration == 'Release':
# This warning is only triggered when fuzz_canvas is inlined.
@ -106,7 +107,7 @@ def compile_fn(api, out_dir):
cxx = emscripten_sdk + '/emscripten/incoming/em++'
extra_cflags.append('-Wno-unknown-warning-option')
extra_cflags.append('-DDUMMY_emscripten_sdk_version=%s' %
api.run.asset_version('emscripten_sdk'))
api.run.asset_version('emscripten_sdk', skia_dir))
if 'Coverage' in extra_tokens:
# See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for
# more info on using llvm to gather coverage information.
@ -139,8 +140,7 @@ def compile_fn(api, out_dir):
if 'ANGLE' in extra_tokens:
args['skia_use_angle'] = 'true'
if 'SwiftShader' in extra_tokens:
swiftshader_root = api.vars.skia_dir.join('third_party', 'externals',
'swiftshader')
swiftshader_root = skia_dir.join('third_party', 'externals', 'swiftshader')
swiftshader_out = out_dir.join('swiftshader_out')
compile_swiftshader(api, swiftshader_root, cc, cxx, swiftshader_out)
args['skia_use_egl'] = 'true'
@ -154,7 +154,8 @@ def compile_fn(api, out_dir):
'-L%s' % swiftshader_out,
])
if 'CommandBuffer' in extra_tokens:
api.run.run_once(build_command_buffer, api)
chrome_dir = checkout_root
api.run.run_once(build_command_buffer, api, chrome_dir, skia_dir, out_dir)
if 'MSAN' in extra_tokens:
args['skia_enable_gpu'] = 'false'
args['skia_use_fontconfig'] = 'false'
@ -215,7 +216,7 @@ def compile_fn(api, out_dir):
if compiler == 'Clang' and 'Win' in os:
args['clang_win'] = '"%s"' % api.vars.slave_dir.join('clang_win')
extra_cflags.append('-DDUMMY_clang_win_version=%s' %
api.run.asset_version('clang_win'))
api.run.asset_version('clang_win', skia_dir))
if target_arch == 'wasm':
args.update({
'skia_use_freetype': 'false',
@ -253,18 +254,18 @@ def compile_fn(api, out_dir):
gn = 'gn.exe' if 'Win' in os else 'gn'
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
gn = api.vars.skia_dir.join('bin', gn)
gn = skia_dir.join('bin', gn)
with api.context(cwd=api.vars.skia_dir):
with api.context(cwd=skia_dir):
api.run(api.python,
'fetch-gn',
script=api.vars.skia_dir.join('bin', 'fetch-gn'),
script=skia_dir.join('bin', 'fetch-gn'),
infra_step=True)
if 'CheckGeneratedFiles' in extra_tokens:
env['PATH'] = '%s:%%(PATH)s' % api.vars.skia_dir.join('bin')
env['PATH'] = '%s:%%(PATH)s' % skia_dir.join('bin')
api.run(api.python,
'fetch-clang-format',
script=api.vars.skia_dir.join('bin', 'fetch-clang-format'),
script=skia_dir.join('bin', 'fetch-clang-format'),
infra_step=True)
if target_arch == 'wasm':
fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin'

14
infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json
View File

@ -141,20 +141,6 @@
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json",
"[START_DIR]/[SWARM_OUT_DIR]/out/devrel"
],
"infra_step": true,
"name": "copy whitelist"
},
{
"name": "$result",
"recipe_result": null,

4
infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json
View File

@ -30,7 +30,7 @@
"cmd": [
"ninja",
"-C",
"out/android_release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"-j100"
],
"cwd": "[START_DIR]/cache/work/flutter/src",
@ -45,7 +45,7 @@
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"[START_DIR]/[SWARM_OUT_DIR]/out/Release"
],
"infra_step": true,

23
infra/bots/recipe_modules/build/examples/full.py
View File

@ -5,6 +5,7 @@
DEPS = [
'build',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/raw_io',
'run',
@ -15,10 +16,24 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
api.build()
api.build.copy_build_products(
api.vars.swarming_out_dir.join(
'out', api.vars.configuration))
# Hackery to avoid changing expectations.
checkout_root = api.vars.cache_dir.join('work')
out_dir = checkout_root.join(
'skia', 'out', api.vars.builder_name, api.vars.configuration)
if ('CheckGeneratedFiles' in api.vars.builder_name or
'-CT_' in api.vars.builder_name):
out_dir = api.vars.build_dir.join('out', api.vars.configuration)
elif 'NoDEPS' in api.vars.builder_name:
checkout_root = api.path['start_dir']
out_dir = checkout_root.join(
'skia', 'out', api.vars.builder_name, api.vars.configuration)
elif 'Flutter' in api.vars.builder_name:
checkout_root = checkout_root.join('flutter')
out_dir = checkout_root.join('src', 'out', 'android_release')
api.build(checkout_root=checkout_root, out_dir=out_dir)
dst = api.vars.swarming_out_dir.join('out', api.vars.configuration)
api.build.copy_build_products(out_dir=out_dir, dst=dst)
api.run.check_failure()

10
infra/bots/recipe_modules/build/flutter.py
View File

@ -3,11 +3,10 @@
# found in the LICENSE file.
def compile_fn(api, _):
flutter_dir = api.vars.checkout_root.join('src')
def compile_fn(api, checkout_root, out_dir):
flutter_dir = checkout_root.join('src')
configuration = api.vars.builder_cfg.get('configuration').lower()
extra_tokens = api.vars.extra_tokens
out_dir = configuration
with api.context(cwd=flutter_dir):
# Setup GN args.
@ -16,10 +15,9 @@ def compile_fn(api, _):
]
if 'Android' in extra_tokens:
gn_args.append('--android')
out_dir = 'android_' + out_dir
# Delete out_dir so that we start from a clean slate. See skbug/6310.
api.run.rmtree(flutter_dir.join('out', out_dir))
api.run.rmtree(out_dir)
# Run GN.
api.run(
@ -31,7 +29,7 @@ def compile_fn(api, _):
api.run(
api.step,
'build_flutter',
cmd=['ninja', '-C', 'out/' + out_dir, '-j100'])
cmd=['ninja', '-C', out_dir, '-j100'])
def copy_extra_build_products(api, src, dst):

56
infra/bots/recipe_modules/core/api.py
View File

@ -17,39 +17,58 @@ from recipe_engine import config_types
class SkiaApi(recipe_api.RecipeApi):
@property
def default_checkout_root(self):
"""The default location for cached persistent checkouts."""
return self.m.vars.cache_dir.join('work')
def patch_ref(self, issue, patchset):
"""Build a ref for the given issue and patchset."""
return 'refs/changes/%s/%s/%s' % (issue[-2:], issue, patchset)
def checkout_git(self):
def checkout_git(self, checkout_root):
"""Run the steps to perform a pure-git checkout without DEPS."""
skia_dir = checkout_root.join('skia')
self.m.git.checkout(
self.m.properties['repository'], dir_path=self.m.vars.skia_dir,
self.m.properties['repository'], dir_path=skia_dir,
ref=self.m.properties['revision'], submodules=False)
if self.m.vars.is_trybot:
ref = self.patch_ref(str(self.m.vars.issue), str(self.m.vars.patchset))
self.m.git('fetch', 'origin', ref)
self.m.git('checkout', 'FETCH_HEAD')
self.m.git('rebase', self.m.properties['revision'])
return self.m.properties['revision']
def checkout_bot_update(self):
def checkout_bot_update(self, checkout_root, gclient_cache=None):
"""Run the steps to obtain a checkout using bot_update."""
if not gclient_cache:
gclient_cache = self.m.vars.cache_dir.join('git')
cfg_kwargs = {}
is_parent_revision = 'ParentRevision' in self.m.vars.extra_tokens
# Use a persistent gclient cache for Swarming.
cfg_kwargs['CACHE_DIR'] = self.m.vars.gclient_cache
cfg_kwargs['CACHE_DIR'] = gclient_cache
# Create the checkout path if necessary.
if not self.m.path.exists(self.m.vars.checkout_root):
self.m.file.ensure_directory('makedirs checkout_path',
self.m.vars.checkout_root)
if not self.m.path.exists(checkout_root):
self.m.file.ensure_directory('makedirs checkout_path', checkout_root)
# Initial cleanup.
gclient_cfg = self.m.gclient.make_config(**cfg_kwargs)
# Some bots also require a checkout of chromium.
need_chromium_checkout = False
gclient_env = {'DEPOT_TOOLS_UPDATE': '0'}
if 'CommandBuffer' in self.m.properties['buildername']:
need_chromium_checkout = True
gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0'
if 'RecreateSKPs' in self.m.properties['buildername']:
need_chromium_checkout = True
gclient_env['CPPFLAGS'] = '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1'
# Add chromium first because of skbug.com/7917.
if self.m.vars.need_chromium_checkout:
if need_chromium_checkout:
chromium = gclient_cfg.solutions.add()
chromium.name = 'src'
chromium.managed = False
@ -57,14 +76,16 @@ class SkiaApi(recipe_api.RecipeApi):
chromium.revision = 'origin/master'
main_repo = self.m.properties['repository']
if self.m.vars.need_flutter_checkout:
need_flutter_checkout = 'Flutter' in self.m.properties['buildername']
if need_flutter_checkout:
main_repo = 'https://github.com/flutter/engine.git'
main_name = self.m.path.basename(main_repo)
if main_name.endswith('.git'):
main_name = main_name[:-len('.git')]
# Special case for flutter because it seems to need a very specific
# directory structure to successfully build.
if self.m.vars.need_flutter_checkout and main_name == 'engine':
if need_flutter_checkout and main_name == 'engine':
main_name = 'src/flutter'
main = gclient_cfg.solutions.add()
main.name = main_name
@ -81,7 +102,7 @@ class SkiaApi(recipe_api.RecipeApi):
if patch_root.endswith('.git'):
patch_root = patch_root[:-4]
if self.m.vars.need_flutter_checkout:
if need_flutter_checkout:
# Skia is a DEP of Flutter; the 'revision' property is a Skia revision,
# and any patch should be applied to Skia, not Flutter.
main.revision = 'origin/master'
@ -99,7 +120,7 @@ class SkiaApi(recipe_api.RecipeApi):
# TODO(rmistry): Remove the below block after there is a solution for
# crbug.com/616443
entries_file = self.m.vars.checkout_root.join('.gclient_entries')
entries_file = checkout_root.join('.gclient_entries')
if self.m.path.exists(entries_file) or self._test_data.enabled:
self.m.file.remove('remove %s' % entries_file,
entries_file)
@ -118,7 +139,7 @@ class SkiaApi(recipe_api.RecipeApi):
main.revision = main.revision + '^'
self.m.gclient.c = gclient_cfg
with self.m.context(cwd=self.m.vars.checkout_root):
with self.m.context(cwd=checkout_root):
update_step = self.m.bot_update.ensure_checkout(
patch_root=patch_root,
# The logic in ensure_checkout for this arg is fairly naive, so if
@ -128,10 +149,7 @@ class SkiaApi(recipe_api.RecipeApi):
patch=not (self.m.vars.is_trybot and is_parent_revision)
)
self.m.vars.got_revision = (
update_step.presentation.properties['got_revision'])
if self.m.vars.need_chromium_checkout or self.m.vars.need_flutter_checkout:
with self.m.context(cwd=self.m.vars.checkout_root,
env=self.m.vars.gclient_env):
if need_chromium_checkout or need_flutter_checkout:
with self.m.context(cwd=checkout_root, env=gclient_env):
self.m.gclient.runhooks()
return update_step.presentation.properties['got_revision']

147
infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json Normal file
View File

@ -0,0 +1,147 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work/flutter"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/flutter/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/flutter/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src/flutter', 'url': 'https://github.com/flutter/engine.git'}]\ntarget_os = ['android']",
"--patch_root",
"src/third_party/skia",
"--revision_mapping_file",
"{\"got_flutter_revision\": \"src/flutter\", \"got_revision\": \"src/third_party/skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"src/flutter@origin/master",
"--revision",
"src/third_party/skia@abc123"
],
"cwd": "[START_DIR]/cache/work/flutter",
"env_prefixes": {
"PATH": [
"RECIPE_PACKAGE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"origin/master\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/third_party/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/third_party/skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision_cp\": \"refs/heads/master@{#84512}\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#143121}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/third_party/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_flutter_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@",
"@@@SET_BUILD_PROPERTY@got_flutter_revision_cp@\"refs/heads/master@{#84512}\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#143121}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
"runhooks"
],
"cwd": "[START_DIR]/cache/work/flutter",
"env": {
"DEPOT_TOOLS_UPDATE": "0",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "gclient runhooks"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

134
infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json Normal file
View File

@ -0,0 +1,134 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}, {'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123",
"--revision",
"src@origin/master"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_PACKAGE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"src\": \"origin/master\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
"runhooks"
],
"cwd": "[START_DIR]/cache/work",
"env": {
"DEPOT_TOOLS_UPDATE": "0",
"GYP_CHROMIUM_NO_ACTION": "0",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "gclient runhooks"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

29
infra/bots/recipe_modules/core/examples/full.py
View File

@ -15,13 +15,18 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
bot_update = True
if 'NoDEPS' in api.properties['buildername']:
bot_update = False
if bot_update:
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
if 'Flutter' in api.vars.builder_name:
checkout_root = checkout_root.join('flutter')
api.core.checkout_bot_update(checkout_root=checkout_root)
else:
api.core.checkout_git()
api.core.checkout_git(checkout_root=api.path['start_dir'])
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
@ -139,3 +144,23 @@ def GenTests(api):
api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
)
)
buildername = 'Build-Mac-Clang-x86_64-Debug-CommandBuffer'
yield (
api.test(buildername) +
api.properties(buildername=buildername,
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]')
)
buildername = 'Build-Debian9-GCC-x86_64-Release-Flutter_Android'
yield (
api.test(buildername) +
api.properties(buildername=buildername,
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]')
)

18
infra/bots/recipe_modules/flavor/api.py
View File

@ -64,6 +64,9 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
def setup(self):
self._f = self.get_flavor(self.m.vars)
self.device_dirs = self._f.device_dirs
self.host_dirs = self._f.host_dirs
self._skia_dir = self.m.path['start_dir'].join('skia')
def step(self, name, cmd, **kwargs):
return self._f.step(name, cmd, **kwargs)
@ -97,12 +100,11 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
def install(self, skps=False, images=False, svgs=False, resources=False):
self._f.install()
self.device_dirs = self._f.device_dirs
# TODO(borenet): Only copy files which have changed.
if resources:
self.copy_directory_contents_to_device(
self.m.vars.resource_dir,
self.m.path['start_dir'].join('skia', 'resources'),
self.device_dirs.resource_dir)
if skps:
@ -138,7 +140,7 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
def _copy_images(self):
"""Download and copy test images if needed."""
version = self.m.run.asset_version('skimage')
version = self.m.run.asset_version('skimage', self._skia_dir)
self.m.run.writefile(
self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SK_IMAGE),
version)
@ -146,13 +148,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
version,
VERSION_FILE_SK_IMAGE,
self.m.vars.tmp_dir,
self.m.vars.images_dir,
self.host_dirs.images_dir,
self.device_dirs.images_dir)
return version
def _copy_skps(self):
"""Download and copy the SKPs if needed."""
version = self.m.run.asset_version('skp')
version = self.m.run.asset_version('skp', self._skia_dir)
self.m.run.writefile(
self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SKP),
version)
@ -160,13 +162,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
version,
VERSION_FILE_SKP,
self.m.vars.tmp_dir,
self.m.vars.local_skp_dir,
self.host_dirs.skp_dir,
self.device_dirs.skp_dir)
return version
def _copy_svgs(self):
"""Download and copy the SVGs if needed."""
version = self.m.run.asset_version('svg')
version = self.m.run.asset_version('svg', self._skia_dir)
self.m.run.writefile(
self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SVG),
version)
@ -174,6 +176,6 @@ class SkiaFlavorApi(recipe_api.RecipeApi):
version,
VERSION_FILE_SVG,
self.m.vars.tmp_dir,
self.m.vars.local_svg_dir,
self.host_dirs.svg_dir,
self.device_dirs.svg_dir)
return version

32
infra/bots/recipe_modules/flavor/default_flavor.py
View File

@ -17,6 +17,7 @@ WIN_TOOLCHAIN_DIR = 't'
class DeviceDirs(object):
def __init__(self,
bin_dir,
dm_dir,
perf_data_dir,
resource_dir,
@ -24,6 +25,7 @@ class DeviceDirs(object):
skp_dir,
svg_dir,
tmp_dir):
self._bin_dir = bin_dir
self._dm_dir = dm_dir
self._perf_data_dir = perf_data_dir
self._resource_dir = resource_dir
@ -32,6 +34,10 @@ class DeviceDirs(object):
self._svg_dir = svg_dir
self._tmp_dir = tmp_dir
@property
def bin_dir(self):
return self._bin_dir
@property
def dm_dir(self):
"""Where DM writes."""
@ -85,11 +91,18 @@ class DefaultFlavorUtils(object):
# syntax as regular recipe modules to run steps, eg: self.m.step(...)
self.m = module.m
self._chrome_path = None
self._win_toolchain_dir = self.m.vars.slave_dir.join(WIN_TOOLCHAIN_DIR)
win_toolchain_asset_path = self.m.vars.infrabots_dir.join(
'assets', 'win_toolchain', 'VERSION')
if not self.m.path.exists(win_toolchain_asset_path):
self._win_toolchain_dir = self.m.vars.slave_dir
self.device_dirs = DeviceDirs(
bin_dir=self.m.vars.build_dir.join('out', self.m.vars.configuration),
dm_dir=self.m.path.join(self.m.vars.swarming_out_dir, 'dm'),
perf_data_dir=self.m.path.join(
self.m.vars.swarming_out_dir,
'perfdata', self.m.vars.builder_name, 'data'),
resource_dir=self.m.path['start_dir'].join('skia', 'resources'),
images_dir=self.m.path['start_dir'].join('skimage'),
skp_dir=self.m.path['start_dir'].join('skp'),
svg_dir=self.m.path['start_dir'].join('svg'),
tmp_dir=self.m.vars.tmp_dir)
self.host_dirs = self.device_dirs
def device_path_join(self, *args):
"""Like os.path.join(), but for paths on a connected device."""
@ -137,14 +150,7 @@ class DefaultFlavorUtils(object):
def install(self):
"""Run device-specific installation steps."""
self.device_dirs = DeviceDirs(
dm_dir=self.m.vars.dm_dir,
perf_data_dir=self.m.vars.perf_data_dir,
resource_dir=self.m.vars.resource_dir,
images_dir=self.m.vars.images_dir,
skp_dir=self.m.vars.local_skp_dir,
svg_dir=self.m.vars.local_svg_dir,
tmp_dir=self.m.vars.tmp_dir)
pass
def cleanup_steps(self):
"""Run any device-specific cleanup steps."""

164
infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json
View File

@ -1,164 +0,0 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"rmtree",
"results_dir"
],
"infra_step": true,
"name": "rmtree results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"results_dir"
],
"infra_step": true,
"name": "makedirs results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"rmtree",
"device_results_dir"
],
"infra_step": true,
"name": "rmtree device_results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"device_results_dir"
],
"infra_step": true,
"name": "makedirs device_results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/skp/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get skp VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SKP_VERSION"
],
"infra_step": true,
"name": "write SKP_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/skimage/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get skimage VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SK_IMAGE_VERSION"
],
"infra_step": true,
"name": "write SK_IMAGE_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/svg/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get svg VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SVG_VERSION"
],
"infra_step": true,
"name": "write SVG_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py",
"[START_DIR]",
"catchsegv",
"[START_DIR]/build/out/Release/nanobench",
"--some-flag"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "symbolized nanobench"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

6
infra/bots/recipe_modules/flavor/examples/full.py
View File

@ -43,11 +43,12 @@ def RunSteps(api):
if 'Test' in api.properties['buildername']:
api.flavor.step('dm', ['dm', '--some-flag'])
api.flavor.copy_directory_contents_to_host(
api.flavor.device_dirs.dm_dir, api.vars.dm_dir)
api.flavor.device_dirs.dm_dir, api.flavor.host_dirs.dm_dir)
elif 'Perf' in api.properties['buildername']:
api.flavor.step('nanobench', ['nanobench', '--some-flag'])
api.flavor.copy_directory_contents_to_host(
api.flavor.device_dirs.perf_data_dir, api.vars.perf_data_dir)
api.flavor.device_dirs.perf_data_dir,
api.flavor.host_dirs.perf_data_dir)
finally:
api.flavor.cleanup_steps()
api.run.check_failure()
@ -62,7 +63,6 @@ TEST_BUILDERS = [
'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All',
'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN',
'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN',
'Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs',
'Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-All-Android',
'Test-Android-Clang-GalaxyS7_G930FD-GPU-MaliT880-arm64-Debug-All-Android',
'Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Debug-All-Android',

37
infra/bots/recipe_modules/flavor/gn_android_flavor.py
View File

@ -22,14 +22,17 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
self.ADB_PUB_KEY = ('/home/chrome-bot/.android/'
'chrome_infrastructure_adbkey')
# Data should go in android_data_dir, which may be preserved across runs.
android_data_dir = '/sdcard/revenge_of_the_skiabot/'
self.device_dirs = default_flavor.DeviceDirs(
dm_dir = self.m.vars.android_data_dir + 'dm_out',
perf_data_dir = self.m.vars.android_data_dir + 'perf',
resource_dir = self.m.vars.android_data_dir + 'resources',
images_dir = self.m.vars.android_data_dir + 'images',
skp_dir = self.m.vars.android_data_dir + 'skps',
svg_dir = self.m.vars.android_data_dir + 'svgs',
tmp_dir = self.m.vars.android_data_dir)
bin_dir = '/data/local/tmp/',
dm_dir = android_data_dir + 'dm_out',
perf_data_dir = android_data_dir + 'perf',
resource_dir = android_data_dir + 'resources',
images_dir = android_data_dir + 'images',
skp_dir = android_data_dir + 'skps',
svg_dir = android_data_dir + 'svgs',
tmp_dir = android_data_dir)
# A list of devices we can't root. If rooting fails and a device is not
# on the list, we fail the task to avoid perf inconsistencies.
@ -71,7 +74,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
}
def _run(self, title, *cmd, **kwargs):
with self.m.context(cwd=self.m.vars.skia_dir):
with self.m.context(cwd=self.m.path['start_dir'].join('skia')):
return self.m.run(self.m.step, title, cmd=list(cmd), **kwargs)
def _adb(self, title, *cmd, **kwargs):
@ -99,7 +102,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
timeout=180, abort_on_failure=False,
fail_build_on_failure=False)
with self.m.context(cwd=self.m.vars.skia_dir):
with self.m.context(cwd=self.m.path['start_dir'].join('skia')):
with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}):
return self.m.run.with_retry(self.m.step, title, attempts,
cmd=[self.ADB_BINARY]+list(cmd),
@ -427,7 +430,7 @@ wait_for_device()
line = line.replace(addr, addr + ' ' + sym.strip())
print line
""" % self.ADB_BINARY,
args=[self.m.vars.skia_out.join(self.m.vars.configuration)],
args=[self.m.vars.skia_out],
infra_step=True,
timeout=300,
abort_on_failure=False)
@ -453,17 +456,17 @@ wait_for_device()
self._scale_for_nanobench()
else:
self._scale_for_dm()
app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
app = self.m.vars.skia_out.join(cmd[0])
self._adb('push %s' % cmd[0],
'push', app, self.m.vars.android_bin_dir)
'push', app, self.device_dirs.bin_dir)
sh = '%s.sh' % cmd[0]
self.m.run.writefile(self.m.vars.tmp_dir.join(sh),
'set -x; %s%s; echo $? >%src' %
(self.m.vars.android_bin_dir, subprocess.list2cmdline(map(str, cmd)),
self.m.vars.android_bin_dir))
'set -x; %s%s; echo $? >%src' % (
self.device_dirs.bin_dir, subprocess.list2cmdline(map(str, cmd)),
self.device_dirs.bin_dir))
self._adb('push %s' % sh,
'push', self.m.vars.tmp_dir.join(sh), self.m.vars.android_bin_dir)
'push', self.m.vars.tmp_dir.join(sh), self.device_dirs.bin_dir)
self._adb('clear log', 'logcat', '-c')
self.m.python.inline('%s' % cmd[0], """
@ -479,7 +482,7 @@ wait_for_device()
print "Couldn't read the return code. Probably killed for OOM."
sys.exit(1)
""" % (self.ADB_BINARY, self.ADB_BINARY),
args=[self.m.vars.android_bin_dir, sh])
args=[self.device_dirs.bin_dir, sh])
def copy_file_to_device(self, host, device):
self._adb('push %s %s' % (host, device), 'push', host, device)

30
infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py
View File

@ -21,16 +21,16 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils):
super(GNChromebookFlavorUtils, self).__init__(m)
self._user_ip = ''
self.chromeos_homedir = '/home/chronos/user/'
self.device_dirs = default_flavor.DeviceDirs(
dm_dir = self.m.vars.chromeos_homedir + 'dm_out',
perf_data_dir = self.m.vars.chromeos_homedir + 'perf',
resource_dir = self.m.vars.chromeos_homedir + 'resources',
images_dir = self.m.vars.chromeos_homedir + 'images',
skp_dir = self.m.vars.chromeos_homedir + 'skps',
svg_dir = self.m.vars.chromeos_homedir + 'svgs',
tmp_dir = self.m.vars.chromeos_homedir)
self._bin_dir = self.m.vars.chromeos_homedir + 'bin'
bin_dir = self.chromeos_homedir + 'bin',
dm_dir = self.chromeos_homedir + 'dm_out',
perf_data_dir = self.chromeos_homedir + 'perf',
resource_dir = self.chromeos_homedir + 'resources',
images_dir = self.chromeos_homedir + 'images',
skp_dir = self.chromeos_homedir + 'skps',
svg_dir = self.chromeos_homedir + 'svgs',
tmp_dir = self.chromeos_homedir)
@property
def user_ip(self):
@ -62,10 +62,10 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils):
self.device_dirs.resource_dir)
# Ensure the home dir is marked executable
self._ssh('remount %s as exec' % self.m.vars.chromeos_homedir,
self._ssh('remount %s as exec' % self.chromeos_homedir,
'sudo', 'mount', '-i', '-o', 'remount,exec', '/home/chronos')
self.create_clean_device_dir(self._bin_dir)
self.create_clean_device_dir(self.device_dirs.bin_dir)
def create_clean_device_dir(self, path):
# use -f to silently return if path doesn't exist
@ -121,13 +121,13 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils):
name = cmd[0]
if name == 'dm':
self.create_clean_host_dir(self.m.vars.dm_dir)
self.create_clean_host_dir(self.host_dirs.dm_dir)
if name == 'nanobench':
self.create_clean_host_dir(self.m.vars.perf_data_dir)
self.create_clean_host_dir(self.host_dirs.perf_data_dir)
app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
app = self.m.vars.skia_out.join(cmd[0])
cmd[0] = '%s/%s' % (self._bin_dir, cmd[0])
cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0])
self.copy_file_to_device(app, cmd[0])
self._ssh('chmod %s' % name, 'chmod', '+x', cmd[0])

31
infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py
View File

@ -15,22 +15,21 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
super(GNChromecastFlavorUtils, self).__init__(m)
self._ever_ran_adb = False
self._user_ip = ''
self.m.vars.android_bin_dir = self.m.path.join(self.m.vars.android_bin_dir,
'bin')
# Disk space is extremely tight on the Chromecasts (~100M) There is not
# enough space on the android_data_dir (/cache/skia) to fit the images,
# resources, executable and output the dm images. So, we have dm_out be
# on the tempfs (i.e. RAM) /dev/shm. (which is about 140M)
data_dir = '/cache/skia/'
self.device_dirs = default_flavor.DeviceDirs(
bin_dir = '/cache/skia/bin',
dm_dir = '/dev/shm/skia/dm_out',
perf_data_dir = self.m.vars.android_data_dir + 'perf',
resource_dir = self.m.vars.android_data_dir + 'resources',
images_dir = self.m.vars.android_data_dir + 'images',
skp_dir = self.m.vars.android_data_dir + 'skps',
svg_dir = self.m.vars.android_data_dir + 'svgs',
tmp_dir = self.m.vars.android_data_dir)
perf_data_dir = data_dir + 'perf',
resource_dir = data_dir + 'resources',
images_dir = data_dir + 'images',
skp_dir = data_dir + 'skps',
svg_dir = data_dir + 'svgs',
tmp_dir = data_dir)
@property
def user_ip_host(self):
@ -53,8 +52,8 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
def install(self):
super(GNChromecastFlavorUtils, self).install()
self._adb('mkdir ' + self.m.vars.android_bin_dir,
'shell', 'mkdir', '-p', self.m.vars.android_bin_dir)
self._adb('mkdir ' + self.device_dirs.bin_dir,
'shell', 'mkdir', '-p', self.device_dirs.bin_dir)
def _adb(self, title, *cmd, **kwargs):
if not self._ever_ran_adb:
@ -104,7 +103,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
def cleanup_steps(self):
if self._ever_ran_adb:
# To clean up disk space for next time
self._ssh('Delete executables', 'rm', '-r', self.m.vars.android_bin_dir,
self._ssh('Delete executables', 'rm', '-r', self.device_dirs.bin_dir,
abort_on_failure=False, infra_step=True)
# Reconnect if was disconnected
self._adb('disconnect', 'disconnect')
@ -125,7 +124,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
line = line.replace(addr, addr + ' ' + sym.strip())
print line
""",
args=[self.m.vars.skia_out.join(self.m.vars.configuration)],
args=[self.m.vars.skia_out],
infra_step=True,
abort_on_failure=False)
@ -145,10 +144,10 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils):
return self.m.run(self.m.step, title, cmd=ssh_cmd, **kwargs)
def step(self, name, cmd, **kwargs):
app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
app = self.m.vars.skia_out.join(cmd[0])
self._adb('push %s' % cmd[0],
'push', app, self.m.vars.android_bin_dir)
'push', app, self.device_dirs.bin_dir)
cmd[0] = '%s/%s' % (self.m.vars.android_bin_dir, cmd[0])
cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0])
self._ssh(str(name), *cmd, infra_step=False)

13
infra/bots/recipe_modules/flavor/gn_flavor.py
View File

@ -16,7 +16,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
infra_step=infra_step)
def step(self, name, cmd):
app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
app = self.device_dirs.bin_dir.join(cmd[0])
cmd = [app] + cmd[1:]
env = self.m.context.env
path = []
@ -43,7 +43,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
ld_library_path.append(slave_dir.join('linux_vulkan_sdk', 'lib'))
if 'SwiftShader' in extra_tokens:
ld_library_path.append(self.m.vars.skia_out.join('swiftshader_out'))
ld_library_path.append(
self.m.vars.build_dir.join('out', 'swiftshader_out'))
if 'MSAN' in extra_tokens:
# Find the MSAN-built libc++.
@ -60,7 +61,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
elif self.m.vars.is_linux:
cmd = ['catchsegv'] + cmd
elif 'ProcDump' in extra_tokens:
self.m.file.ensure_directory('makedirs dumps', self.m.vars.dumps_dir)
dumps_dir = self.m.path.join(self.m.vars.swarming_out_dir, 'dumps')
self.m.file.ensure_directory('makedirs dumps', dumps_dir)
procdump = str(self.m.vars.slave_dir.join('procdump_win',
'procdump64.exe'))
# Full docs for ProcDump here:
@ -70,8 +72,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
# -e 1 tells procdump to dump once
# -x <dump dir> <exe> <args> launches exe and writes dumps to the
# specified dir
cmd = [procdump, '-accepteula', '-mp', '-e', '1',
'-x', self.m.vars.dumps_dir] + cmd
cmd = [procdump, '-accepteula', '-mp', '-e', '1', '-x', dumps_dir] + cmd
if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens:
if 'Mac' in self.m.vars.builder_cfg.get('os', ''):
@ -104,7 +105,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
# Convert path objects or placeholders into strings such that they can
# be passed to symbolize_stack_trace.py
args = [slave_dir] + [str(x) for x in cmd]
with self.m.context(cwd=self.m.vars.skia_dir, env=env):
with self.m.context(cwd=self.m.path['start_dir'].join('skia'), env=env):
self._py('symbolized %s' % name,
self.module.resource('symbolize_stack_trace.py'),
args=args,

29
infra/bots/recipe_modules/flavor/ios_flavor.py
View File

@ -10,6 +10,17 @@ import gn_flavor
import os
class iOSFlavorUtils(gn_flavor.GNFlavorUtils):
def __init__(self, m):
super(iOSFlavorUtils, self).__init__(m)
self.device_dirs = default_flavor.DeviceDirs(
bin_dir='[unused]',
dm_dir='dm',
perf_data_dir='perf',
resource_dir='resources',
images_dir='images',
skp_dir='skps',
svg_dir='svgs',
tmp_dir='tmp')
def install(self):
# Set up the device
@ -17,8 +28,7 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils):
# Install the app.
for app_name in ['dm', 'nanobench']:
app_package = self.m.vars.skia_out.join(self.m.vars.configuration,
'%s.app' % app_name)
app_package = self.m.vars.skia_out.join('%s.app' % app_name)
def uninstall_app(attempt):
# If app ID changes, upgrade will fail, so try uninstalling.
@ -35,15 +45,6 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils):
between_attempts_fn=uninstall_app,
infra_step=True)
self.device_dirs = default_flavor.DeviceDirs(
dm_dir='dm',
perf_data_dir='perf',
resource_dir='resources',
images_dir='images',
skp_dir='skps',
svg_dir='svgs',
tmp_dir='tmp')
def step(self, name, cmd, env=None, **kwargs):
bundle_id = 'com.google.%s' % cmd[0]
self.m.run(self.m.step, name,
@ -51,7 +52,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils):
map(str, cmd[1:]))
def _run_ios_script(self, script, first, *rest):
full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_' + script)
full = self.m.path['start_dir'].join(
'skia', 'platform_tools', 'ios', 'bin', 'ios_' + script)
self.m.run(self.m.step,
name = '%s %s' % (script, first),
cmd = [full, first] + list(rest),
@ -74,7 +76,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils):
self._run_ios_script('mkdir', path)
def read_file_on_device(self, path, **kwargs):
full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_cat_file')
full = self.m.path['start_dir'].join(
'skia', 'platform_tools', 'ios', 'bin', 'ios_cat_file')
rv = self.m.run(self.m.step,
name = 'cat_file %s' % path,
cmd = [full, path],

6
infra/bots/recipe_modules/flavor/valgrind_flavor.py
View File

@ -12,8 +12,8 @@ import gn_flavor
class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils):
def __init__(self, m):
super(ValgrindFlavorUtils, self).__init__(m)
self._suppressions_file = self.m.vars.skia_dir.join(
'tools', 'valgrind.supp')
self._suppressions_file = self.m.path['start_dir'].join(
'skia', 'tools', 'valgrind.supp')
self._valgrind_cipd_dir = self.m.vars.slave_dir.join('valgrind')
self._valgrind_fake_dir = self._valgrind_cipd_dir
self._valgrind = self._valgrind_fake_dir.join('bin', 'valgrind')
@ -23,7 +23,7 @@ class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils):
new_cmd = [self._valgrind, '--gen-suppressions=all', '--leak-check=full',
'--track-origins=yes', '--error-exitcode=1', '--num-callers=40',
'--suppressions=%s' % self._suppressions_file]
path_to_app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
path_to_app = self.m.vars.skia_out.join(cmd[0])
new_cmd.append(path_to_app)
new_cmd.extend(cmd[1:])
with self.m.env({'VALGRIND_LIB': self._lib_dir}):

6
infra/bots/recipe_modules/run/api.py
View File

@ -51,14 +51,14 @@ class SkiaStepApi(recipe_api.RecipeApi):
"""Wrapper around api.file.rmtree."""
self.m.file.rmtree('rmtree %s' % self.m.path.basename(path), path)
def asset_version(self, asset_name, test_data=None):
def asset_version(self, asset_name, skia_dir, test_data=None):
"""Return the contents of VERSION for the given asset as a string.
If test_data is not specified, reads the property
'test_<asset_name>_version' or if not present, uses
TEST_DEFAULT_ASSET_VERSION."""
version_file = self.m.vars.infrabots_dir.join(
'assets', asset_name, 'VERSION')
version_file = skia_dir.join(
'infra', 'bots', 'assets', asset_name, 'VERSION')
if not test_data:
test_data = self.m.properties.get(
'test_%s_version' % asset_name, TEST_DEFAULT_ASSET_VERSION)

2
infra/bots/recipe_modules/run/examples/full.py
View File

@ -35,7 +35,7 @@ def RunSteps(api):
api.run.readfile('myfile.txt')
api.run.writefile('myfile.txt', 'contents')
api.run.rmtree('mydir')
api.run.asset_version('my_asset')
api.run.asset_version('my_asset', api.vars.cache_dir.join('work', 'skia'))
# Merge PATHs.
with api.context(env={'PATH': 'mydir:%(PATH)s'}):

132
infra/bots/recipe_modules/vars/api.py
View File

@ -15,9 +15,6 @@ CONFIG_RELEASE = 'Release'
class SkiaVarsApi(recipe_api.RecipeApi):
override_checkout_root = None
override_gclient_cache = None
def setup(self):
"""Prepare the variables."""
# Setup
@ -27,87 +24,19 @@ class SkiaVarsApi(recipe_api.RecipeApi):
# Special input/output directories.
self.build_dir = self.slave_dir.join('build')
self.test_dir = self.slave_dir.join('test')
self.perf_dir = self.slave_dir.join('perf')
self.checkout_root = self.slave_dir
self.default_env = self.m.context.env
self.default_env['CHROME_HEADLESS'] = '1'
self.default_env['PATH'] = self.m.path.pathsep.join([
self.default_env.get('PATH', '%(PATH)s'),
str(self.m.bot_update._module.PACKAGE_REPO_ROOT),
])
self.gclient_env = {'DEPOT_TOOLS_UPDATE': '0'}
self.is_compile_bot = self.builder_name.startswith('Build-')
self.persistent_checkout = False
# Compile bots keep a persistent checkout.
if self.is_compile_bot and 'NoDEPS' not in self.builder_name:
self.persistent_checkout = True
if 'Housekeeper' in self.builder_name:
self.persistent_checkout = True
if '-CT_' in self.builder_name:
self.persistent_checkout = True
# We need the source code for the Coverage's Upload step to be in the
# same absolute location as when we compiled it so we can map the
# coverage data to actual line numbers. We ensure this by making sure
# we have a checkout on the Coverage's Upload step and that the Upload
# step runs on the same bots that Compile.
if 'Coverage' in self.builder_name and 'Upload' in self.builder_name:
self.persistent_checkout = True
self.cache_dir = self.slave_dir.join('cache')
if self.persistent_checkout:
self.checkout_root = self.cache_dir.join('work')
self.gclient_cache = self.cache_dir.join('git')
if self.override_checkout_root:
self.checkout_root = self.override_checkout_root
self.gclient_cache = self.override_gclient_cache
# got_revision is filled in after checkout steps.
self.got_revision = None
else:
# If there's no persistent checkout, then we have to assume we got the
# correct revision of the files from isolate.
self.got_revision = self.m.properties['revision']
# Some bots also require a checkout of Flutter; in this case we use the
# checkout of Skia obtained through DEPS in src/third_party/skia.
self.need_flutter_checkout = 'Flutter' in self.builder_name
self.skia_dir = self.checkout_root.join('skia')
if self.need_flutter_checkout:
self.checkout_root = self.checkout_root.join('flutter')
self.skia_dir = self.checkout_root.join('src', 'third_party', 'skia')
if not self.persistent_checkout:
self.m.path['checkout'] = self.skia_dir
self.infrabots_dir = self.skia_dir.join('infra', 'bots')
self.resource_dir = self.skia_dir.join('resources')
self.images_dir = self.slave_dir.join('skimage')
self.skia_out = self.skia_dir.join('out', self.builder_name)
self.swarming_out_dir = self.slave_dir.join(
self.m.properties['swarm_out_dir'])
if 'ParentRevision' in self.builder_name:
# Tasks that depend on ParentRevision builds usually also depend on a
# second build task. Use a different path for build results so that the
# binaries end up in different directories in the isolate.
self.swarming_out_dir = self.swarming_out_dir.join('ParentRevision')
self.local_skp_dir = self.slave_dir.join('skp')
self.local_svg_dir = self.slave_dir.join('svg')
if not self.is_compile_bot:
self.skia_out = self.build_dir.join('out')
self.tmp_dir = self.m.path['start_dir'].join('tmp')
# Some bots also require a checkout of chromium.
self.need_chromium_checkout = False
if 'CommandBuffer' in self.builder_name:
self.need_chromium_checkout = True
self.gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0'
if 'RecreateSKPs' in self.builder_name:
self.need_chromium_checkout = True
self.gclient_env['CPPFLAGS'] = (
'-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1')
self.tmp_dir = self.m.path['start_dir'].join('tmp')
self.builder_cfg = self.m.builder_name_schema.DictForBuilderName(
self.builder_name)
@ -120,6 +49,9 @@ class SkiaVarsApi(recipe_api.RecipeApi):
arch = (self.builder_cfg.get('arch') or self.builder_cfg.get('target_arch'))
if ('Win' in self.builder_cfg.get('os', '') and arch == 'x86_64'):
self.configuration += '_x64'
self.skia_out = self.build_dir.join('out', self.configuration)
self.extra_tokens = []
if len(self.builder_cfg.get('extra_config', '')) > 0:
if self.builder_cfg['extra_config'].startswith('SK'):
@ -138,27 +70,9 @@ class SkiaVarsApi(recipe_api.RecipeApi):
self.issue = self.m.properties['patch_issue']
self.patchset = self.m.properties['patch_set']
self.dm_dir = self.m.path.join(
self.swarming_out_dir, 'dm')
self.perf_data_dir = self.m.path.join(self.swarming_out_dir,
'perfdata', self.builder_name, 'data')
self.dumps_dir = self.m.path.join(
self.swarming_out_dir, 'dumps')
self._swarming_bot_id = None
self._swarming_task_id = None
# Data should go under in _data_dir, which may be preserved across runs.
self.android_data_dir = '/sdcard/revenge_of_the_skiabot/'
# Executables go under _bin_dir, which, well, allows executable files.
self.android_bin_dir = '/data/local/tmp/'
if self.builder_cfg.get('os', '') == 'Chromecast':
# On the Chromecast, everything goes in the (~110M) /cache/skia
self.android_bin_dir = '/cache/skia/'
self.android_data_dir = '/cache/skia/'
self.chromeos_homedir = '/home/chronos/user/'
# Internal bot support.
self.internal_hardware_label = (
self.m.properties.get('internal_hardware_label'))
@ -168,44 +82,6 @@ class SkiaVarsApi(recipe_api.RecipeApi):
def is_linux(self):
return 'Ubuntu' in self.builder_name or 'Debian' in self.builder_name
@property
def upload_dm_results(self):
# TODO(borenet): Move this into the swarm_test recipe.
skip_upload_bots = [
'ASAN',
'Coverage',
'MSAN',
'TSAN',
'UBSAN',
'Valgrind',
]
upload_dm_results = True
for s in skip_upload_bots:
if s in self.m.properties['buildername']:
upload_dm_results = False
break
return upload_dm_results
@property
def upload_perf_results(self):
# TODO(borenet): Move this into the swarm_perf recipe.
if 'Release' not in self.m.properties['buildername']:
return False
skip_upload_bots = [
'ASAN',
'Coverage',
'MSAN',
'TSAN',
'UBSAN',
'Valgrind',
]
upload_perf_results = True
for s in skip_upload_bots:
if s in self.m.properties['buildername']:
upload_perf_results = False
break
return upload_perf_results
@property
def swarming_bot_id(self):
if not self._swarming_bot_id:

35
infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json
View File

@ -1,35 +0,0 @@
[
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
],
"name": "get swarming bot id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
],
"name": "get swarming task id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

35
infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json
View File

@ -1,35 +0,0 @@
[
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
],
"name": "get swarming bot id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
],
"name": "get swarming task id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

35
infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json
View File

@ -1,35 +0,0 @@
[
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
],
"name": "get swarming bot id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
],
"name": "get swarming task id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

7
infra/bots/recipe_modules/vars/examples/full.py
View File

@ -16,22 +16,17 @@ def RunSteps(api):
api.vars.override_gclient_cache = api.path['start_dir']
api.vars.setup()
info = [
api.vars.upload_dm_results,
api.vars.upload_perf_results,
api.vars.swarming_bot_id,
api.vars.swarming_task_id,
]
if api.vars.is_linux:
assert len(info) == 4 # Make pylint happy.
assert len(info) == 2 # Make pylint happy.
TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-Override',
'Build-Debian9-Clang-x86_64-Release-ParentRevision',
'Build-Debian9-Clang-x86_64-Release-SKNX_NO_SIMD',
'Build-Debian9-GCC-x86_64-Release-Flutter_Android',
'Build-Mac-Clang-x86_64-Debug-CommandBuffer',
'Build-Win-Clang-x86_64-Release-Vulkan',
'Housekeeper-Weekly-RecreateSKPs',
'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All',

13
infra/bots/recipes/bookmaker.py
View File

@ -39,13 +39,14 @@ def go_get_fiddlecli(api):
def RunSteps(api):
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
api.core.checkout_bot_update(checkout_root=checkout_root)
api.infra.go_version()
go_get_fiddlecli(api)
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
bookmaker_binary = api.path.join(api.vars.skia_out, api.vars.configuration,
'bookmaker')
skia_dir = checkout_root.join('skia')
with api.context(cwd=skia_dir, env=api.infra.go_env):
bookmaker_binary = api.vars.skia_out.join('bookmaker')
buildername = api.vars.builder_name
if 'PerCommit' in buildername:
@ -125,10 +126,10 @@ def RunSteps(api):
# Step 4: Update docs in site/user/api/ with the output of fiddlecli.
# If there are any new changes then upload and commit the changes.
cmd = ['python',
api.vars.skia_dir.join('infra', 'bots', 'upload_md.py'),
skia_dir.join('infra', 'bots', 'upload_md.py'),
'--bookmaker_binary', bookmaker_binary,
'--fiddlecli_output', fiddlecli_output]
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
with api.context(cwd=skia_dir, env=api.infra.go_env):
api.run(api.step, 'Generate and Upload Markdown files', cmd=cmd)

7
infra/bots/recipes/calmbench.py
View File

@ -28,7 +28,8 @@ def RunSteps(api):
api.flavor.install(skps=True, svgs=True)
api.file.ensure_directory('makedirs perf', api.vars.swarming_out_dir)
with api.context(cwd=api.vars.skia_dir):
skia_dir = api.path['start_dir'].join('skia')
with api.context(cwd=skia_dir):
extra_arg = '--svgs %s --skps %s' % (api.flavor.device_dirs.svg_dir,
api.flavor.device_dirs.skp_dir)
@ -41,7 +42,7 @@ def RunSteps(api):
command = [
'python',
api.vars.skia_dir.join('tools', 'calmbench', 'ab.py'),
skia_dir.join('tools', 'calmbench', 'ab.py'),
api.vars.swarming_out_dir,
'modified', 'master',
api.vars.build_dir.join("out", api.vars.configuration, 'nanobench'),
@ -53,7 +54,7 @@ def RunSteps(api):
config,
-1, # threads; let ab.py decide the threads
"false", # noinit
"--githash", api.vars.got_revision,
"--githash", api.properties['revision'],
"--concise"
]

6
infra/bots/recipes/check_generated_files.py
View File

@ -24,7 +24,8 @@ DEPS = [
def RunSteps(api):
# Checkout, compile, etc.
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
api.core.checkout_bot_update(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
api.flavor.setup()
@ -55,7 +56,8 @@ for r, d, files in os.walk('%s'):
""" % cwd)
# Regenerate the SKSL files.
api.build()
api.build(checkout_root=checkout_root,
out_dir=api.vars.build_dir.join('out', 'Release'))
# Get a second diff. If this doesn't match the first, then there have been
# modifications to the generated files.

272
infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json Normal file
View File

@ -0,0 +1,272 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_PACKAGE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/android_ndk_linux/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get android_ndk_linux VERSION"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/cache/work/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/go/go/bin/go",
"version"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOPATH": "[START_DIR]/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "go version"
},
{
"cmd": [
"go",
"version"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOPATH": "[START_DIR]/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "env go version"
},
{
"cmd": [
"[START_DIR]/go/go/bin/go",
"get",
"-u",
"-t",
"go.skia.org/infra/..."
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOPATH": "[START_DIR]/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "update go pkgs"
},
{
"cmd": [
"go",
"build",
"-o",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel/run_testlab",
"[START_DIR]/cache/work/skia/infra/cts/run_testlab.go"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOPATH": "[START_DIR]/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "build firebase runner"
},
{
"cmd": [
"[START_DIR]/cache/work/skia/tools/skqp/make_universal_apk"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"ANDROID_HOME": "[START_DIR]/android_sdk_linux/android-sdk",
"ANDROID_NDK": "[START_DIR]/android_ndk_linux",
"APK_OUTPUT_DIR": "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "make_universal"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel",
"[START_DIR]/[SWARM_OUT_DIR]/out/devrel"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json",
"[START_DIR]/[SWARM_OUT_DIR]/out/devrel"
],
"infra_step": true,
"name": "copy whitelist"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

211
infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json Normal file
View File

@ -0,0 +1,211 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123^"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_PACKAGE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123^\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/clang_linux/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get clang_linux VERSION"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/cache/work/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false target_cpu=\"x86_64\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release",
"[START_DIR]/[SWARM_OUT_DIR]/ParentRevision/out/Release"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

4
infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json
View File

@ -170,7 +170,7 @@
"cmd": [
"ninja",
"-C",
"out/android_release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"-j100"
],
"cwd": "[START_DIR]/cache/work/flutter/src",
@ -185,7 +185,7 @@
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"[START_DIR]/[SWARM_OUT_DIR]/out/Release"
],
"infra_step": true,

4
infra/bots/recipes/compile.expected/flutter_trybot.json
View File

@ -174,7 +174,7 @@
"cmd": [
"ninja",
"-C",
"out/android_release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"-j100"
],
"cwd": "[START_DIR]/cache/work/flutter/src",
@ -189,7 +189,7 @@
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release",
"[START_DIR]/cache/work/flutter/src/out/android_release",
"[START_DIR]/[SWARM_OUT_DIR]/out/Release"
],
"infra_step": true,

35
infra/bots/recipes/compile.py
View File

@ -27,16 +27,35 @@ def RunSteps(api):
# Check out code.
if 'NoDEPS' in api.properties['buildername']:
api.core.checkout_git()
checkout_root = api.path['start_dir']
api.core.checkout_git(checkout_root=checkout_root)
else:
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
if 'Flutter' in api.vars.builder_name:
checkout_root = checkout_root.join('flutter')
api.core.checkout_bot_update(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
out_dir = checkout_root.join(
'skia', 'out', api.vars.builder_name, api.vars.configuration)
if 'Flutter' in api.vars.builder_name:
out_dir = checkout_root.join('src', 'out', 'android_release')
try:
api.build()
api.build.copy_build_products(
api.vars.swarming_out_dir.join(
'out', api.vars.configuration))
api.build(checkout_root=checkout_root, out_dir=out_dir)
# TODO(borenet): Move this out of the try/finally.
dst = api.vars.swarming_out_dir.join('out', api.vars.configuration)
if 'ParentRevision' in api.vars.builder_name:
dst = api.vars.swarming_out_dir.join(
'ParentRevision', 'out', api.vars.configuration)
api.build.copy_build_products(out_dir=out_dir, dst=dst)
if 'SKQP' in api.vars.extra_tokens:
wlist = checkout_root.join(
'skia', 'infra','cts', 'whitelist_devices.json')
api.file.copy('copy whitelist', wlist, dst)
finally:
if 'Win' in api.vars.builder_cfg.get('os', ''):
api.python.inline(
@ -57,8 +76,9 @@ for p in psutil.process_iter():
TEST_BUILDERS = [
'Build-Debian9-Clang-arm-Release-Chromebook_GLES',
'Build-Debian9-Clang-arm64-Release-Android',
'Build-Debian9-Clang-arm64-Release-Android_Vulkan',
'Build-Debian9-Clang-arm64-Release-Android_ASAN',
'Build-Debian9-Clang-arm64-Release-Android_Vulkan',
'Build-Debian9-Clang-universal-devrel-Android_SKQP',
'Build-Debian9-Clang-x86_64-Debug',
'Build-Debian9-Clang-x86_64-Debug-ASAN',
'Build-Debian9-Clang-x86_64-Debug-Coverage',
@ -68,6 +88,7 @@ TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-Fast',
'Build-Debian9-Clang-x86_64-Release-Mini',
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-ParentRevision',
'Build-Debian9-Clang-x86_64-Release-Vulkan',
'Build-Debian9-Clang-x86_64-Release-Vulkan_Coverage',
'Build-Debian9-EMCC-wasm-Release',

23
infra/bots/recipes/ct_skps.py
View File

@ -75,21 +75,22 @@ def RunSteps(api):
else:
raise Exception('Do not recognise the buildername %s.' % buildername)
api.vars.override_checkout_root = make_path(api, '/', 'b', 'work')
api.vars.override_gclient_cache = make_path(api, '/', 'b', 'cache')
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = make_path(api, '/', 'b', 'work')
gclient_cache = make_path(api, '/', 'b', 'cache')
got_revision = api.core.checkout_bot_update(checkout_root=checkout_root,
gclient_cache=gclient_cache)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
api.build()
out_dir = api.vars.build_dir.join('out', api.vars.configuration)
api.build(checkout_root=checkout_root, out_dir=out_dir)
# Required paths.
infrabots_dir = api.vars.skia_dir.join('infra', 'bots')
infrabots_dir = checkout_root.join('skia', 'infra', 'bots')
isolate_dir = infrabots_dir.join('ct')
isolate_path = isolate_dir.join(CT_SKPS_ISOLATE)
api.build.copy_build_products(isolate_dir)
api.build.copy_build_products(out_dir=out_dir, dst=isolate_dir)
api.skia_swarming.setup(
infrabots_dir.join('tools', 'luci-go'),
swarming_rev='')
@ -131,8 +132,8 @@ def RunSteps(api):
# referenced also needs to change. As of 8/8/17 the other places are:
# * infra/bots/ct/ct_skps.isolate
# * infra/bots/ct/run_ct_skps.py
skps_dir = api.vars.checkout_root.join('skps', skps_chromium_build,
ct_page_type, str(ct_num_slaves))
skps_dir = checkout_root.join('skps', skps_chromium_build,
ct_page_type, str(ct_num_slaves))
version_file = skps_dir.join(SKPS_VERSION_FILE)
if api.path.exists(version_file): # pragma: nocover
version_file_contents = api.file.read_text(
@ -150,7 +151,7 @@ def RunSteps(api):
'makedirs %s' % api.path.basename(skps_dir), skps_dir)
# If a blacklist file exists then specify SKPs to be blacklisted.
blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists')
blacklists_dir = infrabots_dir.join('ct', 'blacklists')
blacklist_file = blacklists_dir.join(
'%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build))
blacklist_skps = []
@ -173,7 +174,7 @@ def RunSteps(api):
extra_variables = {
'SLAVE_NUM': str(slave_num),
'TOOL_NAME': skia_tool,
'GIT_HASH': api.vars.got_revision,
'GIT_HASH': got_revision,
'CONFIGURATION': api.vars.configuration,
'BUILDER': buildername,
'CHROMIUM_BUILD': skps_chromium_build,

13
infra/bots/recipes/housekeeper.py
View File

@ -28,15 +28,14 @@ DEPS = [
def RunSteps(api):
# Checkout, compile, etc.
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
got_revision = api.core.checkout_bot_update(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
api.flavor.setup()
cwd = api.path['checkout']
# TODO(borenet): Detect static initializers?
with api.context(cwd=cwd):
with api.context(cwd=checkout_root.join('skia')):
if not api.vars.is_trybot:
api.run(
api.step,
@ -46,12 +45,12 @@ def RunSteps(api):
now = api.time.utcnow()
ts = int(calendar.timegm(now.utctimetuple()))
filename = 'nanobench_%s_%d.json' % (api.vars.got_revision, ts)
dest_dir = api.vars.perf_data_dir
filename = 'nanobench_%s_%d.json' % (got_revision, ts)
dest_dir = api.flavor.host_dirs.perf_data_dir
dest_file = dest_dir + '/' + filename
api.file.ensure_directory('makedirs perf_dir', dest_dir)
cmd = ['python', api.core.resource('run_binary_size_analysis.py'),
'--library', api.vars.skia_out.join('Release', 'libskia.so'),
'--library', api.vars.skia_out.join('libskia.so'),
'--githash', api.properties['revision'],
'--dest', dest_file]
if api.vars.is_trybot:

5
infra/bots/recipes/infra.py
View File

@ -20,14 +20,15 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
api.core.checkout_bot_update(checkout_root=checkout_root)
api.infra.update_go_deps()
# Run the infra tests.
repo_name = api.properties['repository'].split('/')[-1]
if repo_name.endswith('.git'):
repo_name = repo_name[:-len('.git')]
with api.context(cwd=api.vars.checkout_root.join(repo_name),
with api.context(cwd=checkout_root.join(repo_name),
env=api.infra.go_env):
api.step('infra_tests', cmd=['make', '-C', 'infra/bots', 'test'])

35
infra/bots/recipes/perf.py
View File

@ -27,6 +27,23 @@ DEPS = [
]
def upload_perf_results(buildername):
if 'Release' not in buildername:
return False
skip_upload_bots = [
'ASAN',
'Coverage',
'MSAN',
'TSAN',
'UBSAN',
'Valgrind',
]
for s in skip_upload_bots:
if s in buildername:
return False
return True
def nanobench_flags(api, bot):
args = ['--pre_log']
@ -240,14 +257,15 @@ def nanobench_flags(api, bot):
def perf_steps(api):
"""Run Skia benchmarks."""
if api.vars.upload_perf_results:
b = api.properties['buildername']
if upload_perf_results(b):
api.flavor.create_clean_device_dir(
api.flavor.device_dirs.perf_data_dir)
# Run nanobench.
properties = [
'--properties',
'gitHash', api.vars.got_revision,
'gitHash', api.properties['revision'],
]
if api.vars.is_trybot:
properties.extend([
@ -297,12 +315,12 @@ def perf_steps(api):
'~shapes_rrect_inner_rrect_50_500x500', # skia:7551
])
if api.vars.upload_perf_results:
if upload_perf_results(b):
now = api.time.utcnow()
ts = int(calendar.timegm(now.utctimetuple()))
json_path = api.flavor.device_path_join(
api.flavor.device_dirs.perf_data_dir,
'nanobench_%s_%d.json' % (api.vars.got_revision, ts))
'nanobench_%s_%d.json' % (api.properties['revision'], ts))
args.extend(['--outResultsFile', json_path])
args.extend(properties)
@ -320,12 +338,13 @@ def perf_steps(api):
abort_on_failure=False)
# Copy results to swarming out dir.
if api.vars.upload_perf_results:
api.file.ensure_directory('makedirs perf_dir',
api.path.dirname(api.vars.perf_data_dir))
if upload_perf_results(b):
api.file.ensure_directory(
'makedirs perf_dir',
api.path.dirname(api.flavor.host_dirs.perf_data_dir))
api.flavor.copy_directory_contents_to_host(
api.flavor.device_dirs.perf_data_dir,
api.vars.perf_data_dir)
api.flavor.host_dirs.perf_data_dir)
def RunSteps(api):

14
infra/bots/recipes/recreate_skps.py
View File

@ -36,11 +36,13 @@ TEST_BUILDERS = {
def RunSteps(api):
# Check out Chrome.
api.vars.setup()
api.core.checkout_bot_update()
checkout_root = api.core.default_checkout_root
api.core.checkout_bot_update(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
api.flavor.setup()
src_dir = api.vars.checkout_root.join('src')
src_dir = checkout_root.join('src')
skia_dir = checkout_root.join('skia')
out_dir = src_dir.join('out', 'Release')
with api.context(cwd=src_dir):
@ -62,7 +64,7 @@ def RunSteps(api):
api.file.ensure_directory('makedirs skp_output', output_dir)
# Capture the SKPs.
asset_dir = api.vars.infrabots_dir.join('assets', 'skp')
asset_dir = skia_dir.join('infra', 'bots', 'assets', 'skp')
cmd = ['python', asset_dir.join('create.py'),
'--chrome_src_path', src_dir,
'--browser_executable', src_dir.join('out', 'Release', 'chrome'),
@ -70,16 +72,16 @@ def RunSteps(api):
# TODO(rmistry): Uncomment the below after skbug.com/6797 is fixed.
# if 'Canary' not in api.properties['buildername']:
# cmd.append('--upload_to_partner_bucket')
with api.context(cwd=api.vars.skia_dir):
with api.context(cwd=skia_dir):
api.run(api.step, 'Recreate SKPs', cmd=cmd)
# Upload the SKPs.
if 'Canary' not in api.properties['buildername']:
api.infra.update_go_deps()
cmd = ['python',
api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
skia_dir.join('infra', 'bots', 'upload_skps.py'),
'--target_dir', output_dir]
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
with api.context(cwd=skia_dir, env=api.infra.go_env):
api.run(api.step, 'Upload SKPs', cmd=cmd)

25
infra/bots/recipes/skpbench.py
View File

@ -10,7 +10,6 @@ import calendar
DEPS = [
'core',
'flavor',
'recipe_engine/context',
'recipe_engine/file',
@ -28,7 +27,7 @@ ADB_BINARY = 'adb.1.0.35'
def _run(api, title, *cmd, **kwargs):
with api.context(cwd=api.vars.skia_dir):
with api.context(cwd=api.path['start_dir'].join('skia')):
return api.run(api.step, title, cmd=list(cmd), **kwargs)
@ -40,10 +39,11 @@ def _adb(api, title, *cmd, **kwargs):
def skpbench_steps(api):
"""benchmark Skia using skpbench."""
api.file.ensure_directory('makedirs perf_dir', api.vars.perf_data_dir)
api.file.ensure_directory(
'makedirs perf_dir', api.flavor.host_dirs.perf_data_dir)
app = api.vars.skia_out.join(api.vars.configuration, 'skpbench')
_adb(api, 'push skpbench', 'push', app, api.vars.android_bin_dir)
app = api.vars.skia_out.join('skpbench')
_adb(api, 'push skpbench', 'push', app, api.flavor.device_dirs.bin_dir)
skpbench_dir = api.vars.slave_dir.join('skia', 'tools', 'skpbench')
table = api.path.join(api.vars.swarming_out_dir, 'table')
@ -54,7 +54,7 @@ def skpbench_steps(api):
config = 'gles'
skpbench_args = [
api.path.join(api.vars.android_bin_dir, 'skpbench'),
api.path.join(api.flavor.device_dirs.bin_dir, 'skpbench'),
'--adb',
'--adb_binary', ADB_BINARY,
'--resultsfile', table,
@ -65,11 +65,10 @@ def skpbench_steps(api):
skpbench_args += [
'--pr', 'ccpr',
'--nocache',
api.path.join(api.vars.android_data_dir, 'skps/desk_*svg.skp'),
api.path.join(api.vars.android_data_dir, 'skps/desk_chalkboard.skp')]
api.path.join(api.flavor.device_dirs.skp_dir, 'desk_*svg.skp'),
api.path.join(api.flavor.device_dirs.skp_dir, 'desk_chalkboard.skp')]
else:
skpbench_args += [
api.path.join(api.vars.android_data_dir, 'skps')]
skpbench_args += [api.flavor.device_dirs.skp_dir]
api.run(api.python, 'skpbench',
script=skpbench_dir.join('skpbench.py'),
@ -78,7 +77,7 @@ def skpbench_steps(api):
skiaperf_args = [
table,
'--properties',
'gitHash', api.vars.got_revision,
'gitHash', api.properties['revision'],
]
if api.vars.is_trybot:
skiaperf_args.extend([
@ -93,8 +92,8 @@ def skpbench_steps(api):
now = api.time.utcnow()
ts = int(calendar.timegm(now.utctimetuple()))
json_path = api.path.join(
api.vars.perf_data_dir,
'skpbench_%s_%d.json' % (api.vars.got_revision, ts))
api.flavor.host_dirs.perf_data_dir,
'skpbench_%s_%d.json' % (api.properties['revision'], ts))
skiaperf_args.extend([
'--outfile', json_path

30
infra/bots/recipes/test.py
View File

@ -24,6 +24,21 @@ DEPS = [
]
def upload_dm_results(buildername):
skip_upload_bots = [
'ASAN',
'Coverage',
'MSAN',
'TSAN',
'UBSAN',
'Valgrind',
]
for s in skip_upload_bots:
if s in buildername:
return False
return True
def dm_flags(api, bot):
args = []
configs = []
@ -828,10 +843,11 @@ def key_params(api):
def test_steps(api):
"""Run the DM test."""
b = api.properties['buildername']
use_hash_file = False
if api.vars.upload_dm_results:
host_dm_dir = str(api.vars.dm_dir)
api.flavor.create_clean_host_dir(api.vars.test_dir)
if upload_dm_results(b):
host_dm_dir = str(api.flavor.host_dirs.dm_dir)
api.flavor.create_clean_host_dir(api.path['start_dir'].join('test'))
device_dm_dir = str(api.flavor.device_dirs.dm_dir)
if host_dm_dir != device_dm_dir:
api.flavor.create_clean_device_dir(device_dm_dir)
@ -887,7 +903,7 @@ def test_steps(api):
# Run DM.
properties = [
'gitHash', api.vars.got_revision,
'gitHash', api.properties['revision'],
'builder', api.vars.builder_name,
'buildbucket_build_id', api.properties.get('buildbucket_build_id', ''),
]
@ -931,7 +947,7 @@ def test_steps(api):
if use_hash_file:
args.extend(['--uninterestingHashesFile', hashes_file])
if api.vars.upload_dm_results:
if upload_dm_results(b):
args.extend(['--writePath', api.flavor.device_dirs.dm_dir])
args.extend(dm_flags(api, api.vars.builder_name))
@ -946,10 +962,10 @@ def test_steps(api):
api.run(api.flavor.step, 'dm', cmd=args, abort_on_failure=False)
if api.vars.upload_dm_results:
if upload_dm_results(b):
# Copy images and JSON to host machine if needed.
api.flavor.copy_directory_contents_to_host(
api.flavor.device_dirs.dm_dir, api.vars.dm_dir)
api.flavor.device_dirs.dm_dir, api.flavor.host_dirs.dm_dir)
def RunSteps(api):

4
infra/bots/recipes/upload_calmbench_results.py
View File

@ -27,7 +27,7 @@ def FindFile(api, suffix):
with api.context(cwd=api.path['start_dir']):
results = api.file.glob_paths(
'find %s results' % suffix,
api.vars.perf_dir,
api.path['start_dir'].join('perf'),
'*.%s' % suffix,
test_data=['bench_modified_master.%s' % suffix])
if len(results) != 1: # pragma: nocover
@ -48,7 +48,7 @@ def RunSteps(api):
csv_src = FindFile(api, "csv")
ts = int(calendar.timegm(now.utctimetuple()))
basename = "bench_modified_master_%s_%d" % (api.vars.got_revision, ts)
basename = "bench_modified_master_%s_%d" % (api.properties['revision'], ts)
gs_path = '/'.join((
'calmbench-v1', str(now.year).zfill(4),