AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Recipe fixes to support try jobs in task scheduler

Browse Source 
BUG=skia:5723
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2410843002

Review-Url: https://codereview.chromium.org/2410843002
This commit is contained in:
borenet 2016-10-13 06:23:45 -07:00 committed by Commit bot
parent c7165c239a
commit 98b2e7a9e4
8 changed files with 639 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
infra/bots/gen_tasks.go
View File

@ -246,12 +246,18 @@ func compile(cfg *specs.TasksCfg, name string, parts map[string]string) string {
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_compile",
"repository=skia",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER),
fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "compile_skia.isolate",
Priority: 0.8,
@ -308,12 +314,18 @@ func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_test",
"repository=skia",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER),
fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "test_skia.isolate",
Priority: 0.8,
@ -326,12 +338,18 @@ func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask
Dimensions: UPLOAD_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_dm_results",
"repository=skia",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER),
fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "upload_dm_results.isolate",
Priority: 0.8,
@ -350,12 +368,18 @@ func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_perf",
"repository=skia",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER),
fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "perf_skia.isolate",
Priority: 0.8,
@ -368,12 +392,18 @@ func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask
Dimensions: UPLOAD_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_nano_results",
"repository=skia",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER),
fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "upload_nano_results.isolate",
Priority: 0.8,

23
infra/bots/recipe_modules/vars/api.py
View File

@ -34,6 +34,7 @@ class SkiaVarsApi(recipe_api.RecipeApi):
self.default_env = {}
self.gclient_env = {}
self.is_compile_bot = self.builder_name.startswith('Build-')
self.no_buildbot = self.m.properties.get('nobuildbot', '') == 'True'
self.default_env['CHROME_HEADLESS'] = '1'
# The 'depot_tools' directory comes from recipe DEPS and isn't provided by
@ -106,17 +107,27 @@ class SkiaVarsApi(recipe_api.RecipeApi):
self.default_env.update({'SKIA_OUT': self.skia_out,
'BUILDTYPE': self.configuration})
self.is_trybot = self.builder_cfg['is_trybot']
self.patch_storage = self.m.properties.get('patch_storage', 'rietveld')
self.issue = None
self.patchset = None
if self.is_trybot:
if self.patch_storage == 'gerrit':
self.issue = self.m.properties['event.change.number']
self.patchset = self.m.properties['event.patchSet.ref'].split('/')[-1]
else:
if self.no_buildbot:
self.is_trybot = (
self.m.properties.get('issue', '') and
self.m.properties.get('patchset', ''))
if self.is_trybot:
self.issue = self.m.properties['issue']
self.patchset = self.m.properties['patchset']
else:
self.is_trybot = self.builder_cfg['is_trybot']
if self.is_trybot:
if self.patch_storage == 'gerrit':
self.issue = self.m.properties['event.change.number']
self.patchset = self.m.properties['event.patchSet.ref'].split('/')[-1]
else:
self.issue = self.m.properties['issue']
self.patchset = self.m.properties['patchset']
self.dm_dir = self.m.path.join(
self.swarming_out_dir, 'dm')
self.perf_data_dir = self.m.path.join(self.swarming_out_dir,

250
infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_gerrit.json Normal file
View File

@ -0,0 +1,250 @@
[
{
"cmd": [
"python",
"-u",
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
"[CUSTOM_C:\\_B_WORK]",
"511"
],
"name": "makedirs checkout_path",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"config",
"--spec",
"cache_dir = '[CUSTOM_C:\\\\_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient setup"
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"sync",
"--verbose",
"--with_branch_heads",
"--nohooks",
"-j2",
"--reset",
"--force",
"--upstream",
"--no-nag-max",
"--delete_unversioned_trees",
"--revision",
"skia@abc123",
"--output-json",
"/path/to/tmp/json"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient sync",
"~followup_annotations": [
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"recurse",
"git",
"config",
"user.name",
"local_bot"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient recurse (git config user.name)"
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"recurse",
"git",
"config",
"user.email",
"local_bot@example.com"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient recurse (git config user.email)"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]\\resources\\apply_gerrit.py",
"--gerrit_repo",
"skia",
"--gerrit_ref",
"",
"--root",
"[CUSTOM_C:\\_B_WORK]\\skia"
],
"env": {
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "apply_gerrit"
},
{
"cmd": [
"python",
"-u",
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
"[SLAVE_BUILD]\\tmp",
"511"
],
"name": "makedirs tmp_dir",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"[CUSTOM_C:\\_B_WORK]\\skia\\infra\\bots\\bootstrap_win_toolchain_json.py",
"--win_toolchain_json",
"[SLAVE_BUILD]\\src\\build\\win_toolchain.json",
"--depot_tools_parent_dir",
"[SLAVE_BUILD]"
],
"name": "bootstrap win toolchain"
},
{
"cmd": [
"python",
"make.py",
"most"
],
"cwd": "[CUSTOM_C:\\_B_WORK]\\skia",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"CHROME_PATH": "[SLAVE_BUILD]\\src",
"GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "build most"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug\\Debug",
"[CUSTOM_[SWARM_OUT_DIR]]\\out\\Debug"
],
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n"
],
"name": "cleanup",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import psutil@@@",
"@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@",
"@@@STEP_LOG_LINE@python.inline@ try:@@@",
"@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@",
"@@@STEP_LOG_LINE@python.inline@ p.kill()@@@",
"@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@",
"@@@STEP_LOG_LINE@python.inline@ pass@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

253
infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_rietveld.json Normal file
View File

@ -0,0 +1,253 @@
[
{
"cmd": [
"python",
"-u",
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
"[CUSTOM_C:\\_B_WORK]",
"511"
],
"name": "makedirs checkout_path",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"config",
"--spec",
"cache_dir = '[CUSTOM_C:\\\\_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient setup"
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"sync",
"--verbose",
"--with_branch_heads",
"--nohooks",
"-j2",
"--reset",
"--force",
"--upstream",
"--no-nag-max",
"--delete_unversioned_trees",
"--revision",
"skia@abc123",
"--output-json",
"/path/to/tmp/json"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient sync",
"~followup_annotations": [
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"recurse",
"git",
"config",
"user.name",
"local_bot"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient recurse (git config user.name)"
},
{
"cmd": [
"python",
"-u",
"RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py",
"recurse",
"git",
"config",
"user.email",
"local_bot@example.com"
],
"cwd": "[CUSTOM_C:\\_B_WORK]",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "gclient recurse (git config user.email)"
},
{
"cmd": [
"python",
"-u",
"[DEPOT_TOOLS]\\apply_issue.py",
"-r",
"[CUSTOM_C:\\_B_WORK]\\skia",
"-i",
"500",
"-p",
"1",
"-s",
"https://codereview.chromium.org",
"--no-auth"
],
"name": "apply_issue",
"~followup_annotations": [
"@@@STEP_LINK@Applied issue 500@https://codereview.chromium.org/500@@@"
]
},
{
"cmd": [
"python",
"-u",
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
"[SLAVE_BUILD]\\tmp",
"511"
],
"name": "makedirs tmp_dir",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"[CUSTOM_C:\\_B_WORK]\\skia\\infra\\bots\\bootstrap_win_toolchain_json.py",
"--win_toolchain_json",
"[SLAVE_BUILD]\\src\\build\\win_toolchain.json",
"--depot_tools_parent_dir",
"[SLAVE_BUILD]"
],
"name": "bootstrap win toolchain"
},
{
"cmd": [
"python",
"make.py",
"most"
],
"cwd": "[CUSTOM_C:\\_B_WORK]\\skia",
"env": {
"BUILDTYPE": "Debug",
"CHROME_HEADLESS": "1",
"CHROME_PATH": "[SLAVE_BUILD]\\src",
"GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0",
"PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug"
},
"name": "build most"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug\\Debug",
"[CUSTOM_[SWARM_OUT_DIR]]\\out\\Debug"
],
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n"
],
"name": "cleanup",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import psutil@@@",
"@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@",
"@@@STEP_LOG_LINE@python.inline@ try:@@@",
"@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@",
"@@@STEP_LOG_LINE@python.inline@ p.kill()@@@",
"@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@",
"@@@STEP_LOG_LINE@python.inline@ pass@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

38
infra/bots/recipes/swarm_compile.py
View File

@ -319,3 +319,41 @@ def GenTests(api):
**gerrit_kwargs) +
api.platform('win', 64)
)
yield (
api.test('buildbotless_trybot_rietveld') +
api.properties(
repository='skia',
buildername=buildername,
mastername=mastername,
slavename=slavename,
buildnumber=5,
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]',
revision='abc123',
nobuildbot='True',
issue=500,
patchset=1,
patch_storage='rietveld',
rietveld='https://codereview.chromium.org') +
api.platform('win', 64)
)
yield (
api.test('buildbotless_trybot_gerrit') +
api.properties(
repository='skia',
buildername=buildername,
mastername=mastername,
slavename=slavename,
buildnumber=5,
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]',
revision='abc123',
nobuildbot='True',
issue=500,
patchset=1,
patch_storage='gerrit',
gerrit='https://skia-review.googlesource.com') +
api.platform('win', 64)
)

24
infra/bots/recipes/upload_dm_results.py
View File

@ -49,17 +49,6 @@ def RunSteps(api):
builder_name = api.properties['buildername']
revision = api.properties['revision']
patch_storage = api.properties.get('patch_storage', 'rietveld')
issue = None
patchset = None
if builder_name.endswith('-Trybot'):
if patch_storage == 'gerrit':
issue = str(api.properties['event.change.number'])
patchset = str(api.properties['event.patchSet.ref'].split('/')[-1])
else:
issue = str(api.properties['issue'])
patchset = str(api.properties['patchset'])
results_dir = api.path['cwd'].join('dm')
# Move dm.json and verbose.log to their own directory.
@ -95,10 +84,15 @@ def RunSteps(api):
str(int(calendar.timegm(now.utctimetuple())))])
# Trybot results are further siloed by issue/patchset.
if builder_name.endswith('-Trybot'):
if not (issue and patchset): # pragma: nocover
raise Exception('issue and patchset properties are required for trybots.')
summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset))
issue = str(api.properties.get('issue', ''))
patchset = str(api.properties.get('patchset', ''))
if (api.properties.get('patch_storage', '') == 'gerrit' and
api.properties.get('nobuildbot', '') != 'True'):
issue = str(api.properties['event.change.number'])
patchset = str(api.properties['event.patchSet.ref']).split('/')[-1]
if issue and patchset:
summary_dest_path = '/'.join((
'trybot', summary_dest_path, issue, patchset))
summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path))

21
infra/bots/recipes/upload_nano_results.py
View File

@ -19,17 +19,6 @@ def RunSteps(api):
# Upload the nanobench resuls.
builder_name = api.properties['buildername']
patch_storage = api.properties.get('patch_storage', 'rietveld')
issue = None
patchset = None
if builder_name.endswith('-Trybot'):
if patch_storage == 'gerrit':
issue = str(api.properties['event.change.number'])
patchset = str(api.properties['event.patchSet.ref'].split('/')[-1])
else:
issue = str(api.properties['issue'])
patchset = str(api.properties['patchset'])
now = api.time.utcnow()
src_path = api.path['cwd'].join(
@ -50,9 +39,13 @@ def RunSteps(api):
str(now.month).zfill(2), str(now.day).zfill(2), str(now.hour).zfill(2),
builder_name))
if builder_name.endswith('-Trybot'):
if not (issue and patchset): # pragma: nocover
raise Exception('issue and patchset properties are required for trybots.')
issue = str(api.properties.get('issue', ''))
patchset = str(api.properties.get('patchset', ''))
if (api.properties.get('patch_storage', '') == 'gerrit' and
api.properties.get('nobuildbot', '') != 'True'):
issue = str(api.properties['event.change.number'])
patchset = str(api.properties['event.patchSet.ref']).split('/')[-1]
if issue and patchset:
gs_path = '/'.join(('trybot', gs_path, issue, patchset))
dst = '/'.join(('gs://skia-perf', gs_path, basename))

40
infra/bots/tasks.json
View File

@ -30,12 +30,18 @@
"--workdir",
"../../..",
"swarm_compile",
"repository=skia",
"buildername=Build-Ubuntu-GCC-x86_64-Release-GN",
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)"
"revision=<(REVISION)",
"patch_storage=<(PATCH_STORAGE)",
"<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)",
"issue=<(ISSUE)",
"patchset=<(PATCHSET)"
],
"isolate": "compile_skia.isolate",
"priority": 0.8
@ -71,12 +77,18 @@
"--workdir",
"../../..",
"swarm_perf",
"repository=skia",
"buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)"
"revision=<(REVISION)",
"patch_storage=<(PATCH_STORAGE)",
"<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)",
"issue=<(ISSUE)",
"patchset=<(PATCHSET)"
],
"isolate": "perf_skia.isolate",
"priority": 0.8
@ -112,12 +124,18 @@
"--workdir",
"../../..",
"swarm_test",
"repository=skia",
"buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)"
"revision=<(REVISION)",
"patch_storage=<(PATCH_STORAGE)",
"<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)",
"issue=<(ISSUE)",
"patchset=<(PATCHSET)"
],
"isolate": "test_skia.isolate",
"priority": 0.8
@ -136,12 +154,18 @@
"--workdir",
"../../..",
"upload_nano_results",
"repository=skia",
"buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)"
"revision=<(REVISION)",
"patch_storage=<(PATCH_STORAGE)",
"<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)",
"issue=<(ISSUE)",
"patchset=<(PATCHSET)"
],
"isolate": "upload_nano_results.isolate",
"priority": 0.8
@ -160,12 +184,18 @@
"--workdir",
"../../..",
"upload_dm_results",
"repository=skia",
"buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)"
"revision=<(REVISION)",
"patch_storage=<(PATCH_STORAGE)",
"<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)",
"issue=<(ISSUE)",
"patchset=<(PATCHSET)"
],
"isolate": "upload_dm_results.isolate",
"priority": 0.8