diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 0751124368..25bcc5d2e8 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -246,12 +246,18 @@ func compile(cfg *specs.TasksCfg, name string, parts map[string]string) string { Dimensions: swarmDimensions(parts), ExtraArgs: []string{ "--workdir", "../../..", "swarm_compile", + "repository=skia", fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER), + fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "compile_skia.isolate", Priority: 0.8, @@ -308,12 +314,18 @@ func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask Dimensions: swarmDimensions(parts), ExtraArgs: []string{ "--workdir", "../../..", "swarm_test", + "repository=skia", fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER), + fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "test_skia.isolate", Priority: 0.8, @@ -326,12 +338,18 @@ func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask Dimensions: UPLOAD_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_dm_results", + "repository=skia", fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER), + fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "upload_dm_results.isolate", Priority: 0.8, @@ -350,12 +368,18 @@ func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask Dimensions: swarmDimensions(parts), ExtraArgs: []string{ "--workdir", "../../..", "swarm_perf", + "repository=skia", fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER), + fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "perf_skia.isolate", Priority: 0.8, @@ -368,12 +392,18 @@ func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTask Dimensions: UPLOAD_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_nano_results", + "repository=skia", fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("%s=%s", specs.PLACEHOLDER_PATCH_STORAGE, specs.PLACEHOLDER_CODEREVIEW_SERVER), + fmt.Sprintf("issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patchset=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "upload_nano_results.isolate", Priority: 0.8, diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index 84e6d090ed..5d60aa0f25 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -34,6 +34,7 @@ class SkiaVarsApi(recipe_api.RecipeApi): self.default_env = {} self.gclient_env = {} self.is_compile_bot = self.builder_name.startswith('Build-') + self.no_buildbot = self.m.properties.get('nobuildbot', '') == 'True' self.default_env['CHROME_HEADLESS'] = '1' # The 'depot_tools' directory comes from recipe DEPS and isn't provided by @@ -106,17 +107,27 @@ class SkiaVarsApi(recipe_api.RecipeApi): self.default_env.update({'SKIA_OUT': self.skia_out, 'BUILDTYPE': self.configuration}) - self.is_trybot = self.builder_cfg['is_trybot'] + self.patch_storage = self.m.properties.get('patch_storage', 'rietveld') self.issue = None self.patchset = None - if self.is_trybot: - if self.patch_storage == 'gerrit': - self.issue = self.m.properties['event.change.number'] - self.patchset = self.m.properties['event.patchSet.ref'].split('/')[-1] - else: + if self.no_buildbot: + self.is_trybot = ( + self.m.properties.get('issue', '') and + self.m.properties.get('patchset', '')) + if self.is_trybot: self.issue = self.m.properties['issue'] self.patchset = self.m.properties['patchset'] + else: + self.is_trybot = self.builder_cfg['is_trybot'] + if self.is_trybot: + if self.patch_storage == 'gerrit': + self.issue = self.m.properties['event.change.number'] + self.patchset = self.m.properties['event.patchSet.ref'].split('/')[-1] + else: + self.issue = self.m.properties['issue'] + self.patchset = self.m.properties['patchset'] + self.dm_dir = self.m.path.join( self.swarming_out_dir, 'dm') self.perf_data_dir = self.m.path.join(self.swarming_out_dir, diff --git a/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_gerrit.json b/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_gerrit.json new file mode 100644 index 0000000000..5c9abd3d64 --- /dev/null +++ b/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_gerrit.json @@ -0,0 +1,250 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_C:\\_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "config", + "--spec", + "cache_dir = '[CUSTOM_C:\\\\_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient setup" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "sync", + "--verbose", + "--with_branch_heads", + "--nohooks", + "-j2", + "--reset", + "--force", + "--upstream", + "--no-nag-max", + "--delete_unversioned_trees", + "--revision", + "skia@abc123", + "--output-json", + "/path/to/tmp/json" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient sync", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "recurse", + "git", + "config", + "user.name", + "local_bot" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient recurse (git config user.name)" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "recurse", + "git", + "config", + "user.email", + "local_bot@example.com" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient recurse (git config user.email)" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]\\resources\\apply_gerrit.py", + "--gerrit_repo", + "skia", + "--gerrit_ref", + "", + "--root", + "[CUSTOM_C:\\_B_WORK]\\skia" + ], + "env": { + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "apply_gerrit" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]\\tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_C:\\_B_WORK]\\skia\\infra\\bots\\bootstrap_win_toolchain_json.py", + "--win_toolchain_json", + "[SLAVE_BUILD]\\src\\build\\win_toolchain.json", + "--depot_tools_parent_dir", + "[SLAVE_BUILD]" + ], + "name": "bootstrap win toolchain" + }, + { + "cmd": [ + "python", + "make.py", + "most" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]\\skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "CHROME_PATH": "[SLAVE_BUILD]\\src", + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "build most" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug\\Debug", + "[CUSTOM_[SWARM_OUT_DIR]]\\out\\Debug" + ], + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n" + ], + "name": "cleanup", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import psutil@@@", + "@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@", + "@@@STEP_LOG_LINE@python.inline@ p.kill()@@@", + "@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@", + "@@@STEP_LOG_LINE@python.inline@ pass@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_rietveld.json b/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_rietveld.json new file mode 100644 index 0000000000..92e970956e --- /dev/null +++ b/infra/bots/recipes/swarm_compile.expected/buildbotless_trybot_rietveld.json @@ -0,0 +1,253 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_C:\\_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "config", + "--spec", + "cache_dir = '[CUSTOM_C:\\\\_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient setup" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "sync", + "--verbose", + "--with_branch_heads", + "--nohooks", + "-j2", + "--reset", + "--force", + "--upstream", + "--no-nag-max", + "--delete_unversioned_trees", + "--revision", + "skia@abc123", + "--output-json", + "/path/to/tmp/json" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient sync", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "recurse", + "git", + "config", + "user.name", + "local_bot" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient recurse (git config user.name)" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "recurse", + "git", + "config", + "user.email", + "local_bot@example.com" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "gclient recurse (git config user.email)" + }, + { + "cmd": [ + "python", + "-u", + "[DEPOT_TOOLS]\\apply_issue.py", + "-r", + "[CUSTOM_C:\\_B_WORK]\\skia", + "-i", + "500", + "-p", + "1", + "-s", + "https://codereview.chromium.org", + "--no-auth" + ], + "name": "apply_issue", + "~followup_annotations": [ + "@@@STEP_LINK@Applied issue 500@https://codereview.chromium.org/500@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]\\tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_C:\\_B_WORK]\\skia\\infra\\bots\\bootstrap_win_toolchain_json.py", + "--win_toolchain_json", + "[SLAVE_BUILD]\\src\\build\\win_toolchain.json", + "--depot_tools_parent_dir", + "[SLAVE_BUILD]" + ], + "name": "bootstrap win toolchain" + }, + { + "cmd": [ + "python", + "make.py", + "most" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]\\skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "CHROME_PATH": "[SLAVE_BUILD]\\src", + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug" + }, + "name": "build most" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug\\Debug", + "[CUSTOM_[SWARM_OUT_DIR]]\\out\\Debug" + ], + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n" + ], + "name": "cleanup", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import psutil@@@", + "@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@", + "@@@STEP_LOG_LINE@python.inline@ p.kill()@@@", + "@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@", + "@@@STEP_LOG_LINE@python.inline@ pass@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/swarm_compile.py b/infra/bots/recipes/swarm_compile.py index ddf05f8451..eff13e54de 100644 --- a/infra/bots/recipes/swarm_compile.py +++ b/infra/bots/recipes/swarm_compile.py @@ -319,3 +319,41 @@ def GenTests(api): **gerrit_kwargs) + api.platform('win', 64) ) + + yield ( + api.test('buildbotless_trybot_rietveld') + + api.properties( + repository='skia', + buildername=buildername, + mastername=mastername, + slavename=slavename, + buildnumber=5, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + revision='abc123', + nobuildbot='True', + issue=500, + patchset=1, + patch_storage='rietveld', + rietveld='https://codereview.chromium.org') + + api.platform('win', 64) + ) + + yield ( + api.test('buildbotless_trybot_gerrit') + + api.properties( + repository='skia', + buildername=buildername, + mastername=mastername, + slavename=slavename, + buildnumber=5, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + revision='abc123', + nobuildbot='True', + issue=500, + patchset=1, + patch_storage='gerrit', + gerrit='https://skia-review.googlesource.com') + + api.platform('win', 64) + ) diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py index ebff963322..4298b70efc 100644 --- a/infra/bots/recipes/upload_dm_results.py +++ b/infra/bots/recipes/upload_dm_results.py @@ -49,17 +49,6 @@ def RunSteps(api): builder_name = api.properties['buildername'] revision = api.properties['revision'] - patch_storage = api.properties.get('patch_storage', 'rietveld') - issue = None - patchset = None - if builder_name.endswith('-Trybot'): - if patch_storage == 'gerrit': - issue = str(api.properties['event.change.number']) - patchset = str(api.properties['event.patchSet.ref'].split('/')[-1]) - else: - issue = str(api.properties['issue']) - patchset = str(api.properties['patchset']) - results_dir = api.path['cwd'].join('dm') # Move dm.json and verbose.log to their own directory. @@ -95,10 +84,15 @@ def RunSteps(api): str(int(calendar.timegm(now.utctimetuple())))]) # Trybot results are further siloed by issue/patchset. - if builder_name.endswith('-Trybot'): - if not (issue and patchset): # pragma: nocover - raise Exception('issue and patchset properties are required for trybots.') - summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset)) + issue = str(api.properties.get('issue', '')) + patchset = str(api.properties.get('patchset', '')) + if (api.properties.get('patch_storage', '') == 'gerrit' and + api.properties.get('nobuildbot', '') != 'True'): + issue = str(api.properties['event.change.number']) + patchset = str(api.properties['event.patchSet.ref']).split('/')[-1] + if issue and patchset: + summary_dest_path = '/'.join(( + 'trybot', summary_dest_path, issue, patchset)) summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) diff --git a/infra/bots/recipes/upload_nano_results.py b/infra/bots/recipes/upload_nano_results.py index 1508f03080..50883e7c05 100644 --- a/infra/bots/recipes/upload_nano_results.py +++ b/infra/bots/recipes/upload_nano_results.py @@ -19,17 +19,6 @@ def RunSteps(api): # Upload the nanobench resuls. builder_name = api.properties['buildername'] - patch_storage = api.properties.get('patch_storage', 'rietveld') - issue = None - patchset = None - if builder_name.endswith('-Trybot'): - if patch_storage == 'gerrit': - issue = str(api.properties['event.change.number']) - patchset = str(api.properties['event.patchSet.ref'].split('/')[-1]) - else: - issue = str(api.properties['issue']) - patchset = str(api.properties['patchset']) - now = api.time.utcnow() src_path = api.path['cwd'].join( @@ -50,9 +39,13 @@ def RunSteps(api): str(now.month).zfill(2), str(now.day).zfill(2), str(now.hour).zfill(2), builder_name)) - if builder_name.endswith('-Trybot'): - if not (issue and patchset): # pragma: nocover - raise Exception('issue and patchset properties are required for trybots.') + issue = str(api.properties.get('issue', '')) + patchset = str(api.properties.get('patchset', '')) + if (api.properties.get('patch_storage', '') == 'gerrit' and + api.properties.get('nobuildbot', '') != 'True'): + issue = str(api.properties['event.change.number']) + patchset = str(api.properties['event.patchSet.ref']).split('/')[-1] + if issue and patchset: gs_path = '/'.join(('trybot', gs_path, issue, patchset)) dst = '/'.join(('gs://skia-perf', gs_path, basename)) diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index dc9bc5ba53..69d5e25f8d 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -30,12 +30,18 @@ "--workdir", "../../..", "swarm_compile", + "repository=skia", "buildername=Build-Ubuntu-GCC-x86_64-Release-GN", "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", "swarm_out_dir=${ISOLATED_OUTDIR}", - "revision=<(REVISION)" + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)", + "issue=<(ISSUE)", + "patchset=<(PATCHSET)" ], "isolate": "compile_skia.isolate", "priority": 0.8 @@ -71,12 +77,18 @@ "--workdir", "../../..", "swarm_perf", + "repository=skia", "buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN", "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", "swarm_out_dir=${ISOLATED_OUTDIR}", - "revision=<(REVISION)" + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)", + "issue=<(ISSUE)", + "patchset=<(PATCHSET)" ], "isolate": "perf_skia.isolate", "priority": 0.8 @@ -112,12 +124,18 @@ "--workdir", "../../..", "swarm_test", + "repository=skia", "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN", "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", "swarm_out_dir=${ISOLATED_OUTDIR}", - "revision=<(REVISION)" + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)", + "issue=<(ISSUE)", + "patchset=<(PATCHSET)" ], "isolate": "test_skia.isolate", "priority": 0.8 @@ -136,12 +154,18 @@ "--workdir", "../../..", "upload_nano_results", + "repository=skia", "buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN", "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", "swarm_out_dir=${ISOLATED_OUTDIR}", - "revision=<(REVISION)" + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)", + "issue=<(ISSUE)", + "patchset=<(PATCHSET)" ], "isolate": "upload_nano_results.isolate", "priority": 0.8 @@ -160,12 +184,18 @@ "--workdir", "../../..", "upload_dm_results", + "repository=skia", "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN", "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", + "nobuildbot=True", "swarm_out_dir=${ISOLATED_OUTDIR}", - "revision=<(REVISION)" + "revision=<(REVISION)", + "patch_storage=<(PATCH_STORAGE)", + "<(PATCH_STORAGE)=<(CODEREVIEW_SERVER)", + "issue=<(ISSUE)", + "patchset=<(PATCHSET)" ], "isolate": "upload_dm_results.isolate", "priority": 0.8