From b2cf266151a6698a6c667fb97f9c8d3cbdd8b4e6 Mon Sep 17 00:00:00 2001 From: borenet Date: Mon, 24 Oct 2016 08:40:26 -0700 Subject: [PATCH] Fix RecreateSKPs recipe to correctly fail when the upload fails BUG=skia:5879 GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2446633002 Review-Url: https://codereview.chromium.org/2446633002 --- .../failed_upload.json | 339 ++++++++++++++++++ infra/bots/recipes/swarm_RecreateSKPs.py | 58 ++- 2 files changed, 381 insertions(+), 16 deletions(-) create mode 100644 infra/bots/recipes/swarm_RecreateSKPs.expected/failed_upload.json diff --git a/infra/bots/recipes/swarm_RecreateSKPs.expected/failed_upload.json b/infra/bots/recipes/swarm_RecreateSKPs.expected/failed_upload.json new file mode 100644 index 0000000000..a198dfee12 --- /dev/null +++ b/infra/bots/recipes/swarm_RecreateSKPs.expected/failed_upload.json @@ -0,0 +1,339 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_/_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "config", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}, {'deps_file': '.DEPS.git', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}]" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "gclient setup" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "sync", + "--verbose", + "--with_branch_heads", + "--nohooks", + "-j8", + "--reset", + "--force", + "--upstream", + "--no-nag-max", + "--delete_unversioned_trees", + "--revision", + "skia@abc123", + "--revision", + "src@origin/lkgr", + "--output-json", + "/path/to/tmp/json" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "gclient sync", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"f27fede2220bcd326aee3e86ddfd4ebd0fe58cb9\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "recurse", + "git", + "config", + "user.name", + "local_bot" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "gclient recurse (git config user.name)" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "recurse", + "git", + "config", + "user.email", + "local_bot@example.com" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]:RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "gclient recurse (git config user.email)" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "CPPFLAGS": "-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", + "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gclient runhooks" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "[CUSTOM_/_B_WORK]/src/buildtools/linux64/gn", + "gen", + "[CUSTOM_/_B_WORK]/src/out/Release" + ], + "cwd": "[CUSTOM_/_B_WORK]/src", + "env": { + "CPPFLAGS": "-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", + "GYP_GENERATORS": "ninja" + }, + "name": "GN" + }, + { + "cmd": [ + "ninja", + "-C", + "[CUSTOM_/_B_WORK]/src/out/Release", + "chrome" + ], + "cwd": "[CUSTOM_/_B_WORK]/src", + "name": "Build Chrome" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[SLAVE_BUILD]/skp_output" + ], + "name": "rmtree skp_output", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/skp_output", + "511" + ], + "name": "makedirs skp_output", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/skp/create.py", + "--chrome_src_path", + "[CUSTOM_/_B_WORK]/src", + "--browser_executable", + "[CUSTOM_/_B_WORK]/src/out/Release/chrome", + "--target_dir", + "[SLAVE_BUILD]/skp_output", + "--upload_to_partner_bucket" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": "[DEPOT_TOOLS]:%(PATH)s" + }, + "name": "Recreate SKPs" + }, + { + "cmd": [ + "go", + "get", + "-u", + "go.skia.org/infra/..." + ], + "env": { + "GOPATH": "[CUSTOM_/_B_WORK]/gopath" + }, + "name": "update go pkgs" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport urllib2\n\nTOKEN_FILE = '.depot_tools_oauth2_tokens'\nTOKEN_FILE_BACKUP = '.depot_tools_oauth2_tokens.old'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/depot_tools_auth_update_skps'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.rename(token_file, os.path.join(home, TOKEN_FILE_BACKUP))\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n" + ], + "name": "depot-tools-auth login", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = '.depot_tools_oauth2_tokens'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE_BACKUP = '.depot_tools_oauth2_tokens.old'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/depot_tools_auth_update_skps'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.rename(token_file, os.path.join(home, TOKEN_FILE_BACKUP))@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "[CUSTOM_/_B_WORK]/skia/infra/bots/upload_skps.py", + "--target_dir", + "[SLAVE_BUILD]/skp_output" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "CHROME_HEADLESS": "1", + "GOPATH": "[CUSTOM_/_B_WORK]/gopath", + "PATH": "[DEPOT_TOOLS]:%(PATH)s" + }, + "name": "Upload SKPs", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\n\n\nTOKEN_FILE = '.depot_tools_oauth2_tokens'\nTOKEN_FILE_BACKUP = '.depot_tools_oauth2_tokens.old'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n\nbackup_file = os.path.join(home, TOKEN_FILE_BACKUP)\nif os.path.isfile(backup_file):\n os.rename(backup_file, token_file)\n" + ], + "name": "depot-tools-auth logout", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = '.depot_tools_oauth2_tokens'@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE_BACKUP = '.depot_tools_oauth2_tokens.old'@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@", + "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@backup_file = os.path.join(home, TOKEN_FILE_BACKUP)@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(backup_file):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.rename(backup_file, token_file)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "reason": "Step('Upload SKPs') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/swarm_RecreateSKPs.py b/infra/bots/recipes/swarm_RecreateSKPs.py index d13e7507cc..d21b6fb931 100644 --- a/infra/bots/recipes/swarm_RecreateSKPs.py +++ b/infra/bots/recipes/swarm_RecreateSKPs.py @@ -68,7 +68,7 @@ with open(token_file, 'w') as f: ) def __exit__(self, t, v, tb): - return self.m.python.inline( + self.m.python.inline( 'depot-tools-auth logout', """ import os @@ -89,6 +89,7 @@ if os.path.isfile(backup_file): """ % (DEPOT_TOOLS_AUTH_TOKEN_FILE, DEPOT_TOOLS_AUTH_TOKEN_FILE_BACKUP), ) + return v is None def RunSteps(api): @@ -150,18 +151,43 @@ def RunSteps(api): def GenTests(api): - for mastername, slaves in TEST_BUILDERS.iteritems(): - for slavename, builders_by_slave in slaves.iteritems(): - for builder in builders_by_slave: - test = ( - api.test(builder) + - api.properties(buildername=builder, - mastername=mastername, - slavename=slavename, - revision='abc123', - buildnumber=2, - path_config='kitchen', - swarm_out_dir='[SWARM_OUT_DIR]') + - api.path.exists(api.path['slave_build'].join('skp_output')) - ) - yield test + mastername = 'client.skia.compile' + slavename = 'skiabot-linux-swarm-000' + builder = 'Housekeeper-Nightly-RecreateSKPs_Canary' + yield ( + api.test(builder) + + api.properties(buildername=builder, + mastername=mastername, + slavename=slavename, + revision='abc123', + buildnumber=2, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.path.exists(api.path['slave_build'].join('skp_output')) + ) + + builder = 'Housekeeper-Weekly-RecreateSKPs' + yield ( + api.test(builder) + + api.properties(buildername=builder, + mastername=mastername, + slavename=slavename, + revision='abc123', + buildnumber=2, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.path.exists(api.path['slave_build'].join('skp_output')) + ) + + yield ( + api.test('failed_upload') + + api.properties(buildername=builder, + mastername=mastername, + slavename=slavename, + revision='abc123', + buildnumber=2, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.path.exists(api.path['slave_build'].join('skp_output')) + + api.step_data('Upload SKPs', retcode=1) + )