From 1818701746e4ea76631afd6934d6257e2b3d781b Mon Sep 17 00:00:00 2001 From: borenet Date: Tue, 4 Oct 2016 06:32:54 -0700 Subject: [PATCH] Add retries to upload_dm_results BUG=skia:5818 GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2390723003 Review-Url: https://codereview.chromium.org/2390723003 --- .../failed_all.json | 157 ++++++++++++++++++ .../failed_once.json | 124 ++++++++++++++ infra/bots/recipes/upload_dm_results.py | 52 +++++- 3 files changed, 324 insertions(+), 9 deletions(-) create mode 100644 infra/bots/recipes/upload_dm_results.expected/failed_all.json create mode 100644 infra/bots/recipes/upload_dm_results.expected/failed_once.json diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_all.json b/infra/bots/recipes/upload_dm_results.expected/failed_all.json new file mode 100644 index 0000000000..d88927f05e --- /dev/null +++ b/infra/bots/recipes/upload_dm_results.expected/failed_all.json @@ -0,0 +1,157 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CWD]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "[CWD]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/verbose.log", + "[CWD]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[CWD]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 2)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 3)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 4)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 5)", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "name": "$result", + "reason": "Step('upload images (attempt 5)') failed with return_code 1", + "recipe_result": null, + "status_code": 1 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/upload_dm_results.expected/failed_once.json b/infra/bots/recipes/upload_dm_results.expected/failed_once.json new file mode 100644 index 0000000000..48d11423ed --- /dev/null +++ b/infra/bots/recipes/upload_dm_results.expected/failed_once.json @@ -0,0 +1,124 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CWD]/tmp_upload", + "511" + ], + "name": "makedirs tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/dm.json", + "[CWD]/tmp_upload" + ], + "name": "copy dm.json" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[CWD]/dm/verbose.log", + "[CWD]/tmp_upload" + ], + "name": "copy verbose.log" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/dm.json" + ], + "name": "rm old dm.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nos.remove(sys.argv[1])\n", + "[CWD]/dm/verbose.log" + ], + "name": "rm old verbose.log", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n", + "/path/to/tmp/", + "[CWD]/dm/*" + ], + "name": "find images" + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images", + "~followup_annotations": [ + "step returned non-zero exit code: 1", + "@@@STEP_FAILURE@@@" + ] + }, + { + "cmd": [ + "gsutil", + "cp", + "[CWD]/dm/*", + "gs://skia-infra-gm/dm-images-v1" + ], + "name": "upload images (attempt 2)" + }, + { + "cmd": [ + "gsutil", + "cp", + "-z", + "json,log", + "[CWD]/tmp_upload/*", + "gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001" + ], + "name": "upload JSON and logs" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/upload_dm_results.py b/infra/bots/recipes/upload_dm_results.py index 6d1b0eacd0..53e68451bd 100644 --- a/infra/bots/recipes/upload_dm_results.py +++ b/infra/bots/recipes/upload_dm_results.py @@ -22,9 +22,29 @@ import time DM_JSON = 'dm.json' GS_BUCKET = 'gs://skia-infra-gm' +UPLOAD_ATTEMPTS = 5 VERBOSE_LOG = 'verbose.log' +def cp(api, name, src, dst, extra_args=None): + cmd = ['gsutil', 'cp'] + if extra_args: + cmd.extend(extra_args) + cmd.extend([src, dst]) + + name = 'upload %s' % name + for i in xrange(UPLOAD_ATTEMPTS): + step_name = name + if i > 0: + step_name += ' (attempt %d)' % (i+1) + try: + api.step(step_name, cmd=cmd) + break + except api.step.StepFailure: + if i == UPLOAD_ATTEMPTS - 1: + raise + + def RunSteps(api): builder_name = api.properties['buildername'] revision = api.properties['revision'] @@ -60,10 +80,7 @@ def RunSteps(api): test_data=['someimage.png'], infra_step=True) if len(files_to_upload) > 0: - api.step( - 'upload images', - cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path], - ) + cp(api, 'images', results_dir.join('*'), image_dest_path) # Upload the JSON summary and verbose.log. now = api.time.utcnow() @@ -85,11 +102,8 @@ def RunSteps(api): summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) - api.step( - 'upload JSON and logs', - cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'), - summary_dest_path], - ) + cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path, + ['-z', 'json,log']) def GenTests(api): @@ -101,6 +115,26 @@ def GenTests(api): path_config='kitchen') ) + yield ( + api.test('failed_once') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + ) + + yield ( + api.test('failed_all') + + api.properties(buildername=builder, + revision='abc123', + path_config='kitchen') + + api.step_data('upload images', retcode=1) + + api.step_data('upload images (attempt 2)', retcode=1) + + api.step_data('upload images (attempt 3)', retcode=1) + + api.step_data('upload images (attempt 4)', retcode=1) + + api.step_data('upload images (attempt 5)', retcode=1) + ) + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' yield ( api.test('trybot') +