Add retries to upload_dm_results
BUG=skia:5818 GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2390723003 Review-Url: https://codereview.chromium.org/2390723003
This commit is contained in:
borenet
Commit bot
parent
ee843b2ae4
commit
1818701746
157
infra/bots/recipes/upload_dm_results.expected/failed_all.json
Normal file
157
infra/bots/recipes/upload_dm_results.expected/failed_all.json
Normal file
@ -0,0 +1,157 @@
|
||||
[
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
|
||||
"[CWD]/tmp_upload",
|
||||
"511"
|
||||
],
|
||||
"name": "makedirs tmp dir",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
|
||||
"[CWD]/dm/dm.json",
|
||||
"[CWD]/tmp_upload"
|
||||
],
|
||||
"name": "copy dm.json"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
|
||||
"[CWD]/dm/verbose.log",
|
||||
"[CWD]/tmp_upload"
|
||||
],
|
||||
"name": "copy verbose.log"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
|
||||
"[CWD]/dm/dm.json"
|
||||
],
|
||||
"name": "rm old dm.json",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
|
||||
"[CWD]/dm/verbose.log"
|
||||
],
|
||||
"name": "rm old verbose.log",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
|
||||
"/path/to/tmp/",
|
||||
"[CWD]/dm/*"
|
||||
],
|
||||
"name": "find images"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images (attempt 2)",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images (attempt 3)",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images (attempt 4)",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images (attempt 5)",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "$result",
|
||||
"reason": "Step('upload images (attempt 5)') failed with return_code 1",
|
||||
"recipe_result": null,
|
||||
"status_code": 1
|
||||
}
|
||||
]
|
||||
124
infra/bots/recipes/upload_dm_results.expected/failed_once.json
Normal file
124
infra/bots/recipes/upload_dm_results.expected/failed_once.json
Normal file
@ -0,0 +1,124 @@
|
||||
[
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
|
||||
"[CWD]/tmp_upload",
|
||||
"511"
|
||||
],
|
||||
"name": "makedirs tmp dir",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
|
||||
"[CWD]/dm/dm.json",
|
||||
"[CWD]/tmp_upload"
|
||||
],
|
||||
"name": "copy dm.json"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n",
|
||||
"[CWD]/dm/verbose.log",
|
||||
"[CWD]/tmp_upload"
|
||||
],
|
||||
"name": "copy verbose.log"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
|
||||
"[CWD]/dm/dm.json"
|
||||
],
|
||||
"name": "rm old dm.json",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport os\nimport sys\nos.remove(sys.argv[1])\n",
|
||||
"[CWD]/dm/verbose.log"
|
||||
],
|
||||
"name": "rm old verbose.log",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@os.remove(sys.argv[1])@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport glob\nimport sys\nwith open(sys.argv[1], 'w') as f:\n f.write('\\n'.join(glob.glob(sys.argv[2])))\n",
|
||||
"/path/to/tmp/",
|
||||
"[CWD]/dm/*"
|
||||
],
|
||||
"name": "find images"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images",
|
||||
"~followup_annotations": [
|
||||
"step returned non-zero exit code: 1",
|
||||
"@@@STEP_FAILURE@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"[CWD]/dm/*",
|
||||
"gs://skia-infra-gm/dm-images-v1"
|
||||
],
|
||||
"name": "upload images (attempt 2)"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"gsutil",
|
||||
"cp",
|
||||
"-z",
|
||||
"json,log",
|
||||
"[CWD]/tmp_upload/*",
|
||||
"gs://skia-infra-gm/dm-json-v1/2012/05/14/12/abc123/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug/1337018001"
|
||||
],
|
||||
"name": "upload JSON and logs"
|
||||
},
|
||||
{
|
||||
"name": "$result",
|
||||
"recipe_result": null,
|
||||
"status_code": 0
|
||||
}
|
||||
]
|
||||
@ -22,9 +22,29 @@ import time
|
||||
|
||||
DM_JSON = 'dm.json'
|
||||
GS_BUCKET = 'gs://skia-infra-gm'
|
||||
UPLOAD_ATTEMPTS = 5
|
||||
VERBOSE_LOG = 'verbose.log'
|
||||
|
||||
|
||||
def cp(api, name, src, dst, extra_args=None):
|
||||
cmd = ['gsutil', 'cp']
|
||||
if extra_args:
|
||||
cmd.extend(extra_args)
|
||||
cmd.extend([src, dst])
|
||||
|
||||
name = 'upload %s' % name
|
||||
for i in xrange(UPLOAD_ATTEMPTS):
|
||||
step_name = name
|
||||
if i > 0:
|
||||
step_name += ' (attempt %d)' % (i+1)
|
||||
try:
|
||||
api.step(step_name, cmd=cmd)
|
||||
break
|
||||
except api.step.StepFailure:
|
||||
if i == UPLOAD_ATTEMPTS - 1:
|
||||
raise
|
||||
|
||||
|
||||
def RunSteps(api):
|
||||
builder_name = api.properties['buildername']
|
||||
revision = api.properties['revision']
|
||||
@ -60,10 +80,7 @@ def RunSteps(api):
|
||||
test_data=['someimage.png'],
|
||||
infra_step=True)
|
||||
if len(files_to_upload) > 0:
|
||||
api.step(
|
||||
'upload images',
|
||||
cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path],
|
||||
)
|
||||
cp(api, 'images', results_dir.join('*'), image_dest_path)
|
||||
|
||||
# Upload the JSON summary and verbose.log.
|
||||
now = api.time.utcnow()
|
||||
@ -85,11 +102,8 @@ def RunSteps(api):
|
||||
|
||||
summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path))
|
||||
|
||||
api.step(
|
||||
'upload JSON and logs',
|
||||
cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'),
|
||||
summary_dest_path],
|
||||
)
|
||||
cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path,
|
||||
['-z', 'json,log'])
|
||||
|
||||
|
||||
def GenTests(api):
|
||||
@ -101,6 +115,26 @@ def GenTests(api):
|
||||
path_config='kitchen')
|
||||
)
|
||||
|
||||
yield (
|
||||
api.test('failed_once') +
|
||||
api.properties(buildername=builder,
|
||||
revision='abc123',
|
||||
path_config='kitchen') +
|
||||
api.step_data('upload images', retcode=1)
|
||||
)
|
||||
|
||||
yield (
|
||||
api.test('failed_all') +
|
||||
api.properties(buildername=builder,
|
||||
revision='abc123',
|
||||
path_config='kitchen') +
|
||||
api.step_data('upload images', retcode=1) +
|
||||
api.step_data('upload images (attempt 2)', retcode=1) +
|
||||
api.step_data('upload images (attempt 3)', retcode=1) +
|
||||
api.step_data('upload images (attempt 4)', retcode=1) +
|
||||
api.step_data('upload images (attempt 5)', retcode=1)
|
||||
)
|
||||
|
||||
builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot'
|
||||
yield (
|
||||
api.test('trybot') +
|
||||
|
||||
Loading…
Reference in New Issue
Block a user