AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upload multiple dm images at once

Browse Source 
Use the -m file, as recommended in
https://luci-milo.appspot.com/swarming/task/3b197fbd667a1510/steps/upload_images/0/stdout

Bug: skia:
Change-Id: I58d101f58594bf221d11a787057ee7950b197f86
Reviewed-on: https://skia-review.googlesource.com/94965
Reviewed-by: Stephan Altmueller <stephana@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>
This commit is contained in:
Kevin Lubick 2018-01-16 16:06:09 -05:00 committed by Skia Commit-Bot
parent 279643019e
commit d452434d17
12 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
infra/bots/recipe_modules/gsutil/api.py
View File

@ -12,7 +12,7 @@ class GSUtilApi(recipe_api.RecipeApi):
"""Run gsutil with the given args."""
return self.m.step(step_name, cmd=['gsutil'] + list(args))
def cp(self, name, src, dst, extra_args=None):
def cp(self, name, src, dst, extra_args=None, multithread=False):
"""Attempt to upload or download files to/from Google Cloud Storage (GCS).
Args:
@ -21,10 +21,14 @@ class GSUtilApi(recipe_api.RecipeApi):
dst: string. Same as src.
extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks
all files be compressed with gzip after upload and before download.
multi_thread: if the -m argument should be used to copy multiple items
at once (e.g. gsutil -m cp foo* gs://bar/dir)
If the operation fails, it will be retried multiple times.
"""
cmd = ['cp']
if multithread:
cmd = ['-m'] + cmd
if extra_args:
cmd.extend(extra_args)
cmd.extend([src, dst])

5
infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json
View File

@ -2,6 +2,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",
@ -16,6 +17,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",
@ -30,6 +32,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",
@ -44,6 +47,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",
@ -58,6 +62,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",

2
infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json
View File

@ -2,6 +2,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",
@ -16,6 +17,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",

1
infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json
View File

@ -2,6 +2,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"-Z",
"/foo/file",

3
infra/bots/recipe_modules/gsutil/examples/full.py
View File

@ -18,7 +18,8 @@ DEPS = [
def RunSteps(api):
api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', ['-Z'])
api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file',
extra_args=['-Z'], multithread=True)
def GenTests(api):
yield (

8
infra/bots/recipes/upload_coverage_results.py
View File

@ -91,7 +91,7 @@ def RunSteps(api):
gcs_file = PARSED_FILE % builder_name
api.gsutil.cp('parsed data', indexed_data,
'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
# Create text coverage output
output_data = api.path['start_dir'].join('coverage_text')
@ -108,7 +108,7 @@ def RunSteps(api):
# downloading and untarring all the coverage data.
gcs_file = SUMMARY_FILE % builder_name
api.gsutil.cp('coverage summary', output_data.join('index.txt'),
'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
tar_file = api.path['start_dir'].join('coverage.text.tar')
@ -119,7 +119,7 @@ def RunSteps(api):
gcs_file = COVERAGE_TEXT_FILE % builder_name
api.gsutil.cp('text coverage data', tar_file,
'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
# Create html coverage output
output_data = api.path['start_dir'].join('coverage_html')
@ -141,7 +141,7 @@ def RunSteps(api):
gcs_file = COVERAGE_HTML_FILE % builder_name
api.gsutil.cp('html coverage data', tar_file,
'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
def GenTests(api):

1
infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json
View File

@ -90,6 +90,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"

5
infra/bots/recipes/upload_dm_results.expected/failed_all.json
View File

@ -90,6 +90,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"
@ -103,6 +104,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"
@ -116,6 +118,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"
@ -129,6 +132,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"
@ -142,6 +146,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"

2
infra/bots/recipes/upload_dm_results.expected/failed_once.json
View File

@ -90,6 +90,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"
@ -103,6 +104,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"

1
infra/bots/recipes/upload_dm_results.expected/normal_bot.json
View File

@ -90,6 +90,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"

1
infra/bots/recipes/upload_dm_results.expected/trybot.json
View File

@ -90,6 +90,7 @@
{
"cmd": [
"gsutil",
"-m",
"cp",
"[START_DIR]/dm/*.png",
"gs://skia-infra-gm/dm-images-v1"

4
infra/bots/recipes/upload_dm_results.py
View File

@ -53,7 +53,7 @@ def RunSteps(api):
files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)]
if len(files_to_upload) > 0:
api.gsutil.cp('images', results_dir.join('*%s' % ext),
image_dest_path)
image_dest_path, multithread=True)
# Upload the JSON summary and verbose.log.
now = api.time.utcnow()
@ -78,7 +78,7 @@ def RunSteps(api):
summary_dest_path)
api.gsutil.cp('JSON and logs', tmp_dir.join('*'), summary_dest_path,
['-z', 'json,log'])
extra_args=['-z', 'json,log'])
def GenTests(api):