Upload multiple dm images at once

Use the -m file, as recommended in
https://luci-milo.appspot.com/swarming/task/3b197fbd667a1510/steps/upload_images/0/stdout

Bug: skia:
Change-Id: I58d101f58594bf221d11a787057ee7950b197f86
Reviewed-on: https://skia-review.googlesource.com/94965
Reviewed-by: Stephan Altmueller <stephana@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>

infra/bots/recipe_modules/gsutil/api.py

@@ -12,7 +12,7 @@ class GSUtilApi(recipe_api.RecipeApi):
     """Run gsutil with the given args."""
     return self.m.step(step_name, cmd=['gsutil'] + list(args))
 
-  def cp(self, name, src, dst, extra_args=None):
+  def cp(self, name, src, dst, extra_args=None, multithread=False):
     """Attempt to upload or download files to/from Google Cloud Storage (GCS).
 
     Args:
@@ -21,10 +21,14 @@ class GSUtilApi(recipe_api.RecipeApi):
       dst: string. Same as src.
       extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks
         all files be compressed with gzip after upload and before download.
+      multi_thread: if the -m argument should be used to copy multiple items
+        at once (e.g. gsutil -m cp foo* gs://bar/dir)
 
     If the operation fails, it will be retried multiple times.
     """
     cmd = ['cp']
+    if multithread:
+      cmd = ['-m'] + cmd
     if extra_args:
       cmd.extend(extra_args)
     cmd.extend([src, dst])

infra/bots/recipe_modules/gsutil/examples/full.expected/failed_all_uploads.json

@@ -2,6 +2,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",
@@ -16,6 +17,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",
@@ -30,6 +32,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",
@@ -44,6 +47,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",
@@ -58,6 +62,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",

infra/bots/recipe_modules/gsutil/examples/full.expected/failed_one_upload.json

@@ -2,6 +2,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",
@@ -16,6 +17,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",

infra/bots/recipe_modules/gsutil/examples/full.expected/gsutil_tests.json

@@ -2,6 +2,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "-Z",
       "/foo/file",

infra/bots/recipe_modules/gsutil/examples/full.py

@@ -18,7 +18,8 @@ DEPS = [
 
 
 def RunSteps(api):
-  api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file', ['-Z'])
+  api.gsutil.cp('test file', '/foo/file', 'gs://bar-bucket/file',
+                extra_args=['-Z'], multithread=True)
 
 def GenTests(api):
   yield (

infra/bots/recipes/upload_coverage_results.py

@@ -91,7 +91,7 @@ def RunSteps(api):
 
   gcs_file = PARSED_FILE % builder_name
   api.gsutil.cp('parsed data', indexed_data,
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
 
   # Create text coverage output
   output_data = api.path['start_dir'].join('coverage_text')
@@ -108,7 +108,7 @@ def RunSteps(api):
   # downloading and untarring all the coverage data.
   gcs_file = SUMMARY_FILE % builder_name
   api.gsutil.cp('coverage summary', output_data.join('index.txt'),
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
 
   tar_file = api.path['start_dir'].join('coverage.text.tar')
 
@@ -119,7 +119,7 @@ def RunSteps(api):
 
   gcs_file = COVERAGE_TEXT_FILE % builder_name
   api.gsutil.cp('text coverage data', tar_file,
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
 
   # Create html coverage output
   output_data = api.path['start_dir'].join('coverage_html')
@@ -141,7 +141,7 @@ def RunSteps(api):
 
   gcs_file = COVERAGE_HTML_FILE % builder_name
   api.gsutil.cp('html coverage data', tar_file,
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+                   'gs://%s/%s%s' % (bucket, path, gcs_file), extra_args=['-Z'])
 
 
 def GenTests(api):

infra/bots/recipes/upload_dm_results.expected/alternate_bucket.json

@@ -90,6 +90,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"

infra/bots/recipes/upload_dm_results.expected/failed_all.json

@@ -90,6 +90,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"
@@ -103,6 +104,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"
@@ -116,6 +118,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"
@@ -129,6 +132,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"
@@ -142,6 +146,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"

infra/bots/recipes/upload_dm_results.expected/failed_once.json

@@ -90,6 +90,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"
@@ -103,6 +104,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"

infra/bots/recipes/upload_dm_results.expected/normal_bot.json

@@ -90,6 +90,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"

infra/bots/recipes/upload_dm_results.expected/trybot.json

@@ -90,6 +90,7 @@
   {
     "cmd": [
       "gsutil",
+      "-m",
       "cp",
       "[START_DIR]/dm/*.png",
       "gs://skia-infra-gm/dm-images-v1"

infra/bots/recipes/upload_dm_results.py

@@ -53,7 +53,7 @@ def RunSteps(api):
     files_to_upload = [f for f in files_to_upload if str(f).endswith(ext)]
     if len(files_to_upload) > 0:
       api.gsutil.cp('images', results_dir.join('*%s' % ext),
-                       image_dest_path)
+                       image_dest_path, multithread=True)
 
   # Upload the JSON summary and verbose.log.
   now = api.time.utcnow()
@@ -78,7 +78,7 @@ def RunSteps(api):
                                       summary_dest_path)
 
   api.gsutil.cp('JSON and logs', tmp_dir.join('*'), summary_dest_path,
-                   ['-z', 'json,log'])
+                   extra_args=['-z', 'json,log'])
 
 
 def GenTests(api):