Shard GCE Coverage bot to decrease latency

Bug: skia:
Change-Id: I28d28f514b46ad0c30109beba90ae8a1bd8fc4e3
Reviewed-on: https://skia-review.googlesource.com/59961
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>

infra/bots/README.recipes.md

@@ -836,17 +836,17 @@ benchmark Skia using skpbench.
 
 [DEPS](/infra/bots/recipes/test.py#9): [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step], [core](#recipe_modules-core), [env](#recipe_modules-env), [flavor](#recipe_modules-flavor), [run](#recipe_modules-run), [vars](#recipe_modules-vars)
 
-— **def [RunSteps](/infra/bots/recipes/test.py#842)(api):**
+— **def [RunSteps](/infra/bots/recipes/test.py#852)(api):**
 
 — **def [dm\_flags](/infra/bots/recipes/test.py#27)(api, bot):**
 
-— **def [key\_params](/infra/bots/recipes/test.py#698)(api):**
+— **def [key\_params](/infra/bots/recipes/test.py#708)(api):**
 
 Build a unique key from the builder name (as a list).
 
 E.g.  arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6
 
-— **def [test\_steps](/infra/bots/recipes/test.py#714)(api):**
+— **def [test\_steps](/infra/bots/recipes/test.py#724)(api):**
 
 Run the DM test.
 ### *recipes* / [update\_meta\_config](/infra/bots/recipes/update_meta_config.py)
@@ -858,9 +858,9 @@ Recipe for the Bot that updates meta config.
 — **def [RunSteps](/infra/bots/recipes/update_meta_config.py#38)(api):**
 ### *recipes* / [upload\_coverage\_results](/infra/bots/recipes/upload_coverage_results.py)
 
-[DEPS](/infra/bots/recipes/upload_coverage_results.py#12): [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time], [gsutil](#recipe_modules-gsutil)
+[DEPS](/infra/bots/recipes/upload_coverage_results.py#12): [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/python][recipe_engine/recipe_modules/python], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time], [gsutil](#recipe_modules-gsutil)
 
-— **def [RunSteps](/infra/bots/recipes/upload_coverage_results.py#33)(api):**
+— **def [RunSteps](/infra/bots/recipes/upload_coverage_results.py#38)(api):**
 ### *recipes* / [upload\_dm\_results](/infra/bots/recipes/upload_dm_results.py)
 
 [DEPS](/infra/bots/recipes/upload_dm_results.py#12): [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/time][recipe_engine/recipe_modules/time], [gsutil](#recipe_modules-gsutil)

infra/bots/gen_tasks.go

@@ -19,6 +19,7 @@ import (
 	"regexp"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -795,39 +796,94 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
 			Priority: 0.8,
 		})
 		return uploadName
-	} else if strings.Contains(name, "Coverage") {
+	}
-		uploadName := fmt.Sprintf("%s%s%s", "Upload", jobNameSchema.Sep, name)
+
-		// We need clang_linux to get access to the llvm-profdata and llvm-cov binaries
+	return name
-		// which are used to deal with the raw coverage data output by the Test step.
+}
-		pkgs := []*specs.CipdPackage{}
+
-		pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux"))
+func coverage(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
-		b.MustAddTask(uploadName, &specs.TaskSpec{
+	shards := 1
-			// A dependency on compileTaskName makes the TaskScheduler link the
+	deps := []string{}
-			// isolated output of the compile step to the input of the upload step,
+
-			// which gives us access to the instrumented binary. The binary is
+	tf := parts["test_filter"]
-			// needed to figure out symbol names and line numbers.
+	if strings.Contains(tf, "Shard") {
-			Dependencies: []string{name, compileTaskName},
+		// Expected Shard_NN
-			Dimensions:   linuxGceDimensions(),
+		shardstr := strings.Split(tf, "_")[1]
-			CipdPackages: pkgs,
+		var err error
+		shards, err = strconv.Atoi(shardstr)
+		if err != nil {
+			glog.Fatalf("Expected int for number of shards %q in %s: %s", shardstr, name, err)
+		}
+	}
+	for i := 0; i < shards; i++ {
+		n := strings.Replace(name, tf, fmt.Sprintf("shard_%02d_%02d", i, shards), 1)
+		s := &specs.TaskSpec{
+			CipdPackages:     pkgs,
+			Dependencies:     []string{compileTaskName},
+			Dimensions:       swarmDimensions(parts),
+			ExecutionTimeout: 4 * time.Hour,
+			Expiration:       20 * time.Hour,
 			ExtraArgs: []string{
-				"--workdir", "../../..", "upload_coverage_results",
+				"--workdir", "../../..", "test",
 				fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
-				fmt.Sprintf("buildername=%s", name),
+				fmt.Sprintf("buildername=%s", n),
 				fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
 				fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
 				fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO),
 				fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
 				fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
 				fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
-				fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketCoverage),
 			},
-			Isolate:  relpath("upload_coverage_results.isolate"),
+			IoTimeout:   40 * time.Minute,
-			Priority: 0.8,
+			Isolate:     relpath("test_skia.isolate"),
-		})
+			MaxAttempts: 1,
-		return uploadName
+			Priority:    0.8,
+		}
+		if useBundledRecipes(parts) {
+			s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME)
+			if strings.Contains(parts["os"], "Win") {
+				s.Isolate = relpath("test_skia_bundled_win.isolate")
+			} else {
+				s.Isolate = relpath("test_skia_bundled_unix.isolate")
+			}
+		}
+		if deps := getIsolatedCIPDDeps(parts); len(deps) > 0 {
+			s.Dependencies = append(s.Dependencies, deps...)
+		}
+		b.MustAddTask(n, s)
+		deps = append(deps, n)
 	}
 
-	return name
+	uploadName := fmt.Sprintf("%s%s%s", "Upload", jobNameSchema.Sep, name)
+	// We need clang_linux to get access to the llvm-profdata and llvm-cov binaries
+	// which are used to deal with the raw coverage data output by the Test step.
+	pkgs = append([]*specs.CipdPackage{}, b.MustGetCipdPackageFromAsset("clang_linux"))
+	deps = append(deps, compileTaskName)
+
+	b.MustAddTask(uploadName, &specs.TaskSpec{
+		// A dependency on compileTaskName makes the TaskScheduler link the
+		// isolated output of the compile step to the input of the upload step,
+		// which gives us access to the instrumented binary. The binary is
+		// needed to figure out symbol names and line numbers.
+		Dependencies: deps,
+		Dimensions:   linuxGceDimensions(),
+		CipdPackages: pkgs,
+		ExtraArgs: []string{
+			"--workdir", "../../..", "upload_coverage_results",
+			fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
+			fmt.Sprintf("buildername=%s", name),
+			fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
+			fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
+			fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO),
+			fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
+			fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
+			fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
+			fmt.Sprintf("gs_bucket=%s", CONFIG.GsBucketCoverage),
+		},
+		Isolate:  relpath("upload_coverage_results.isolate"),
+		Priority: 0.8,
+	})
+	return uploadName
 }
 
 // perf generates a Perf task. Returns the name of the last task in the
@@ -1028,8 +1084,14 @@ func process(b *specs.TasksCfgBuilder, name string) {
 	}
 
 	// Test bots.
-	if parts["role"] == "Test" && !strings.Contains(name, "-CT_") {
+
-		deps = append(deps, test(b, name, parts, compileTaskName, pkgs))
+	if parts["role"] == "Test" {
+		if strings.Contains(parts["extra_config"], "Coverage") {
+			deps = append(deps, coverage(b, name, parts, compileTaskName, pkgs))
+		} else if !strings.Contains(name, "-CT_") {
+			deps = append(deps, test(b, name, parts, compileTaskName, pkgs))
+		}
+
 	}
 
 	// Perf bots.

infra/bots/jobs.json

@@ -329,7 +329,7 @@
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN_FSAA",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN_FAAA",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN_FDAA",
-  "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage",
+  "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Shard_16-Coverage",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN_FSAA",
   "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN_FAAA",

infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage.json

@@ -200,7 +200,7 @@
     "env": {
       "BUILDTYPE": "Debug",
       "CHROME_HEADLESS": "1",
-      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/All.profraw",
       "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
       "SKIA_OUT": "[START_DIR]/out"
     },

infra/bots/recipe_modules/flavor/gn_flavor.py

@@ -242,8 +242,9 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
       # This is the output file for the coverage data. Just running the binary
       # will produce the output. The output_file is in the swarming_out_dir and
       # thus will be an isolated output of the Test step.
+      profname = '%s.profraw' % self.m.vars.builder_cfg.get('test_filter','o')
       env['LLVM_PROFILE_FILE'] = self.m.path.join(self.m.vars.swarming_out_dir,
-                                                  'output.profraw')
+                                                  profname)
 
     if path:
       env['PATH'] = '%%(PATH)s:%s' % ':'.join('%s' % p for p in path)

infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage.json

@@ -132,7 +132,7 @@
       "gitHash",
       "abc123",
       "builder",
-      "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage",
+      "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage",
       "swarming_bot_id",
       "skia-bot-123",
       "swarming_task_id",
@@ -160,6 +160,10 @@
       "pdf",
       "--randomProcessorTest",
       "--nogpu",
+      "--shard",
+      "00",
+      "--shards",
+      "10",
       "--config",
       "8888",
       "srgb",
@@ -461,7 +465,7 @@
     "env": {
       "BUILDTYPE": "Debug",
       "CHROME_HEADLESS": "1",
-      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/shard_00_10.profraw",
       "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
       "SKIA_OUT": "[START_DIR]/out"
     },

infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Debug-All-Vulkan_Coverage.json

@@ -311,7 +311,7 @@
       "CHROME_HEADLESS": "1",
       "LD_LIBRARY_PATH": "[START_DIR]/linux_vulkan_intel_driver_debug:[START_DIR]/linux_vulkan_sdk/lib",
       "LIBGL_DRIVERS_PATH": "[START_DIR]/linux_vulkan_intel_driver_debug",
-      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/output.profraw",
+      "LLVM_PROFILE_FILE": "[CUSTOM_[SWARM_OUT_DIR]]/All.profraw",
       "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/linux_vulkan_sdk/bin",
       "SKIA_OUT": "[START_DIR]/out",
       "VK_ICD_FILENAMES": "[START_DIR]/linux_vulkan_intel_driver_debug/intel_icd.x86_64.json"

infra/bots/recipes/test.py

@@ -229,6 +229,16 @@ def dm_flags(api, bot):
       configs = [c for c in configs if c == 'gl' or c == 'gles']
       args.extend(['--pr', 'ccpr', '--cachePathMasks', 'false'])
 
+  tf = api.vars.builder_cfg.get('test_filter')
+  if 'All' != tf:
+    # Expected format: shard_XX_YY
+    parts = tf.split('_')
+    if len(parts) == 3:
+      args.extend(['--shard', parts[1]])
+      args.extend(['--shards', parts[2]])
+    else:
+      raise Exception('Invalid task name - bad shards') #pragma: nocover
+
   args.append('--config')
   args.extend(configs)
 
@@ -876,7 +886,7 @@ TEST_BUILDERS = [
   'Test-ChromeOS-Clang-Chromebook_CB5_312T-GPU-PowerVRGX6250-arm-Debug-All',
   'Test-Chromecast-GCC-Chorizo-GPU-Cortex_A7-arm-Release-All',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN',
-  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage',
+  'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_10-Coverage',
   'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN',
   ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All'
    '-SK_USE_DISCARDABLE_SCALEDIMAGECACHE'),

infra/bots/recipes/upload_coverage_results.expected/alternate_bucket.json

@@ -1,22 +1,55 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
   {
     "cmd": [
       "gsutil",
       "cp",
-      "-Z",
+      "[START_DIR]/raw_data.profraw.tar.gz",
-      "[START_DIR]/output.profraw",
+      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
-      "gs://skia-coverage-alt/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },

infra/bots/recipes/upload_coverage_results.expected/failed_all.json

@@ -1,13 +1,67 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
+    ],
+    "name": "upload raw data archive"
+  },
+  {
+    "cmd": [
+      "[START_DIR]/clang_linux/bin/llvm-profdata",
+      "merge",
+      "-sparse",
+      "-o",
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "merge and index"
+  },
   {
     "cmd": [
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
+      "[START_DIR]/output.profdata",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data",
+    "name": "upload parsed data",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -18,10 +72,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
+      "[START_DIR]/output.profdata",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 2)",
+    "name": "upload parsed data (attempt 2)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -32,10 +86,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
+      "[START_DIR]/output.profdata",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 3)",
+    "name": "upload parsed data (attempt 3)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -46,10 +100,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
+      "[START_DIR]/output.profdata",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 4)",
+    "name": "upload parsed data (attempt 4)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -60,10 +114,10 @@
       "gsutil",
       "cp",
       "-Z",
-      "[START_DIR]/output.profraw",
+      "[START_DIR]/output.profdata",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload raw data (attempt 5)",
+    "name": "upload parsed data (attempt 5)",
     "~followup_annotations": [
       "step returned non-zero exit code: 1",
       "@@@STEP_FAILURE@@@"
@@ -71,7 +125,7 @@
   },
   {
     "name": "$result",
-    "reason": "Step('upload raw data (attempt 5)') failed with return_code 1",
+    "reason": "Step('upload parsed data (attempt 5)') failed with return_code 1",
     "recipe_result": null,
     "status_code": 1
   }

infra/bots/recipes/upload_coverage_results.expected/failed_once.json

@@ -1,36 +1,55 @@
 [
   {
     "cmd": [
-      "gsutil",
+      "python",
-      "cp",
+      "-u",
-      "-Z",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
-      "[START_DIR]/output.profraw",
+      "--json-output",
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
     ],
-    "name": "upload raw data",
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
     "~followup_annotations": [
-      "step returned non-zero exit code: 1",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
-      "@@@STEP_FAILURE@@@"
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
     ]
   },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
   {
     "cmd": [
       "gsutil",
       "cp",
-      "-Z",
+      "[START_DIR]/raw_data.profraw.tar.gz",
-      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
     ],
-    "name": "upload raw data (attempt 2)"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },
@@ -42,7 +61,21 @@
       "[START_DIR]/output.profdata",
       "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
     ],
-    "name": "upload parsed data"
+    "name": "upload parsed data",
+    "~followup_annotations": [
+      "step returned non-zero exit code: 1",
+      "@@@STEP_FAILURE@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "gsutil",
+      "cp",
+      "-Z",
+      "[START_DIR]/output.profdata",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profdata"
+    ],
+    "name": "upload parsed data (attempt 2)"
   },
   {
     "cmd": [

infra/bots/recipes/upload_coverage_results.expected/normal_bot.json

@@ -1,22 +1,55 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
   {
     "cmd": [
       "gsutil",
       "cp",
-      "-Z",
+      "[START_DIR]/raw_data.profraw.tar.gz",
-      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
-      "gs://skia-coverage/commit/abc123/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },

infra/bots/recipes/upload_coverage_results.expected/trybot.json

@@ -1,22 +1,55 @@
 [
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+      "--json-output",
+      "/path/to/tmp/json",
+      "glob",
+      "[START_DIR]",
+      "*.profraw"
+    ],
+    "infra_step": true,
+    "name": "find raw inputs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/a.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/b.raw@@@",
+      "@@@STEP_LOG_LINE@glob@[START_DIR]/c.raw@@@",
+      "@@@STEP_LOG_END@glob@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "tar",
+      "-zcvf",
+      "[START_DIR]/raw_data.profraw.tar.gz",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
+    ],
+    "name": "create raw data archive"
+  },
   {
     "cmd": [
       "gsutil",
       "cp",
-      "-Z",
+      "[START_DIR]/raw_data.profraw.tar.gz",
-      "[START_DIR]/output.profraw",
+      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw.tar.gz"
-      "gs://skia-coverage/trybot/456789/12/Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All.profraw"
     ],
-    "name": "upload raw data"
+    "name": "upload raw data archive"
   },
   {
     "cmd": [
       "[START_DIR]/clang_linux/bin/llvm-profdata",
       "merge",
       "-sparse",
-      "[START_DIR]/output.profraw",
       "-o",
-      "[START_DIR]/output.profdata"
+      "[START_DIR]/output.profdata",
+      "[START_DIR]/a.raw",
+      "[START_DIR]/b.raw",
+      "[START_DIR]/c.raw"
     ],
     "name": "merge and index"
   },

infra/bots/recipes/upload_coverage_results.py

@@ -10,21 +10,26 @@ import calendar
 
 
 DEPS = [
+  'gsutil',
+  'recipe_engine/file',
   'recipe_engine/json',
   'recipe_engine/path',
   'recipe_engine/properties',
+  'recipe_engine/python',
+  'recipe_engine/raw_io',
   'recipe_engine/step',
   'recipe_engine/time',
-  'gsutil',
 ]
 
 
 TRY_JOB_FOLDER = 'trybot/%s/%s/' # % (issue_number, patchset_number)
 COMMIT_FOLDER = 'commit/%s/'      # % (git_revision)
 
-RAW_FILE = '%s.profraw'
+RAW_FILE = '*.profraw'
 PARSED_FILE = '%s.profdata'
 SUMMARY_FILE = '%s.summary'
+
+COVERAGE_RAW_ARCHIVE = '%s.profraw.tar.gz'
 # Text is an easier format to read with machines (e.g. for Gerrit).
 COVERAGE_TEXT_FILE = '%s.text.tar'
 # HTML is a quick and dirty browsable format. (e.g. for coverage.skia.org)
@@ -41,8 +46,13 @@ def RunSteps(api):
   builder_name = api.properties['buildername']
   bucket = api.properties['gs_bucket']
 
-  # The raw data is brought in as an isolated input.
+  # The raw data files are brought in as isolated inputs. It is possible
-  raw_data = api.path['start_dir'].join('output.profraw')
+  # for there to be 1 if the coverage task wasn't broken up.
+  raw_inputs = api.file.glob_paths('find raw inputs', api.path['start_dir'],
+                                   RAW_FILE,
+                                   test_data=['a.raw', 'b.raw', 'c.raw'])
+
+
   # The instrumented executable is brought in as an isolated input.
   executable = api.path['start_dir'].join('out','Debug','dm')
   # clang_dir is brought in via CIPD.
@@ -56,19 +66,28 @@ def RunSteps(api):
   if issue and patchset:
     path = TRY_JOB_FOLDER % (issue, patchset)
 
-  gcs_file = RAW_FILE % builder_name
+  # Upload the raw files, tarred together to decrease upload time and
-  api.gsutil.cp('raw data', raw_data,
+  # improve compression.
-                   'gs://%s/%s%s' % (bucket, path, gcs_file), ['-Z'])
+  tar_file = api.path['start_dir'].join('raw_data.profraw.tar.gz')
+  cmd = ['tar', '-zcvf', tar_file]
+  cmd.extend(raw_inputs)
+  api.step('create raw data archive', cmd=cmd)
 
-  # Merge and Index the data.
+  gcs_file = COVERAGE_RAW_ARCHIVE % builder_name
+  api.gsutil.cp('raw data archive', tar_file,
+                'gs://%s/%s%s' % (bucket, path, gcs_file))
+
+  # Merge all the raw data files together, then index the data.
+  # This creates one cohesive
   indexed_data = api.path['start_dir'].join('output.profdata')
+  cmd = [clang_dir.join('llvm-profdata'),
+         'merge',
+         '-sparse',
+         '-o',
+         indexed_data]
+  cmd.extend(raw_inputs)
   api.step('merge and index',
-           cmd=[clang_dir.join('llvm-profdata'),
+           cmd=cmd)
-               'merge',
-               '-sparse',
-               raw_data,
-               '-o',
-               indexed_data ])
 
   gcs_file = PARSED_FILE % builder_name
   api.gsutil.cp('parsed data', indexed_data,
@@ -149,7 +168,7 @@ def GenTests(api):
                    gs_bucket='skia-coverage',
                    revision='abc123',
                    path_config='kitchen') +
-    api.step_data('upload raw data', retcode=1)
+    api.step_data('upload parsed data', retcode=1)
   )
 
   yield (
@@ -158,11 +177,11 @@ def GenTests(api):
                    gs_bucket='skia-coverage',
                    revision='abc123',
                    path_config='kitchen') +
-    api.step_data('upload raw data', retcode=1) +
+    api.step_data('upload parsed data', retcode=1) +
-    api.step_data('upload raw data (attempt 2)', retcode=1) +
+    api.step_data('upload parsed data (attempt 2)', retcode=1) +
-    api.step_data('upload raw data (attempt 3)', retcode=1) +
+    api.step_data('upload parsed data (attempt 3)', retcode=1) +
-    api.step_data('upload raw data (attempt 4)', retcode=1) +
+    api.step_data('upload parsed data (attempt 4)', retcode=1) +
-    api.step_data('upload raw data (attempt 5)', retcode=1)
+    api.step_data('upload parsed data (attempt 5)', retcode=1)
   )
 
   yield (