AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Isolate the biggest CIPD assets to trim down on overhead

Browse Source 
This change, when combined with https://skia-review.googlesource.com/c/buildbot/+/77701
should cut down on the duplicated work of extracting the large
toolchains from CIPD.

Since the Isolate* steps can be cached (i.e. are idempotent)
they will only run about 1/week (unless updated) and
all subsequent tasks (primarily Builds) will go much faster.

We estimate the overhead on Build-Debian-Android to go from
about 90s (which was more time than the actual build) to
about 10s.  Build-Win-Vulkan's overhead will improve from
about 180s to about 35s (1/3 of which is uploading to isolate).

Other CIPD assets could be handled in a similar fashion;
the ones here are the biggest offenders and the lowest
hanging fruit.  Doing this to other assets (e.g. clang_win)
would have minimal improvements (<10s).

There are other tasks with large amounts of overhead
(e.g. Build-Mac-Android, Build-Win-Android, Build-WASM)
but none of those are depended on by any Tests, so any
speed-ups would have less wide-reaching-impact, at the
cost of using more Isolate cache/diskspace.

See https://docs.google.com/document/d/1DFlcpqg7XqEPE5oYT1V3so2ih2285heS5w3mPT-GMBA/edit#
for more information.

Bug: skia:
Change-Id: I40dd87fe72c3d49292762a09dad6df0dfbe78f61
Reviewed-on: https://skia-review.googlesource.com/77560
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Kevin Lubick <kjlubick@google.com>
This commit is contained in:
Kevin Lubick 2017-11-29 14:45:14 -05:00 committed by Skia Commit-Bot
parent def86243a9
commit 814b149ce3
5 changed files with 176 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
infra/bots/gen_tasks.go
View File

@ -30,10 +30,13 @@ import (
)
const (
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
ISOLATE_SKIMAGE_NAME = "Housekeeper-PerCommit-IsolateSkImage"
ISOLATE_SKP_NAME = "Housekeeper-PerCommit-IsolateSKP"
ISOLATE_SVG_NAME = "Housekeeper-PerCommit-IsolateSVG"
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
ISOLATE_SKIMAGE_NAME = "Housekeeper-PerCommit-IsolateSkImage"
ISOLATE_SKP_NAME = "Housekeeper-PerCommit-IsolateSKP"
ISOLATE_SVG_NAME = "Housekeeper-PerCommit-IsolateSVG"
ISOLATE_NDK_LINUX_NAME = "Housekeeper-PerCommit-IsolateAndroidNDKLinux"
ISOLATE_WIN_TOOLCHAIN_NAME = "Housekeeper-PerCommit-IsolateWinToolchain"
ISOLATE_WIN_VULKAN_SDK_NAME = "Housekeeper-PerCommit-IsolateWinVulkanSDK"
DEFAULT_OS_DEBIAN = "Debian-9.1"
DEFAULT_OS_LINUX_GCE = "Debian-9.2"
@ -418,6 +421,18 @@ var ISOLATE_ASSET_MAPPING = map[string]isolateAssetCfg{
isolateFile: "isolate_svg.isolate",
cipdPkg: "svg",
},
ISOLATE_NDK_LINUX_NAME: {
isolateFile: "isolate_ndk_linux.isolate",
cipdPkg: "android_ndk_linux",
},
ISOLATE_WIN_TOOLCHAIN_NAME: {
isolateFile: "isolate_win_toolchain.isolate",
cipdPkg: "win_toolchain",
},
ISOLATE_WIN_VULKAN_SDK_NAME: {
isolateFile: "isolate_win_vulkan_sdk.isolate",
cipdPkg: "win_vulkan_sdk",
},
}
// bundleRecipes generates the task to bundle and isolate the recipes.
@ -462,6 +477,7 @@ func getIsolatedCIPDDeps(parts map[string]string) []string {
func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string {
// Collect the necessary CIPD packages.
pkgs := []*specs.CipdPackage{}
deps := []string{}
// Android bots require a toolchain.
if strings.Contains(name, "Android") {
@ -472,7 +488,7 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
pkg.Path = "n"
pkgs = append(pkgs, pkg)
} else {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("android_ndk_linux"))
deps = append(deps, isolateCIPDAsset(b, ISOLATE_NDK_LINUX_NAME))
}
} else if strings.Contains(name, "Chromecast") {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("cast_toolchain"))
@ -496,12 +512,12 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("emscripten_sdk"))
}
} else if strings.Contains(name, "Win") {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("win_toolchain"))
deps = append(deps, isolateCIPDAsset(b, ISOLATE_WIN_TOOLCHAIN_NAME))
if strings.Contains(name, "Clang") {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_win"))
}
if strings.Contains(name, "Vulkan") {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("win_vulkan_sdk"))
deps = append(deps, isolateCIPDAsset(b, ISOLATE_WIN_VULKAN_SDK_NAME))
}
}
@ -511,6 +527,7 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
b.MustAddTask(name, &specs.TaskSpec{
CipdPackages: pkgs,
Dimensions: dimensions,
Dependencies: deps,
ExtraArgs: []string{
"--workdir", "../../..", "compile",
fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),

7
infra/bots/isolate_ndk_linux.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'command': [
'/bin/cp', '-rL', 'android_ndk_linux', '${ISOLATED_OUTDIR}',
],
},
}

7
infra/bots/isolate_win_toolchain.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'command': [
'/bin/cp', '-rL', 't', '${ISOLATED_OUTDIR}',
],
},
}

7
infra/bots/isolate_win_vulkan_sdk.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'command': [
'/bin/cp', '-rL', 'win_vulkan_sdk', '${ISOLATED_OUTDIR}',
],
},
}

355
infra/bots/tasks.json
View File

@ -3080,12 +3080,8 @@
},
"tasks": {
"Build-Debian9-Clang-arm-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3150,12 +3146,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3180,12 +3172,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm-Release-Android_API26": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3250,12 +3238,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm64-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3280,12 +3264,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm64-Debug-Android_Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3310,12 +3290,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm64-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3340,12 +3316,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-arm64-Release-Android_Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3370,12 +3342,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-mips64el-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3400,12 +3368,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-mips64el-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3430,12 +3394,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-mipsel-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3460,12 +3420,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-mipsel-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3490,12 +3446,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x64-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3520,12 +3472,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x64-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3580,12 +3528,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x86-Debug-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3610,12 +3554,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x86-Debug-Android_Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3640,12 +3580,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x86-Release-Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -3670,12 +3606,8 @@
"priority": 0.8
},
"Build-Debian9-Clang-x86-Release-Android_Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -4623,12 +4555,8 @@
"priority": 0.8
},
"Build-Debian9-GCC-x86_64-Release-Flutter_Android": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateAndroidNDKLinux"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5082,17 +5010,15 @@
},
"Build-Win-Clang-x86_64-Debug": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5117,17 +5043,15 @@
},
"Build-Win-Clang-x86_64-Debug-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5152,22 +5076,16 @@
},
"Build-Win-Clang-x86_64-Debug-Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
},
{
"name": "skia/bots/win_vulkan_sdk",
"path": "win_vulkan_sdk",
"version": "version:2"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain",
"Housekeeper-PerCommit-IsolateWinVulkanSDK"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5192,17 +5110,15 @@
},
"Build-Win-Clang-x86_64-Release": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5227,17 +5143,15 @@
},
"Build-Win-Clang-x86_64-Release-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5262,22 +5176,16 @@
},
"Build-Win-Clang-x86_64-Release-Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/clang_win",
"path": "clang_win",
"version": "version:0"
},
{
"name": "skia/bots/win_vulkan_sdk",
"path": "win_vulkan_sdk",
"version": "version:2"
}
],
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain",
"Housekeeper-PerCommit-IsolateWinVulkanSDK"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
@ -5301,12 +5209,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86-Debug": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5331,12 +5235,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86-Debug-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5361,12 +5261,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86-Debug-Exceptions": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5391,12 +5287,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86-Release": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5421,12 +5313,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86-Release-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5451,12 +5339,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Debug": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5481,12 +5365,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Debug-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5511,17 +5391,9 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Debug-Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/win_vulkan_sdk",
"path": "win_vulkan_sdk",
"version": "version:2"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain",
"Housekeeper-PerCommit-IsolateWinVulkanSDK"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5546,12 +5418,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Release": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5576,12 +5444,8 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Release-ANGLE": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5606,17 +5470,9 @@
"priority": 0.8
},
"Build-Win-MSVC-x86_64-Release-Vulkan": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
},
{
"name": "skia/bots/win_vulkan_sdk",
"path": "win_vulkan_sdk",
"version": "version:2"
}
"dependencies": [
"Housekeeper-PerCommit-IsolateWinToolchain",
"Housekeeper-PerCommit-IsolateWinVulkanSDK"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
@ -5876,6 +5732,23 @@
"isolate": "infra_skia.isolate",
"priority": 0.8
},
"Housekeeper-PerCommit-IsolateAndroidNDKLinux": {
"cipd_packages": [
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:10"
}
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"isolate": "isolate_ndk_linux.isolate",
"priority": 0.7
},
"Housekeeper-PerCommit-IsolateSKP": {
"cipd_packages": [
{
@ -5927,6 +5800,40 @@
"isolate": "isolate_skimage.isolate",
"priority": 0.7
},
"Housekeeper-PerCommit-IsolateWinToolchain": {
"cipd_packages": [
{
"name": "skia/bots/win_toolchain",
"path": "t",
"version": "version:6"
}
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"isolate": "isolate_win_toolchain.isolate",
"priority": 0.7
},
"Housekeeper-PerCommit-IsolateWinVulkanSDK": {
"cipd_packages": [
{
"name": "skia/bots/win_vulkan_sdk",
"path": "win_vulkan_sdk",
"version": "version:2"
}
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"isolate": "isolate_win_vulkan_sdk.isolate",
"priority": 0.7
},
"Housekeeper-Weekly-RecreateSKPs": {
"cipd_packages": [
{