Revert "Reland "[infra] Switch from isolate to RBE-CAS""

This reverts commit 540b51ce8f.

Reason for revert: RPI disk space

Original change's description:
> Reland "[infra] Switch from isolate to RBE-CAS"
>
> Bug: skia:10883
> Change-Id: Iec82a07fdf3c0807a9bb1870309eded85d4f0b1e
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/341679
> Commit-Queue: Eric Boren <borenet@google.com>
> Reviewed-by: Weston Tracey <westont@google.com>

TBR=borenet@google.com,westont@google.com

Change-Id: I0156842b995d275d8deb5bd73c6877de7f8af72e
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: skia:10883
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/342596
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
This commit is contained in:
Eric Boren 2020-12-09 18:09:20 +00:00 committed by Skia Commit-Bot
parent 050ea3a580
commit 32c6099d92
31 changed files with 1687 additions and 1757 deletions

View File

@ -201,6 +201,27 @@ def _CheckIncludesFormatted(input_api, output_api):
return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))] return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
return [] return []
def _CheckCompileIsolate(input_api, output_api):
"""Ensure that gen_compile_isolate.py does not change compile.isolate."""
# Only run the check if files were added or removed.
results = []
script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
isolate = os.path.join('infra', 'bots', 'compile.isolated')
for f in input_api.AffectedFiles():
if f.Action() in ('A', 'D', 'R'):
break
if f.LocalPath() in (script, isolate):
break
else:
return results
cmd = ['python', script, 'test']
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
results.append(output_api.PresubmitError(e.output))
return results
class _WarningsAsErrors(): class _WarningsAsErrors():
def __init__(self, output_api): def __init__(self, output_api):
@ -255,6 +276,7 @@ def _CommonChecks(input_api, output_api):
results.extend(_IfDefChecks(input_api, output_api)) results.extend(_IfDefChecks(input_api, output_api))
results.extend(_CopyrightChecks(input_api, output_api, results.extend(_CopyrightChecks(input_api, output_api,
source_file_filter=sources)) source_file_filter=sources))
results.extend(_CheckCompileIsolate(input_api, output_api))
results.extend(_CheckDEPSValid(input_api, output_api)) results.extend(_CheckDEPSValid(input_api, output_api))
results.extend(_CheckIncludesFormatted(input_api, output_api)) results.extend(_CheckIncludesFormatted(input_api, output_api))
results.extend(_CheckGNFormatted(input_api, output_api)) results.extend(_CheckGNFormatted(input_api, output_api))

View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'assets/',
],
},
}

View File

@ -0,0 +1,13 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../modules/canvaskit',
'../../modules/pathkit/perf/perfReporter.js',
'../../modules/pathkit/tests/testReporter.js',
'../canvaskit',
],
},
}

View File

@ -0,0 +1,49 @@
{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
'../../../.gclient',
'../../.clang-format',
'../../.clang-tidy',
'../../.gn',
'../../BUILD.gn',
'../../bench',
'../../bin/fetch-clang-format',
'../../bin/fetch-gn',
'../../build/fuchsia',
'../../build_overrides',
'../../buildtools',
'../../client_utils/android',
'../../dm',
'../../docs/examples',
'../../example',
'../../experimental',
'../../fuzz',
'../../gm',
'../../gn',
'../../include',
'../../modules',
'../../platform_tools/android/apps/skottie/skottielib/src/main/cpp/native-lib.cpp',
'../../platform_tools/android/launcher/skia_launcher.cpp',
'../../platform_tools/android/vulkan/Skia_Vulkan_Android.h',
'../../platform_tools/ios/app/LaunchScreen.storyboard',
'../../platform_tools/libraries/include/arcore_c_api.h',
'../../resources',
'../../samplecode',
'../../src',
'../../tests',
'../../third_party',
'../../tools',
'../canvaskit',
'../pathkit',
'assets/android_ndk_darwin/VERSION',
'assets/android_ndk_linux/VERSION',
'assets/android_ndk_windows/VERSION',
'assets/cast_toolchain/VERSION',
'assets/clang_linux/VERSION',
'assets/clang_win/VERSION',
],
},
}

View File

@ -0,0 +1,11 @@
{
'includes': [
'infrabots.isolate',
'run_recipe.isolate',
],
'variables': {
'files': [
'../../../.gclient',
],
},
}

1
infra/bots/empty.isolate Normal file
View File

@ -0,0 +1 @@
{}

234
infra/bots/gen_compile_isolate.py Executable file
View File

@ -0,0 +1,234 @@
#!/usr/bin/env python
#
# Copyright 2019 Google LLC
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import difflib
import os
import re
import subprocess
import sys
# Any files in Git which match these patterns will be included, either directly
# or indirectly via a parent dir.
PATH_PATTERNS = [
r'.*\.c$',
r'.*\.cc$',
r'.*\.cpp$',
r'.*\.gn$',
r'.*\.gni$',
r'.*\.h$',
r'.*\.mm$',
r'.*\.storyboard$',
]
# These paths are always added to the inclusion list. Note that they may not
# appear in the isolate if they are included indirectly via a parent dir.
EXPLICIT_PATHS = [
'../.gclient',
'.clang-format',
'.clang-tidy',
'bin/fetch-clang-format',
'bin/fetch-gn',
'buildtools',
'infra/bots/assets/android_ndk_darwin/VERSION',
'infra/bots/assets/android_ndk_linux/VERSION',
'infra/bots/assets/android_ndk_windows/VERSION',
'infra/bots/assets/cast_toolchain/VERSION',
'infra/bots/assets/clang_linux/VERSION',
'infra/bots/assets/clang_win/VERSION',
'infra/canvaskit',
'infra/pathkit',
'resources',
'third_party/externals',
]
# If a parent path contains more than this many immediate child paths (ie. files
# and dirs which are directly inside it as opposed to indirect descendants), we
# will include the parent in the isolate file instead of the children. This
# results in a simpler isolate file which should need to be changed less often.
COMBINE_PATHS_THRESHOLD = 3
# Template for the isolate file content.
ISOLATE_TMPL = '''{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
%s
],
},
}
'''
# Absolute path to the infra/bots dir.
INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
# Absolute path to the compile.isolate file.
ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
def all_paths():
"""Return all paths which are checked in to git."""
repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
return output.splitlines()
def get_relevant_paths():
"""Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
paths = []
for f in all_paths():
for regexp in PATH_PATTERNS:
if re.match(regexp, f):
paths.append(f)
break
paths.extend(EXPLICIT_PATHS)
return paths
class Tree(object):
"""Tree helps with deduplicating and collapsing paths."""
class Node(object):
"""Node represents an individual node in a Tree."""
def __init__(self, name):
self._children = {}
self._name = name
self._is_leaf = False
@property
def is_root(self):
"""Return True iff this is the root node."""
return self._name is None
def add(self, entry):
"""Add the given entry (given as a list of strings) to the Node."""
# Remove the first element if we're not the root node.
if not self.is_root:
if entry[0] != self._name:
raise ValueError('Cannot add a non-matching entry to a Node!')
entry = entry[1:]
# If the entry is now empty, this node is a leaf.
if not entry:
self._is_leaf = True
return
# Add a child node.
if not self._is_leaf:
child = self._children.get(entry[0])
if not child:
child = Tree.Node(entry[0])
self._children[entry[0]] = child
child.add(entry)
# If we have more than COMBINE_PATHS_THRESHOLD immediate children,
# combine them into this node.
immediate_children = 0
for child in self._children.itervalues():
if child._is_leaf:
immediate_children += 1
if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
self._is_leaf = True
self._children = {}
def entries(self):
"""Return the entries represented by this node and its children.
Will not return children in the following cases:
- This Node is a leaf, ie. it represents an entry which was explicitly
inserted into the Tree, as opposed to only part of a path to other
entries.
- This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
thus has been upgraded to a leaf node.
"""
if self._is_leaf:
return [self._name]
rv = []
for child in self._children.itervalues():
for entry in child.entries():
if not self.is_root:
entry = self._name + '/' + entry
rv.append(entry)
return rv
def __init__(self):
self._root = Tree.Node(None)
def add(self, entry):
"""Add the given entry to the tree."""
split = entry.split('/')
if split[-1] == '':
split = split[:-1]
self._root.add(split)
def entries(self):
"""Return the list of entries in the tree.
Entries will be de-duplicated as follows:
- Any entry which is a sub-path of another entry will not be returned.
- Any entry which was not explicitly inserted but has children exceeding
the COMBINE_PATHS_THRESHOLD will be returned while its children will not
be returned.
"""
return self._root.entries()
def relpath(repo_path):
"""Return a relative path to the given path within the repo.
The path is relative to the infra/bots dir, where the compile.isolate file
lives.
"""
repo_path = '../../' + repo_path
repo_path = repo_path.replace('../../infra/', '../')
repo_path = repo_path.replace('../bots/', '')
return repo_path
def get_isolate_content(paths):
"""Construct the new content of the isolate file based on the given paths."""
lines = [' \'%s\',' % relpath(p) for p in paths]
lines.sort()
return ISOLATE_TMPL % '\n'.join(lines)
def main():
"""Regenerate the compile.isolate file, or verify that it hasn't changed."""
testing = False
if len(sys.argv) == 2 and sys.argv[1] == 'test':
testing = True
elif len(sys.argv) != 1:
print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
sys.exit(1)
tree = Tree()
for p in get_relevant_paths():
tree.add(p)
content = get_isolate_content(tree.entries())
if testing:
with open(ISOLATE_FILE, 'rb') as f:
expect_content = f.read()
if content != expect_content:
print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
a = expect_content.splitlines()
b = content.splitlines()
diff = difflib.context_diff(a, b, lineterm='')
for line in diff:
sys.stderr.write(line + '\n')
print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
sys.exit(1)
else:
with open(ISOLATE_FILE, 'wb') as f:
f.write(content)
if __name__ == '__main__':
main()

View File

@ -1,224 +0,0 @@
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package gen_tasks_logic
import (
"log"
"os/exec"
"regexp"
"sort"
"strings"
"go.skia.org/infra/go/cas/rbe"
"go.skia.org/infra/task_scheduler/go/specs"
)
const (
// If a parent path contains more than this many immediate child paths (ie.
// files and dirs which are directly inside it as opposed to indirect
// descendants), we will include the parent in the isolate file instead of
// the children. This results in a simpler CasSpec which should need to be
// changed less often.
combinePathsThreshold = 3
)
var (
// Any files in Git which match these patterns will be included, either
// directly or indirectly via a parent dir.
pathRegexes = []*regexp.Regexp{
regexp.MustCompile(`.*\.c$`),
regexp.MustCompile(`.*\.cc$`),
regexp.MustCompile(`.*\.cpp$`),
regexp.MustCompile(`.*\.gn$`),
regexp.MustCompile(`.*\.gni$`),
regexp.MustCompile(`.*\.h$`),
regexp.MustCompile(`.*\.mm$`),
regexp.MustCompile(`.*\.storyboard$`),
}
// These paths are always added to the inclusion list. Note that they may
// not appear in the CasSpec if they are included indirectly via a parent
// dir.
explicitPaths = []string{
".clang-format",
".clang-tidy",
"bin/fetch-clang-format",
"bin/fetch-gn",
"buildtools",
"infra/bots/assets/android_ndk_darwin/VERSION",
"infra/bots/assets/android_ndk_linux/VERSION",
"infra/bots/assets/android_ndk_windows/VERSION",
"infra/bots/assets/cast_toolchain/VERSION",
"infra/bots/assets/clang_linux/VERSION",
"infra/bots/assets/clang_win/VERSION",
"infra/bots/run_recipe.py",
"infra/canvaskit",
"infra/pathkit",
"resources",
"third_party/externals",
}
)
// getAllCheckedInPaths returns every path checked in to the repo.
func getAllCheckedInPaths() []string {
cmd := exec.Command("git", "ls-files")
cmd.Dir = CheckoutRoot()
output, err := cmd.CombinedOutput()
if err != nil {
log.Fatal(err)
}
split := strings.Split(string(output), "\n")
rv := make([]string, 0, len(split))
for _, line := range split {
if line != "" {
rv = append(rv, line)
}
}
return rv
}
// getRelevantPaths returns all paths needed by compile tasks.
func getRelevantPaths() []string {
rv := []string{}
for _, path := range getAllCheckedInPaths() {
for _, regex := range pathRegexes {
if regex.MatchString(path) {
rv = append(rv, path)
break
}
}
}
return append(rv, explicitPaths...)
}
// node is a single node in a directory tree of task inputs.
type node struct {
children map[string]*node
name string
isLeaf bool
}
// newNode returns a node instance.
func newNode(name string) *node {
return &node{
children: map[string]*node{},
name: name,
isLeaf: false,
}
}
// isRoot returns true iff this is the root node.
func (n *node) isRoot() bool {
return n.name == ""
}
// add the given entry (given as a slice of path components) to the node.
func (n *node) add(entry []string) {
// Remove the first element if we're not the root node.
if !n.isRoot() {
if entry[0] != n.name {
log.Fatalf("Failed to compute compile CAS inputs; attempting to add entry %v to node %q", entry, n.name)
}
entry = entry[1:]
// If the entry is now empty, this node is a leaf.
if len(entry) == 0 {
n.isLeaf = true
return
}
}
// Add a child node.
if !n.isLeaf {
name := entry[0]
child, ok := n.children[name]
if !ok {
child = newNode(name)
n.children[name] = child
}
child.add(entry)
// If we have more than combinePathsThreshold immediate children,
// combine them into this node.
immediateChilden := 0
for _, child := range n.children {
if child.isLeaf {
immediateChilden++
}
if !n.isRoot() && immediateChilden >= combinePathsThreshold {
n.isLeaf = true
n.children = map[string]*node{}
}
}
}
}
// entries returns the entries represented by this node and its children.
// Will not return children in the following cases:
// - This Node is a leaf, ie. it represents an entry which was explicitly
// inserted into the Tree, as opposed to only part of a path to other
// entries.
// - This Node has immediate children exceeding combinePathsThreshold and
// thus has been upgraded to a leaf node.
func (n *node) entries() [][]string {
if n.isLeaf {
return [][]string{{n.name}}
}
rv := [][]string{}
for _, child := range n.children {
for _, entry := range child.entries() {
if !n.isRoot() {
entry = append([]string{n.name}, entry...)
}
rv = append(rv, entry)
}
}
return rv
}
// tree represents a directory tree of task inputs.
type tree struct {
root *node
}
// newTree returns a tree instance.
func newTree() *tree {
return &tree{
root: newNode(""),
}
}
// add the given path to the tree. Entries may be combined as defined by
// combinePathsThreshold.
func (t *tree) add(p string) {
split := strings.Split(p, "/")
t.root.add(split)
}
// entries returns all entries in the tree. Entries may be combined as defined
// by combinePathsThreshold.
func (t *tree) entries() []string {
entries := t.root.entries()
rv := make([]string, 0, len(entries))
for _, entry := range entries {
rv = append(rv, strings.Join(append([]string{"skia"}, entry...), "/"))
}
sort.Strings(rv)
return rv
}
// generateCompileCAS creates the CasSpec used for tasks which build Skia.
func generateCompileCAS(b *specs.TasksCfgBuilder) {
t := newTree()
for _, path := range getRelevantPaths() {
t.add(path)
}
spec := &specs.CasSpec{
Root: "..",
Paths: t.entries(),
Excludes: []string{rbe.ExcludeGitDir},
}
b.MustAddCasSpec(CAS_COMPILE, spec)
}

View File

@ -22,30 +22,11 @@ import (
"strings" "strings"
"time" "time"
"go.skia.org/infra/go/cas/rbe"
"go.skia.org/infra/go/cipd" "go.skia.org/infra/go/cipd"
"go.skia.org/infra/task_scheduler/go/specs" "go.skia.org/infra/task_scheduler/go/specs"
) )
const ( const (
CAS_CANVASKIT = "canvaskit"
CAS_COMPILE = "compile"
CAS_EMPTY = "empty" // TODO(borenet): It'd be nice if this wasn't necessary.
CAS_LOTTIE_CI = "lottie-ci"
CAS_LOTTIE_WEB = "lottie-web"
CAS_PATHKIT = "pathkit"
CAS_PERF = "perf"
CAS_PUPPETEER = "puppeteer"
CAS_RUN_RECIPE = "run-recipe"
CAS_RECIPES = "recipes"
CAS_SKOTTIE_WASM = "skottie-wasm"
CAS_SKPBENCH = "skpbench"
CAS_SKQP = "skqp"
CAS_TASK_DRIVERS = "task-drivers"
CAS_TEST = "test"
CAS_WASM_GM = "wasm-gm"
CAS_WHOLE_REPO = "whole-repo"
BUILD_TASK_DRIVERS_NAME = "Housekeeper-PerCommit-BuildTaskDrivers" BUILD_TASK_DRIVERS_NAME = "Housekeeper-PerCommit-BuildTaskDrivers"
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes" BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
ISOLATE_GCLOUD_LINUX_NAME = "Housekeeper-PerCommit-IsolateGCloudLinux" ISOLATE_GCLOUD_LINUX_NAME = "Housekeeper-PerCommit-IsolateGCloudLinux"
@ -89,33 +70,33 @@ var (
// Named caches used by tasks. // Named caches used by tasks.
CACHES_GIT = []*specs.Cache{ CACHES_GIT = []*specs.Cache{
{ &specs.Cache{
Name: "git", Name: "git",
Path: "cache/git", Path: "cache/git",
}, },
{ &specs.Cache{
Name: "git_cache", Name: "git_cache",
Path: "cache/git_cache", Path: "cache/git_cache",
}, },
} }
CACHES_GO = []*specs.Cache{ CACHES_GO = []*specs.Cache{
{ &specs.Cache{
Name: "go_cache", Name: "go_cache",
Path: "cache/go_cache", Path: "cache/go_cache",
}, },
{ &specs.Cache{
Name: "gopath", Name: "gopath",
Path: "cache/gopath", Path: "cache/gopath",
}, },
} }
CACHES_WORKDIR = []*specs.Cache{ CACHES_WORKDIR = []*specs.Cache{
{ &specs.Cache{
Name: "work", Name: "work",
Path: "cache/work", Path: "cache/work",
}, },
} }
CACHES_CCACHE = []*specs.Cache{ CACHES_CCACHE = []*specs.Cache{
{ &specs.Cache{
Name: "ccache", Name: "ccache",
Path: "cache/ccache", Path: "cache/ccache",
}, },
@ -128,21 +109,12 @@ var (
// not also use the normal "work" cache, to prevent issues like // not also use the normal "work" cache, to prevent issues like
// https://bugs.chromium.org/p/skia/issues/detail?id=9749. // https://bugs.chromium.org/p/skia/issues/detail?id=9749.
CACHES_DOCKER = []*specs.Cache{ CACHES_DOCKER = []*specs.Cache{
{ &specs.Cache{
Name: "docker", Name: "docker",
Path: "cache/docker", Path: "cache/docker",
}, },
} }
// CAS_SPEC_WHOLE_REPO is a CasSpec which includes the entire repo. This is
// global so that it can be overridden by other repositories which import
// this file.
CAS_SPEC_WHOLE_REPO = &specs.CasSpec{
Root: "..",
Paths: []string{"skia"},
Excludes: []string{rbe.ExcludeGitDir},
}
// TODO(borenet): This hacky and bad. // TODO(borenet): This hacky and bad.
CIPD_PKG_LUCI_AUTH = cipd.MustGetPackage("infra/tools/luci-auth/${platform}") CIPD_PKG_LUCI_AUTH = cipd.MustGetPackage("infra/tools/luci-auth/${platform}")
@ -151,7 +123,7 @@ var (
CIPD_PKGS_XCODE = []*specs.CipdPackage{ CIPD_PKGS_XCODE = []*specs.CipdPackage{
// https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#317 // https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#317
// This package is really just an installer for XCode. // This package is really just an installer for XCode.
{ &specs.CipdPackage{
Name: "infra/tools/mac_toolchain/${platform}", Name: "infra/tools/mac_toolchain/${platform}",
Path: "mac_toolchain", Path: "mac_toolchain",
// When this is updated, also update // When this is updated, also update
@ -177,39 +149,39 @@ var (
// ISOLATE_ASSET_MAPPING maps the name of an asset to the configuration // ISOLATE_ASSET_MAPPING maps the name of an asset to the configuration
// for how the CIPD package should be installed for a given task. // for how the CIPD package should be installed for a given task.
ISOLATE_ASSET_MAPPING = map[string]uploadAssetCASCfg{ ISOLATE_ASSET_MAPPING = map[string]isolateAssetCfg{
"gcloud_linux": { "gcloud_linux": {
uploadTaskName: ISOLATE_GCLOUD_LINUX_NAME, isolateTaskName: ISOLATE_GCLOUD_LINUX_NAME,
path: "gcloud_linux", path: "gcloud_linux",
}, },
"skimage": { "skimage": {
uploadTaskName: ISOLATE_SKIMAGE_NAME, isolateTaskName: ISOLATE_SKIMAGE_NAME,
path: "skimage", path: "skimage",
}, },
"skp": { "skp": {
uploadTaskName: ISOLATE_SKP_NAME, isolateTaskName: ISOLATE_SKP_NAME,
path: "skp", path: "skp",
}, },
"svg": { "svg": {
uploadTaskName: ISOLATE_SVG_NAME, isolateTaskName: ISOLATE_SVG_NAME,
path: "svg", path: "svg",
}, },
"mskp": { "mskp": {
uploadTaskName: ISOLATE_MSKP_NAME, isolateTaskName: ISOLATE_MSKP_NAME,
path: "mskp", path: "mskp",
}, },
"android_ndk_linux": { "android_ndk_linux": {
uploadTaskName: ISOLATE_NDK_LINUX_NAME, isolateTaskName: ISOLATE_NDK_LINUX_NAME,
path: "android_ndk_linux", path: "android_ndk_linux",
}, },
"android_sdk_linux": { "android_sdk_linux": {
uploadTaskName: ISOLATE_SDK_LINUX_NAME, isolateTaskName: ISOLATE_SDK_LINUX_NAME,
path: "android_sdk_linux", path: "android_sdk_linux",
}, },
"win_toolchain": { "win_toolchain": {
alwaysIsolate: true, alwaysIsolate: true,
uploadTaskName: ISOLATE_WIN_TOOLCHAIN_NAME, isolateTaskName: ISOLATE_WIN_TOOLCHAIN_NAME,
path: "win_toolchain", path: "win_toolchain",
}, },
} }
) )
@ -344,153 +316,18 @@ func GenTasks(cfg *Config) {
// Create Tasks and Jobs. // Create Tasks and Jobs.
builder := &builder{ builder := &builder{
TasksCfgBuilder: b, TasksCfgBuilder: b,
cfg: cfg, cfg: cfg,
jobNameSchema: schema, jobNameSchema: schema,
jobs: jobs, jobs: jobs,
relpathBaseDir: relpathBaseDir,
relpathTargetDir: relpathTargetDir,
} }
for _, name := range jobs { for _, name := range jobs {
jb := newJobBuilder(builder, name) jb := newJobBuilder(builder, name)
jb.genTasksForJob() jb.genTasksForJob()
jb.finish() jb.finish()
} }
// Create CasSpecs.
b.MustAddCasSpec(CAS_CANVASKIT, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/canvaskit",
"skia/modules/canvaskit",
"skia/modules/pathkit/perf/perfReporter.js",
"skia/modules/pathkit/tests/testReporter.js",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_EMPTY, specs.EmptyCasSpec)
b.MustAddCasSpec(CAS_LOTTIE_CI, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/lottiecap",
"skia/tools/lottie-web-perf",
"skia/tools/lottiecap",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_LOTTIE_WEB, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/tools/lottie-web-perf",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PATHKIT, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/pathkit",
"skia/modules/pathkit",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PERF, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/platform_tools/ios/bin",
"skia/resources",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PUPPETEER, &specs.CasSpec{
Root: ".",
Paths: []string{
"tools/perf-canvaskit-puppeteer",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_RECIPES, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/config/recipes.cfg",
"skia/infra/bots/bundle_recipes.sh",
"skia/infra/bots/README.recipes.md",
"skia/infra/bots/recipe_modules",
"skia/infra/bots/recipes",
"skia/infra/bots/recipes.py",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_RUN_RECIPE, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKOTTIE_WASM, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/tools/skottie-wasm-perf",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKPBENCH, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/tools/skpbench",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKQP, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/skqp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_TASK_DRIVERS, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/go.mod",
"skia/go.sum",
"skia/infra/bots/build_task_drivers.sh",
"skia/infra/bots/run_recipe.py",
"skia/infra/bots/task_drivers",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_TEST, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/platform_tools/ios/bin",
"skia/resources",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_WASM_GM, &specs.CasSpec{
Root: ".",
Paths: []string{
"resources",
"tools/run-wasm-gm-tests",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_WHOLE_REPO, CAS_SPEC_WHOLE_REPO)
generateCompileCAS(b)
builder.MustFinish() builder.MustFinish()
} }
@ -519,9 +356,11 @@ func getCallingDirName() string {
// builder is a wrapper for specs.TasksCfgBuilder. // builder is a wrapper for specs.TasksCfgBuilder.
type builder struct { type builder struct {
*specs.TasksCfgBuilder *specs.TasksCfgBuilder
cfg *Config cfg *Config
jobNameSchema *JobNameSchema jobNameSchema *JobNameSchema
jobs []string jobs []string
relpathBaseDir string
relpathTargetDir string
} }
// marshalJson encodes the given data as JSON and fixes escaping of '<' which Go // marshalJson encodes the given data as JSON and fixes escaping of '<' which Go
@ -547,7 +386,7 @@ func (b *taskBuilder) kitchenTaskNoBundle(recipe string, outputDir string) {
python := "cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}" python := "cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}"
b.cmd(python, "-u", "skia/infra/bots/run_recipe.py", "${ISOLATED_OUTDIR}", recipe, b.getRecipeProps(), b.cfg.Project) b.cmd(python, "-u", "skia/infra/bots/run_recipe.py", "${ISOLATED_OUTDIR}", recipe, b.getRecipeProps(), b.cfg.Project)
// Most recipes want this isolate; they can override if necessary. // Most recipes want this isolate; they can override if necessary.
b.cas(CAS_RUN_RECIPE) b.isolate("swarm_recipe.isolate")
b.timeout(time.Hour) b.timeout(time.Hour)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
b.Spec.ExtraTags = map[string]string{ b.Spec.ExtraTags = map[string]string{
@ -938,6 +777,16 @@ func (b *taskBuilder) defaultSwarmDimensions() {
b.dimension(dims...) b.dimension(dims...)
} }
// relpath returns the relative path to the given file from the config file.
func (b *builder) relpath(f string) string {
target := filepath.Join(b.relpathTargetDir, f)
rv, err := filepath.Rel(b.relpathBaseDir, target)
if err != nil {
log.Fatal(err)
}
return rv
}
// bundleRecipes generates the task to bundle and isolate the recipes. Returns // bundleRecipes generates the task to bundle and isolate the recipes. Returns
// the name of the task, which may be added as a dependency. // the name of the task, which may be added as a dependency.
func (b *jobBuilder) bundleRecipes() string { func (b *jobBuilder) bundleRecipes() string {
@ -948,7 +797,7 @@ func (b *jobBuilder) bundleRecipes() string {
b.linuxGceDimensions(MACHINE_TYPE_SMALL) b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
b.idempotent() b.idempotent()
b.cas(CAS_RECIPES) b.isolate("recipes.isolate")
}) })
return BUNDLE_RECIPES_NAME return BUNDLE_RECIPES_NAME
} }
@ -963,7 +812,7 @@ func (b *jobBuilder) buildTaskDrivers() string {
b.linuxGceDimensions(MACHINE_TYPE_SMALL) b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.idempotent() b.idempotent()
b.cas(CAS_TASK_DRIVERS) b.isolate("task_drivers.isolate")
}) })
return BUILD_TASK_DRIVERS_NAME return BUILD_TASK_DRIVERS_NAME
} }
@ -991,7 +840,7 @@ func (b *jobBuilder) updateGoDeps() {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM) b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.serviceAccount(b.cfg.ServiceAccountRecreateSKPs) b.serviceAccount(b.cfg.ServiceAccountRecreateSKPs)
}) })
} }
@ -1032,7 +881,7 @@ func (b *jobBuilder) createDockerImage(wasm bool) string {
) )
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM) b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker() b.usesDocker()
@ -1065,7 +914,7 @@ func (b *jobBuilder) createPushAppsFromSkiaDockerImage() {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.dep(b.createDockerImage(false)) b.dep(b.createDockerImage(false))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM) b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker() b.usesDocker()
@ -1097,7 +946,7 @@ func (b *jobBuilder) createPushAppsFromWASMDockerImage() {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.dep(b.createDockerImage(true)) b.dep(b.createDockerImage(true))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin") b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM) b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker() b.usesDocker()
@ -1144,10 +993,10 @@ func (b *jobBuilder) compile() string {
} else { } else {
b.addTask(name, func(b *taskBuilder) { b.addTask(name, func(b *taskBuilder) {
recipe := "compile" recipe := "compile"
casSpec := CAS_COMPILE isolate := "compile.isolate"
if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter", "SKQP") { if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter", "SKQP") {
recipe = "sync_and_compile" recipe = "sync_and_compile"
casSpec = CAS_RUN_RECIPE isolate = "swarm_recipe.isolate"
b.recipeProps(EXTRA_PROPS) b.recipeProps(EXTRA_PROPS)
b.usesGit() b.usesGit()
if !b.extraConfig("NoDEPS") { if !b.extraConfig("NoDEPS") {
@ -1157,7 +1006,7 @@ func (b *jobBuilder) compile() string {
b.idempotent() b.idempotent()
} }
b.kitchenTask(recipe, OUTPUT_BUILD) b.kitchenTask(recipe, OUTPUT_BUILD)
b.cas(casSpec) b.isolate(isolate)
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.swarmDimensions() b.swarmDimensions()
if b.extraConfig("Docker", "LottieWeb", "SKQP", "CMake") || b.compiler("EMCC") { if b.extraConfig("Docker", "LottieWeb", "SKQP", "CMake") || b.compiler("EMCC") {
@ -1266,7 +1115,7 @@ func (b *jobBuilder) checkGeneratedFiles() {
// checkGnToBp verifies that the gn_to_bp.py script continues to work. // checkGnToBp verifies that the gn_to_bp.py script continues to work.
func (b *jobBuilder) checkGnToBp() { func (b *jobBuilder) checkGnToBp() {
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
b.cas(CAS_COMPILE) b.isolate("compile.isolate")
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.cmd("./run_gn_to_bp", b.cmd("./run_gn_to_bp",
"--local=false", "--local=false",
@ -1297,7 +1146,7 @@ func (b *jobBuilder) housekeeper() {
// should add as a dependency. // should add as a dependency.
func (b *jobBuilder) g3FrameworkCanary() { func (b *jobBuilder) g3FrameworkCanary() {
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.cmd("./g3_canary", b.cmd("./g3_canary",
"--local=false", "--local=false",
@ -1335,10 +1184,13 @@ func (b *jobBuilder) infra() {
} }
b.recipeProp("repository", specs.PLACEHOLDER_REPO) b.recipeProp("repository", specs.PLACEHOLDER_REPO)
b.kitchenTask("infra", OUTPUT_NONE) b.kitchenTask("infra", OUTPUT_NONE)
b.cas(CAS_WHOLE_REPO) b.isolate("infra_tests.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cipd(specs.CIPD_PKGS_GSUTIL...) b.cipd(specs.CIPD_PKGS_GSUTIL...)
b.idempotent() b.idempotent()
// Repos which call into Skia's gen_tasks.go should define their own
// infra_tests.isolate and therefore should not use relpath().
b.Spec.Isolate = "infra_tests.isolate"
b.usesGo() b.usesGo()
}) })
} }
@ -1437,33 +1289,30 @@ func (b *jobBuilder) dm() {
compileTaskName = b.compile() compileTaskName = b.compile()
} }
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
cas := CAS_TEST isolate := "test_skia_bundled.isolate"
recipe := "test" recipe := "test"
if b.extraConfig("SKQP") { if b.extraConfig("SKQP") {
cas = CAS_SKQP isolate = "skqp.isolate"
recipe = "skqp_test" recipe = "skqp_test"
if b.cpu("Emulator") { if b.cpu("Emulator") {
recipe = "test_skqp_emulator" recipe = "test_skqp_emulator"
} }
} else if b.extraConfig("OpenCL") { } else if b.extraConfig("OpenCL") {
// TODO(dogben): Longer term we may not want this to be called a // TODO(dogben): Longer term we may not want this to be called a "Test" task, but until we start
// "Test" task, but until we start running hs_bench or kx, it will // running hs_bench or kx, it will be easier to fit into the current job name schema.
// be easier to fit into the current job name schema.
recipe = "compute_test" recipe = "compute_test"
} else if b.extraConfig("PathKit") { } else if b.extraConfig("PathKit") {
cas = CAS_PATHKIT isolate = "pathkit.isolate"
recipe = "test_pathkit" recipe = "test_pathkit"
} else if b.extraConfig("CanvasKit") { } else if b.extraConfig("CanvasKit") {
cas = CAS_CANVASKIT isolate = "canvaskit.isolate"
recipe = "test_canvaskit" recipe = "test_canvaskit"
} else if b.extraConfig("LottieWeb") { } else if b.extraConfig("LottieWeb") {
// CAS_LOTTIE_CI differs from CAS_LOTTIE_WEB in that it includes // lottie_ci.isolate differs from lottie_web.isolate in that it includes more of the files,
// more of the files, especially those brought in via DEPS in the // especially those brought in via DEPS in the lottie-ci repo. The main difference between
// lottie-ci repo. The main difference between Perf.+LottieWeb and // Perf.+LottieWeb and Test.+LottieWeb is that the former pulls in the lottie build via
// Test.+LottieWeb is that the former pulls in the lottie build via // npm and the latter always tests at lottie's ToT.
// npm and the latter always tests at lottie's isolate = "lottie_ci.isolate"
// ToT.
cas = CAS_LOTTIE_CI
recipe = "test_lottie_web" recipe = "test_lottie_web"
} }
b.recipeProp("gold_hashes_url", b.cfg.GoldHashesURL) b.recipeProp("gold_hashes_url", b.cfg.GoldHashesURL)
@ -1477,7 +1326,7 @@ func (b *jobBuilder) dm() {
b.dmFlags(iidStr) b.dmFlags(iidStr)
} }
b.kitchenTask(recipe, OUTPUT_TEST) b.kitchenTask(recipe, OUTPUT_TEST)
b.cas(cas) b.isolate(isolate)
b.swarmDimensions() b.swarmDimensions()
if b.extraConfig("CanvasKit", "Docker", "LottieWeb", "PathKit", "SKQP") { if b.extraConfig("CanvasKit", "Docker", "LottieWeb", "PathKit", "SKQP") {
b.usesDocker() b.usesDocker()
@ -1540,7 +1389,7 @@ func (b *jobBuilder) dm() {
func (b *jobBuilder) fm() { func (b *jobBuilder) fm() {
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
b.cas(CAS_TEST) b.isolate("test_skia_bundled.isolate")
b.dep(b.buildTaskDrivers(), b.compile()) b.dep(b.buildTaskDrivers(), b.compile())
b.cmd("./fm_driver", b.cmd("./fm_driver",
"--local=false", "--local=false",
@ -1560,7 +1409,7 @@ func (b *jobBuilder) fm() {
// Canary-G3 does not use this path because it is very different from other autorollers. // Canary-G3 does not use this path because it is very different from other autorollers.
func (b *jobBuilder) canary(rollerName string) { func (b *jobBuilder) canary(rollerName string) {
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.cmd("./canary", b.cmd("./canary",
"--local=false", "--local=false",
@ -1593,7 +1442,7 @@ func (b *jobBuilder) puppeteer() {
b.dep(b.buildTaskDrivers(), compileTaskName) b.dep(b.buildTaskDrivers(), compileTaskName)
b.output(OUTPUT_PERF) b.output(OUTPUT_PERF)
b.timeout(20 * time.Minute) b.timeout(20 * time.Minute)
b.cas(CAS_PUPPETEER) b.isolate("perf_puppeteer.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
webglversion := "2" webglversion := "2"
@ -1704,7 +1553,7 @@ func (b *jobBuilder) cifuzz() {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.output("cifuzz_out") b.output("cifuzz_out")
b.timeout(60 * time.Minute) b.timeout(60 * time.Minute)
b.cas(CAS_WHOLE_REPO) b.isolate("whole_repo.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cmd( b.cmd(
"./cifuzz", "./cifuzz",
@ -1730,31 +1579,31 @@ func (b *jobBuilder) perf() {
doUpload := b.release() && b.doUpload() doUpload := b.release() && b.doUpload()
b.addTask(b.Name, func(b *taskBuilder) { b.addTask(b.Name, func(b *taskBuilder) {
recipe := "perf" recipe := "perf"
cas := CAS_PERF isolate := "perf_skia_bundled.isolate"
if b.extraConfig("Skpbench") { if b.extraConfig("Skpbench") {
recipe = "skpbench" recipe = "skpbench"
cas = CAS_SKPBENCH isolate = "skpbench_skia_bundled.isolate"
} else if b.extraConfig("PathKit") { } else if b.extraConfig("PathKit") {
cas = CAS_PATHKIT isolate = "pathkit.isolate"
recipe = "perf_pathkit" recipe = "perf_pathkit"
} else if b.extraConfig("CanvasKit") { } else if b.extraConfig("CanvasKit") {
cas = CAS_CANVASKIT isolate = "canvaskit.isolate"
recipe = "perf_canvaskit" recipe = "perf_canvaskit"
} else if b.extraConfig("SkottieTracing") { } else if b.extraConfig("SkottieTracing") {
recipe = "perf_skottietrace" recipe = "perf_skottietrace"
} else if b.extraConfig("SkottieWASM") { } else if b.extraConfig("SkottieWASM") {
recipe = "perf_skottiewasm_lottieweb" recipe = "perf_skottiewasm_lottieweb"
cas = CAS_SKOTTIE_WASM isolate = "skottie_wasm.isolate"
} else if b.extraConfig("LottieWeb") { } else if b.extraConfig("LottieWeb") {
recipe = "perf_skottiewasm_lottieweb" recipe = "perf_skottiewasm_lottieweb"
cas = CAS_LOTTIE_WEB isolate = "lottie_web.isolate"
} }
b.recipeProps(EXTRA_PROPS) b.recipeProps(EXTRA_PROPS)
if recipe == "perf" { if recipe == "perf" {
b.nanobenchFlags(doUpload) b.nanobenchFlags(doUpload)
} }
b.kitchenTask(recipe, OUTPUT_PERF) b.kitchenTask(recipe, OUTPUT_PERF)
b.cas(cas) b.isolate(isolate)
b.swarmDimensions() b.swarmDimensions()
if b.extraConfig("CanvasKit", "Docker", "PathKit") { if b.extraConfig("CanvasKit", "Docker", "PathKit") {
b.usesDocker() b.usesDocker()
@ -1829,7 +1678,7 @@ func (b *jobBuilder) presubmit() {
}) })
b.recipeProps(EXTRA_PROPS) b.recipeProps(EXTRA_PROPS)
b.kitchenTaskNoBundle("run_presubmit", OUTPUT_NONE) b.kitchenTaskNoBundle("run_presubmit", OUTPUT_NONE)
b.cas(CAS_RUN_RECIPE) b.isolate("run_recipe.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
// Use MACHINE_TYPE_LARGE because it seems to save time versus // Use MACHINE_TYPE_LARGE because it seems to save time versus
// MEDIUM and we want presubmit to be fast. // MEDIUM and we want presubmit to be fast.
@ -1855,7 +1704,7 @@ func (b *jobBuilder) compileWasmGMTests(compileName string) {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.output("wasm_out") b.output("wasm_out")
b.timeout(60 * time.Minute) b.timeout(60 * time.Minute)
b.cas(CAS_COMPILE) b.isolate("compile.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile) b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cache(CACHES_DOCKER...) b.cache(CACHES_DOCKER...)
// For now, we only have one compile mode - a GPU release mode. This should be sufficient to // For now, we only have one compile mode - a GPU release mode. This should be sufficient to
@ -1890,7 +1739,7 @@ func (b *jobBuilder) runWasmGMTests() {
b.dep(b.buildTaskDrivers()) b.dep(b.buildTaskDrivers())
b.dep(compileTaskName) b.dep(compileTaskName)
b.timeout(60 * time.Minute) b.timeout(60 * time.Minute)
b.cas(CAS_WASM_GM) b.isolate("wasm_gm_tests.isolate")
b.serviceAccount(b.cfg.ServiceAccountUploadGM) b.serviceAccount(b.cfg.ServiceAccountUploadGM)
b.cmd( b.cmd(
"./run_wasm_gm_tests", "./run_wasm_gm_tests",

View File

@ -58,21 +58,21 @@ func (b *jobBuilder) addTask(name string, fn func(*taskBuilder)) {
b.Spec.TaskSpecs = newSpecs b.Spec.TaskSpecs = newSpecs
} }
// uploadCIPDAssetToCAS generates a task to isolate the given CIPD asset. Returns // isolateCIPDAsset generates a task to isolate the given CIPD asset. Returns
// the name of the task. // the name of the task.
func (b *jobBuilder) uploadCIPDAssetToCAS(asset string) string { func (b *jobBuilder) isolateCIPDAsset(asset string) string {
cfg, ok := ISOLATE_ASSET_MAPPING[asset] cfg, ok := ISOLATE_ASSET_MAPPING[asset]
if !ok { if !ok {
log.Fatalf("No isolate task for asset %q", asset) log.Fatalf("No isolate task for asset %q", asset)
} }
b.addTask(cfg.uploadTaskName, func(b *taskBuilder) { b.addTask(cfg.isolateTaskName, func(b *taskBuilder) {
b.cipd(b.MustGetCipdPackageFromAsset(asset)) b.cipd(b.MustGetCipdPackageFromAsset(asset))
b.cmd("/bin/cp", "-rL", cfg.path, "${ISOLATED_OUTDIR}") b.cmd("/bin/cp", "-rL", cfg.path, "${ISOLATED_OUTDIR}")
b.linuxGceDimensions(MACHINE_TYPE_SMALL) b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.idempotent() b.idempotent()
b.cas(CAS_EMPTY) b.isolate("empty.isolate")
}) })
return cfg.uploadTaskName return cfg.isolateTaskName
} }
// genTasksForJob generates the tasks needed by this job. // genTasksForJob generates the tasks needed by this job.
@ -90,8 +90,8 @@ func (b *jobBuilder) genTasksForJob() {
// Isolate CIPD assets. // Isolate CIPD assets.
if b.matchExtraConfig("Isolate") { if b.matchExtraConfig("Isolate") {
for asset, cfg := range ISOLATE_ASSET_MAPPING { for asset, cfg := range ISOLATE_ASSET_MAPPING {
if cfg.uploadTaskName == b.Name { if cfg.isolateTaskName == b.Name {
b.uploadCIPDAssetToCAS(asset) b.isolateCIPDAsset(asset)
return return
} }
} }

View File

@ -84,9 +84,9 @@ func (b *taskBuilder) idempotent() {
b.Spec.Idempotent = true b.Spec.Idempotent = true
} }
// cas sets the CasSpec used by the task. // isolate sets the isolate file used by the task.
func (b *taskBuilder) cas(casSpec string) { func (b *taskBuilder) isolate(i string) {
b.Spec.CasSpec = casSpec b.Spec.Isolate = b.relpath(i)
} }
// env appends the given values to the given environment variable for the task. // env appends the given values to the given environment variable for the task.
@ -166,12 +166,12 @@ func (b *taskBuilder) useIsolatedAssets() bool {
return false return false
} }
// uploadAssetCASCfg represents a task which copies a CIPD package into // isolateAssetConfig represents a task which copies a CIPD package into
// isolate. // isolate.
type uploadAssetCASCfg struct { type isolateAssetCfg struct {
alwaysIsolate bool alwaysIsolate bool
uploadTaskName string isolateTaskName string
path string path string
} }
// asset adds the given assets to the task as CIPD packages. // asset adds the given assets to the task as CIPD packages.
@ -180,7 +180,7 @@ func (b *taskBuilder) asset(assets ...string) {
pkgs := make([]*specs.CipdPackage, 0, len(assets)) pkgs := make([]*specs.CipdPackage, 0, len(assets))
for _, asset := range assets { for _, asset := range assets {
if cfg, ok := ISOLATE_ASSET_MAPPING[asset]; ok && (cfg.alwaysIsolate || shouldIsolate) { if cfg, ok := ISOLATE_ASSET_MAPPING[asset]; ok && (cfg.alwaysIsolate || shouldIsolate) {
b.dep(b.uploadCIPDAssetToCAS(asset)) b.dep(b.isolateCIPDAsset(asset))
} else { } else {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset(asset)) pkgs = append(pkgs, b.MustGetCipdPackageFromAsset(asset))
} }

View File

@ -0,0 +1,13 @@
{
'includes': [
'infrabots.isolate',
'recipes.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../go.mod',
'../../go.sum',
],
},
}

View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'./',
],
},
}

View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'../../platform_tools/ios/bin/',
],
},
}

View File

@ -0,0 +1,13 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../../lottie',
'../../tools/lottie-web-perf',
'../../tools/lottiecap',
'../lottiecap',
],
},
}

View File

@ -0,0 +1,10 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/lottie-web-perf',
],
},
}

View File

@ -0,0 +1,11 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../modules/pathkit',
'../pathkit',
],
},
}

View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'../../tools/perf-canvaskit-puppeteer/',
],
},
}

View File

@ -0,0 +1,13 @@
{
'includes': [
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
],
},
}

View File

@ -0,0 +1,13 @@
{
'variables': {
'files': [
'../../../.gclient',
'../config/recipes.cfg',
'bundle_recipes.sh',
'README.recipes.md',
'recipe_modules/',
'recipes/',
'recipes.py',
],
},
}

View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'../../resources/',
],
},
}

View File

@ -0,0 +1,8 @@
{
'variables': {
'files': [
'../../../.gclient',
'run_recipe.py',
],
},
}

View File

@ -0,0 +1,10 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/skottie-wasm-perf',
],
},
}

View File

@ -0,0 +1,12 @@
{
'includes': [
'assets.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
'../../tools/skpbench/',
],
},
}

10
infra/bots/skqp.isolate Normal file
View File

@ -0,0 +1,10 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../skqp',
],
},
}

View File

@ -0,0 +1,8 @@
{
'variables': {
'files': [
'../../../.gclient',
'run_recipe.py',
],
},
}

View File

@ -0,0 +1,11 @@
{
'variables': {
'files': [
'../../../.gclient',
'../../go.mod',
'../../go.sum',
'./build_task_drivers.sh',
'./task_drivers/',
],
},
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,13 @@
{
'includes': [
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
],
},
}

View File

@ -0,0 +1,8 @@
{
'variables': {
'files': [
'../../resources',
'../../tools/run-wasm-gm-tests',
],
},
}

View File

@ -0,0 +1,8 @@
{
'variables': {
'files': [
'../..',
'../../../.gclient',
],
},
}