AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Reland "[infra] Switch from isolate to RBE-CAS"

Browse Source 
Bug: skia:10883
Change-Id: Ie702a4699cda48603ceff690d2ba946bf7a0d0b2
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/344898
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Weston Tracey <westont@google.com>
This commit is contained in:
Eric Boren 2021-02-17 14:55:20 -05:00
parent d6d7957648
commit 1f95e6c6b4
31 changed files with 1722 additions and 1652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
PRESUBMIT.py
View File

@ -219,27 +219,6 @@ def _CheckIncludesFormatted(input_api, output_api):
return [output_api.PresubmitError('`%s` failed' % ' '.join(cmd))]
return []
def _CheckCompileIsolate(input_api, output_api):
"""Ensure that gen_compile_isolate.py does not change compile.isolate."""
# Only run the check if files were added or removed.
results = []
script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
isolate = os.path.join('infra', 'bots', 'compile.isolated')
for f in input_api.AffectedFiles():
if f.Action() in ('A', 'D', 'R'):
break
if f.LocalPath() in (script, isolate):
break
else:
return results
cmd = ['python', script, 'test']
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
results.append(output_api.PresubmitError(e.output))
return results
class _WarningsAsErrors():
def __init__(self, output_api):
@ -294,7 +273,6 @@ def _CommonChecks(input_api, output_api):
results.extend(_IfDefChecks(input_api, output_api))
results.extend(_CopyrightChecks(input_api, output_api,
source_file_filter=sources))
results.extend(_CheckCompileIsolate(input_api, output_api))
results.extend(_CheckDEPSValid(input_api, output_api))
results.extend(_CheckIncludesFormatted(input_api, output_api))
results.extend(_CheckGNFormatted(input_api, output_api))

7
infra/bots/assets.isolate
View File

@ -1,7 +0,0 @@
{
'variables': {
'files': [
'assets/',
],
},
}

13
infra/bots/canvaskit.isolate
View File

@ -1,13 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../modules/canvaskit',
'../../modules/pathkit/perf/perfReporter.js',
'../../modules/pathkit/tests/testReporter.js',
'../canvaskit',
],
},
}

49
infra/bots/compile.isolate
View File

@ -1,49 +0,0 @@
{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
'../../../.gclient',
'../../.clang-format',
'../../.clang-tidy',
'../../.gn',
'../../BUILD.gn',
'../../bench',
'../../bin/fetch-clang-format',
'../../bin/fetch-gn',
'../../build/fuchsia',
'../../build_overrides',
'../../buildtools',
'../../client_utils/android',
'../../dm',
'../../docs/examples',
'../../example',
'../../experimental',
'../../fuzz',
'../../gm',
'../../gn',
'../../include',
'../../modules',
'../../platform_tools/android/apps/skottie/skottielib/src/main/cpp/native-lib.cpp',
'../../platform_tools/android/launcher/skia_launcher.cpp',
'../../platform_tools/android/vulkan/Skia_Vulkan_Android.h',
'../../platform_tools/ios/app/LaunchScreen.storyboard',
'../../platform_tools/libraries/include/arcore_c_api.h',
'../../resources',
'../../samplecode',
'../../src',
'../../tests',
'../../third_party',
'../../tools',
'../canvaskit',
'../pathkit',
'assets/android_ndk_darwin/VERSION',
'assets/android_ndk_linux/VERSION',
'assets/android_ndk_windows/VERSION',
'assets/cast_toolchain/VERSION',
'assets/clang_linux/VERSION',
'assets/clang_win/VERSION',
],
},
}

11
infra/bots/compile_android_framework.isolate
View File

@ -1,11 +0,0 @@
{
'includes': [
'infrabots.isolate',
'run_recipe.isolate',
],
'variables': {
'files': [
'../../../.gclient',
],
},
}

1
infra/bots/empty.isolate
View File

@ -1 +0,0 @@
{}

234
infra/bots/gen_compile_isolate.py
View File

@ -1,234 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2019 Google LLC
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import difflib
import os
import re
import subprocess
import sys
# Any files in Git which match these patterns will be included, either directly
# or indirectly via a parent dir.
PATH_PATTERNS = [
r'.*\.c$',
r'.*\.cc$',
r'.*\.cpp$',
r'.*\.gn$',
r'.*\.gni$',
r'.*\.h$',
r'.*\.mm$',
r'.*\.storyboard$',
]
# These paths are always added to the inclusion list. Note that they may not
# appear in the isolate if they are included indirectly via a parent dir.
EXPLICIT_PATHS = [
'../.gclient',
'.clang-format',
'.clang-tidy',
'bin/fetch-clang-format',
'bin/fetch-gn',
'buildtools',
'infra/bots/assets/android_ndk_darwin/VERSION',
'infra/bots/assets/android_ndk_linux/VERSION',
'infra/bots/assets/android_ndk_windows/VERSION',
'infra/bots/assets/cast_toolchain/VERSION',
'infra/bots/assets/clang_linux/VERSION',
'infra/bots/assets/clang_win/VERSION',
'infra/canvaskit',
'infra/pathkit',
'resources',
'third_party/externals',
]
# If a parent path contains more than this many immediate child paths (ie. files
# and dirs which are directly inside it as opposed to indirect descendants), we
# will include the parent in the isolate file instead of the children. This
# results in a simpler isolate file which should need to be changed less often.
COMBINE_PATHS_THRESHOLD = 3
# Template for the isolate file content.
ISOLATE_TMPL = '''{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
%s
],
},
}
'''
# Absolute path to the infra/bots dir.
INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
# Absolute path to the compile.isolate file.
ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
def all_paths():
"""Return all paths which are checked in to git."""
repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
return output.splitlines()
def get_relevant_paths():
"""Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
paths = []
for f in all_paths():
for regexp in PATH_PATTERNS:
if re.match(regexp, f):
paths.append(f)
break
paths.extend(EXPLICIT_PATHS)
return paths
class Tree(object):
"""Tree helps with deduplicating and collapsing paths."""
class Node(object):
"""Node represents an individual node in a Tree."""
def __init__(self, name):
self._children = {}
self._name = name
self._is_leaf = False
@property
def is_root(self):
"""Return True iff this is the root node."""
return self._name is None
def add(self, entry):
"""Add the given entry (given as a list of strings) to the Node."""
# Remove the first element if we're not the root node.
if not self.is_root:
if entry[0] != self._name:
raise ValueError('Cannot add a non-matching entry to a Node!')
entry = entry[1:]
# If the entry is now empty, this node is a leaf.
if not entry:
self._is_leaf = True
return
# Add a child node.
if not self._is_leaf:
child = self._children.get(entry[0])
if not child:
child = Tree.Node(entry[0])
self._children[entry[0]] = child
child.add(entry)
# If we have more than COMBINE_PATHS_THRESHOLD immediate children,
# combine them into this node.
immediate_children = 0
for child in self._children.itervalues():
if child._is_leaf:
immediate_children += 1
if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
self._is_leaf = True
self._children = {}
def entries(self):
"""Return the entries represented by this node and its children.
Will not return children in the following cases:
- This Node is a leaf, ie. it represents an entry which was explicitly
inserted into the Tree, as opposed to only part of a path to other
entries.
- This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
thus has been upgraded to a leaf node.
"""
if self._is_leaf:
return [self._name]
rv = []
for child in self._children.itervalues():
for entry in child.entries():
if not self.is_root:
entry = self._name + '/' + entry
rv.append(entry)
return rv
def __init__(self):
self._root = Tree.Node(None)
def add(self, entry):
"""Add the given entry to the tree."""
split = entry.split('/')
if split[-1] == '':
split = split[:-1]
self._root.add(split)
def entries(self):
"""Return the list of entries in the tree.
Entries will be de-duplicated as follows:
- Any entry which is a sub-path of another entry will not be returned.
- Any entry which was not explicitly inserted but has children exceeding
the COMBINE_PATHS_THRESHOLD will be returned while its children will not
be returned.
"""
return self._root.entries()
def relpath(repo_path):
"""Return a relative path to the given path within the repo.
The path is relative to the infra/bots dir, where the compile.isolate file
lives.
"""
repo_path = '../../' + repo_path
repo_path = repo_path.replace('../../infra/', '../')
repo_path = repo_path.replace('../bots/', '')
return repo_path
def get_isolate_content(paths):
"""Construct the new content of the isolate file based on the given paths."""
lines = [' \'%s\',' % relpath(p) for p in paths]
lines.sort()
return ISOLATE_TMPL % '\n'.join(lines)
def main():
"""Regenerate the compile.isolate file, or verify that it hasn't changed."""
testing = False
if len(sys.argv) == 2 and sys.argv[1] == 'test':
testing = True
elif len(sys.argv) != 1:
print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
sys.exit(1)
tree = Tree()
for p in get_relevant_paths():
tree.add(p)
content = get_isolate_content(tree.entries())
if testing:
with open(ISOLATE_FILE, 'rb') as f:
expect_content = f.read()
if content != expect_content:
print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
a = expect_content.splitlines()
b = content.splitlines()
diff = difflib.context_diff(a, b, lineterm='')
for line in diff:
sys.stderr.write(line + '\n')
print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
sys.exit(1)
else:
with open(ISOLATE_FILE, 'wb') as f:
f.write(content)
if __name__ == '__main__':
main()

224
infra/bots/gen_tasks_logic/compile_cas.go Normal file
View File

@ -0,0 +1,224 @@
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package gen_tasks_logic
import (
"log"
"os/exec"
"regexp"
"sort"
"strings"
"go.skia.org/infra/go/cas/rbe"
"go.skia.org/infra/task_scheduler/go/specs"
)
const (
// If a parent path contains more than this many immediate child paths (ie.
// files and dirs which are directly inside it as opposed to indirect
// descendants), we will include the parent in the isolate file instead of
// the children. This results in a simpler CasSpec which should need to be
// changed less often.
combinePathsThreshold = 3
)
var (
// Any files in Git which match these patterns will be included, either
// directly or indirectly via a parent dir.
pathRegexes = []*regexp.Regexp{
regexp.MustCompile(`.*\.c$`),
regexp.MustCompile(`.*\.cc$`),
regexp.MustCompile(`.*\.cpp$`),
regexp.MustCompile(`.*\.gn$`),
regexp.MustCompile(`.*\.gni$`),
regexp.MustCompile(`.*\.h$`),
regexp.MustCompile(`.*\.mm$`),
regexp.MustCompile(`.*\.storyboard$`),
}
// These paths are always added to the inclusion list. Note that they may
// not appear in the CasSpec if they are included indirectly via a parent
// dir.
explicitPaths = []string{
".clang-format",
".clang-tidy",
"bin/fetch-clang-format",
"bin/fetch-gn",
"buildtools",
"infra/bots/assets/android_ndk_darwin/VERSION",
"infra/bots/assets/android_ndk_linux/VERSION",
"infra/bots/assets/android_ndk_windows/VERSION",
"infra/bots/assets/cast_toolchain/VERSION",
"infra/bots/assets/clang_linux/VERSION",
"infra/bots/assets/clang_win/VERSION",
"infra/bots/run_recipe.py",
"infra/canvaskit",
"infra/pathkit",
"resources",
"third_party/externals",
}
)
// getAllCheckedInPaths returns every path checked in to the repo.
func getAllCheckedInPaths() []string {
cmd := exec.Command("git", "ls-files")
cmd.Dir = CheckoutRoot()
output, err := cmd.CombinedOutput()
if err != nil {
log.Fatal(err)
}
split := strings.Split(string(output), "\n")
rv := make([]string, 0, len(split))
for _, line := range split {
if line != "" {
rv = append(rv, line)
}
}
return rv
}
// getRelevantPaths returns all paths needed by compile tasks.
func getRelevantPaths() []string {
rv := []string{}
for _, path := range getAllCheckedInPaths() {
for _, regex := range pathRegexes {
if regex.MatchString(path) {
rv = append(rv, path)
break
}
}
}
return append(rv, explicitPaths...)
}
// node is a single node in a directory tree of task inputs.
type node struct {
children map[string]*node
name string
isLeaf bool
}
// newNode returns a node instance.
func newNode(name string) *node {
return &node{
children: map[string]*node{},
name: name,
isLeaf: false,
}
}
// isRoot returns true iff this is the root node.
func (n *node) isRoot() bool {
return n.name == ""
}
// add the given entry (given as a slice of path components) to the node.
func (n *node) add(entry []string) {
// Remove the first element if we're not the root node.
if !n.isRoot() {
if entry[0] != n.name {
log.Fatalf("Failed to compute compile CAS inputs; attempting to add entry %v to node %q", entry, n.name)
}
entry = entry[1:]
// If the entry is now empty, this node is a leaf.
if len(entry) == 0 {
n.isLeaf = true
return
}
}
// Add a child node.
if !n.isLeaf {
name := entry[0]
child, ok := n.children[name]
if !ok {
child = newNode(name)
n.children[name] = child
}
child.add(entry)
// If we have more than combinePathsThreshold immediate children,
// combine them into this node.
immediateChilden := 0
for _, child := range n.children {
if child.isLeaf {
immediateChilden++
}
if !n.isRoot() && immediateChilden >= combinePathsThreshold {
n.isLeaf = true
n.children = map[string]*node{}
}
}
}
}
// entries returns the entries represented by this node and its children.
// Will not return children in the following cases:
// - This Node is a leaf, ie. it represents an entry which was explicitly
// inserted into the Tree, as opposed to only part of a path to other
// entries.
// - This Node has immediate children exceeding combinePathsThreshold and
// thus has been upgraded to a leaf node.
func (n *node) entries() [][]string {
if n.isLeaf {
return [][]string{{n.name}}
}
rv := [][]string{}
for _, child := range n.children {
for _, entry := range child.entries() {
if !n.isRoot() {
entry = append([]string{n.name}, entry...)
}
rv = append(rv, entry)
}
}
return rv
}
// tree represents a directory tree of task inputs.
type tree struct {
root *node
}
// newTree returns a tree instance.
func newTree() *tree {
return &tree{
root: newNode(""),
}
}
// add the given path to the tree. Entries may be combined as defined by
// combinePathsThreshold.
func (t *tree) add(p string) {
split := strings.Split(p, "/")
t.root.add(split)
}
// entries returns all entries in the tree. Entries may be combined as defined
// by combinePathsThreshold.
func (t *tree) entries() []string {
entries := t.root.entries()
rv := make([]string, 0, len(entries))
for _, entry := range entries {
rv = append(rv, strings.Join(append([]string{"skia"}, entry...), "/"))
}
sort.Strings(rv)
return rv
}
// generateCompileCAS creates the CasSpec used for tasks which build Skia.
func generateCompileCAS(b *specs.TasksCfgBuilder) {
t := newTree()
for _, path := range getRelevantPaths() {
t.add(path)
}
spec := &specs.CasSpec{
Root: "..",
Paths: t.entries(),
Excludes: []string{rbe.ExcludeGitDir},
}
b.MustAddCasSpec(CAS_COMPILE, spec)
}

313
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -22,11 +22,30 @@ import (
"strings"
"time"
"go.skia.org/infra/go/cas/rbe"
"go.skia.org/infra/go/cipd"
"go.skia.org/infra/task_scheduler/go/specs"
)
const (
CAS_CANVASKIT = "canvaskit"
CAS_COMPILE = "compile"
CAS_EMPTY = "empty" // TODO(borenet): It'd be nice if this wasn't necessary.
CAS_LOTTIE_CI = "lottie-ci"
CAS_LOTTIE_WEB = "lottie-web"
CAS_PATHKIT = "pathkit"
CAS_PERF = "perf"
CAS_PUPPETEER = "puppeteer"
CAS_RUN_RECIPE = "run-recipe"
CAS_RECIPES = "recipes"
CAS_SKOTTIE_WASM = "skottie-wasm"
CAS_SKPBENCH = "skpbench"
CAS_SKQP = "skqp"
CAS_TASK_DRIVERS = "task-drivers"
CAS_TEST = "test"
CAS_WASM_GM = "wasm-gm"
CAS_WHOLE_REPO = "whole-repo"
BUILD_TASK_DRIVERS_PREFIX = "Housekeeper-PerCommit-BuildTaskDrivers"
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
ISOLATE_GCLOUD_LINUX_NAME = "Housekeeper-PerCommit-IsolateGCloudLinux"
@ -115,6 +134,15 @@ var (
},
}
// CAS_SPEC_WHOLE_REPO is a CasSpec which includes the entire repo. This is
// global so that it can be overridden by other repositories which import
// this file.
CAS_SPEC_WHOLE_REPO = &specs.CasSpec{
Root: "..",
Paths: []string{"skia"},
Excludes: []string{rbe.ExcludeGitDir},
}
// TODO(borenet): This hacky and bad.
CIPD_PKG_LUCI_AUTH = cipd.MustGetPackage("infra/tools/luci-auth/${platform}")
@ -149,39 +177,39 @@ var (
// ISOLATE_ASSET_MAPPING maps the name of an asset to the configuration
// for how the CIPD package should be installed for a given task.
ISOLATE_ASSET_MAPPING = map[string]isolateAssetCfg{
ISOLATE_ASSET_MAPPING = map[string]uploadAssetCASCfg{
"gcloud_linux": {
isolateTaskName: ISOLATE_GCLOUD_LINUX_NAME,
path: "gcloud_linux",
uploadTaskName: ISOLATE_GCLOUD_LINUX_NAME,
path: "gcloud_linux",
},
"skimage": {
isolateTaskName: ISOLATE_SKIMAGE_NAME,
path: "skimage",
uploadTaskName: ISOLATE_SKIMAGE_NAME,
path: "skimage",
},
"skp": {
isolateTaskName: ISOLATE_SKP_NAME,
path: "skp",
uploadTaskName: ISOLATE_SKP_NAME,
path: "skp",
},
"svg": {
isolateTaskName: ISOLATE_SVG_NAME,
path: "svg",
uploadTaskName: ISOLATE_SVG_NAME,
path: "svg",
},
"mskp": {
isolateTaskName: ISOLATE_MSKP_NAME,
path: "mskp",
uploadTaskName: ISOLATE_MSKP_NAME,
path: "mskp",
},
"android_ndk_linux": {
isolateTaskName: ISOLATE_NDK_LINUX_NAME,
path: "android_ndk_linux",
uploadTaskName: ISOLATE_NDK_LINUX_NAME,
path: "android_ndk_linux",
},
"android_sdk_linux": {
isolateTaskName: ISOLATE_SDK_LINUX_NAME,
path: "android_sdk_linux",
uploadTaskName: ISOLATE_SDK_LINUX_NAME,
path: "android_sdk_linux",
},
"win_toolchain": {
alwaysIsolate: true,
isolateTaskName: ISOLATE_WIN_TOOLCHAIN_NAME,
path: "win_toolchain",
alwaysIsolate: true,
uploadTaskName: ISOLATE_WIN_TOOLCHAIN_NAME,
path: "win_toolchain",
},
}
)
@ -316,18 +344,153 @@ func GenTasks(cfg *Config) {
// Create Tasks and Jobs.
builder := &builder{
TasksCfgBuilder: b,
cfg: cfg,
jobNameSchema: schema,
jobs: jobs,
relpathBaseDir: relpathBaseDir,
relpathTargetDir: relpathTargetDir,
TasksCfgBuilder: b,
cfg: cfg,
jobNameSchema: schema,
jobs: jobs,
}
for _, name := range jobs {
jb := newJobBuilder(builder, name)
jb.genTasksForJob()
jb.finish()
}
// Create CasSpecs.
b.MustAddCasSpec(CAS_CANVASKIT, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/canvaskit",
"skia/modules/canvaskit",
"skia/modules/pathkit/perf/perfReporter.js",
"skia/modules/pathkit/tests/testReporter.js",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_EMPTY, specs.EmptyCasSpec)
b.MustAddCasSpec(CAS_LOTTIE_CI, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/lottiecap",
"skia/tools/lottie-web-perf",
"skia/tools/lottiecap",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_LOTTIE_WEB, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/tools/lottie-web-perf",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PATHKIT, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/pathkit",
"skia/modules/pathkit",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PERF, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/platform_tools/ios/bin",
"skia/resources",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_PUPPETEER, &specs.CasSpec{
Root: ".",
Paths: []string{
"tools/perf-canvaskit-puppeteer",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_RECIPES, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/config/recipes.cfg",
"skia/infra/bots/bundle_recipes.sh",
"skia/infra/bots/README.recipes.md",
"skia/infra/bots/recipe_modules",
"skia/infra/bots/recipes",
"skia/infra/bots/recipes.py",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_RUN_RECIPE, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKOTTIE_WASM, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/tools/skottie-wasm-perf",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKPBENCH, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/tools/skpbench",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_SKQP, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/run_recipe.py",
"skia/infra/skqp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_TASK_DRIVERS, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/go.mod",
"skia/go.sum",
"skia/infra/bots/build_task_drivers.sh",
"skia/infra/bots/run_recipe.py",
"skia/infra/bots/task_drivers",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_TEST, &specs.CasSpec{
Root: "..",
Paths: []string{
"skia/infra/bots/assets",
"skia/infra/bots/run_recipe.py",
"skia/platform_tools/ios/bin",
"skia/resources",
"skia/tools/valgrind.supp",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_WASM_GM, &specs.CasSpec{
Root: ".",
Paths: []string{
"resources",
"tools/run-wasm-gm-tests",
},
Excludes: []string{rbe.ExcludeGitDir},
})
b.MustAddCasSpec(CAS_WHOLE_REPO, CAS_SPEC_WHOLE_REPO)
generateCompileCAS(b)
builder.MustFinish()
}
@ -356,11 +519,9 @@ func getCallingDirName() string {
// builder is a wrapper for specs.TasksCfgBuilder.
type builder struct {
*specs.TasksCfgBuilder
cfg *Config
jobNameSchema *JobNameSchema
jobs []string
relpathBaseDir string
relpathTargetDir string
cfg *Config
jobNameSchema *JobNameSchema
jobs []string
}
// marshalJson encodes the given data as JSON and fixes escaping of '<' which Go
@ -386,7 +547,7 @@ func (b *taskBuilder) kitchenTaskNoBundle(recipe string, outputDir string) {
python := "cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}"
b.cmd(python, "-u", "skia/infra/bots/run_recipe.py", "${ISOLATED_OUTDIR}", recipe, b.getRecipeProps(), b.cfg.Project)
// Most recipes want this isolate; they can override if necessary.
b.isolate("swarm_recipe.isolate")
b.cas(CAS_RUN_RECIPE)
b.timeout(time.Hour)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
b.Spec.ExtraTags = map[string]string{
@ -774,16 +935,6 @@ func (b *taskBuilder) defaultSwarmDimensions() {
b.dimension(dims...)
}
// relpath returns the relative path to the given file from the config file.
func (b *builder) relpath(f string) string {
target := filepath.Join(b.relpathTargetDir, f)
rv, err := filepath.Rel(b.relpathBaseDir, target)
if err != nil {
log.Fatal(err)
}
return rv
}
// bundleRecipes generates the task to bundle and isolate the recipes. Returns
// the name of the task, which may be added as a dependency.
func (b *jobBuilder) bundleRecipes() string {
@ -794,7 +945,7 @@ func (b *jobBuilder) bundleRecipes() string {
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
b.idempotent()
b.isolate("recipes.isolate")
b.cas(CAS_RECIPES)
})
return BUNDLE_RECIPES_NAME
}
@ -813,7 +964,7 @@ func (b *jobBuilder) buildTaskDrivers(goos, goarch string) string {
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.idempotent()
b.isolate("task_drivers.isolate")
b.cas(CAS_TASK_DRIVERS)
})
return name
}
@ -841,7 +992,7 @@ func (b *jobBuilder) updateGoDeps() {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.serviceAccount(b.cfg.ServiceAccountRecreateSKPs)
})
}
@ -882,7 +1033,7 @@ func (b *jobBuilder) createDockerImage(wasm bool) string {
)
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker()
@ -915,7 +1066,7 @@ func (b *jobBuilder) createPushAppsFromSkiaDockerImage() {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.dep(b.createDockerImage(false))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker()
@ -947,7 +1098,7 @@ func (b *jobBuilder) createPushAppsFromWASMDockerImage() {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.dep(b.createDockerImage(true))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker()
@ -994,10 +1145,10 @@ func (b *jobBuilder) compile() string {
} else {
b.addTask(name, func(b *taskBuilder) {
recipe := "compile"
isolate := "compile.isolate"
casSpec := CAS_COMPILE
if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter", "SKQP") {
recipe = "sync_and_compile"
isolate = "swarm_recipe.isolate"
casSpec = CAS_RUN_RECIPE
b.recipeProps(EXTRA_PROPS)
b.usesGit()
if !b.extraConfig("NoDEPS") {
@ -1007,7 +1158,7 @@ func (b *jobBuilder) compile() string {
b.idempotent()
}
b.kitchenTask(recipe, OUTPUT_BUILD)
b.isolate(isolate)
b.cas(casSpec)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.swarmDimensions()
if b.extraConfig("Docker", "LottieWeb", "SKQP", "CMake") || b.compiler("EMCC") {
@ -1116,7 +1267,7 @@ func (b *jobBuilder) checkGeneratedFiles() {
// checkGnToBp verifies that the gn_to_bp.py script continues to work.
func (b *jobBuilder) checkGnToBp() {
b.addTask(b.Name, func(b *taskBuilder) {
b.isolate("compile.isolate")
b.cas(CAS_COMPILE)
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.cmd("./run_gn_to_bp",
"--local=false",
@ -1147,7 +1298,7 @@ func (b *jobBuilder) housekeeper() {
// should add as a dependency.
func (b *jobBuilder) g3FrameworkCanary() {
b.addTask(b.Name, func(b *taskBuilder) {
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.cmd("./g3_canary",
"--local=false",
@ -1185,13 +1336,10 @@ func (b *jobBuilder) infra() {
}
b.recipeProp("repository", specs.PLACEHOLDER_REPO)
b.kitchenTask("infra", OUTPUT_NONE)
b.isolate("infra_tests.isolate")
b.cas(CAS_WHOLE_REPO)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cipd(specs.CIPD_PKGS_GSUTIL...)
b.idempotent()
// Repos which call into Skia's gen_tasks.go should define their own
// infra_tests.isolate and therefore should not use relpath().
b.Spec.Isolate = "infra_tests.isolate"
b.usesGo()
})
}
@ -1290,30 +1438,33 @@ func (b *jobBuilder) dm() {
compileTaskName = b.compile()
}
b.addTask(b.Name, func(b *taskBuilder) {
isolate := "test_skia_bundled.isolate"
cas := CAS_TEST
recipe := "test"
if b.extraConfig("SKQP") {
isolate = "skqp.isolate"
cas = CAS_SKQP
recipe = "skqp_test"
if b.cpu("Emulator") {
recipe = "test_skqp_emulator"
}
} else if b.extraConfig("OpenCL") {
// TODO(dogben): Longer term we may not want this to be called a "Test" task, but until we start
// running hs_bench or kx, it will be easier to fit into the current job name schema.
// TODO(dogben): Longer term we may not want this to be called a
// "Test" task, but until we start running hs_bench or kx, it will
// be easier to fit into the current job name schema.
recipe = "compute_test"
} else if b.extraConfig("PathKit") {
isolate = "pathkit.isolate"
cas = CAS_PATHKIT
recipe = "test_pathkit"
} else if b.extraConfig("CanvasKit") {
isolate = "canvaskit.isolate"
cas = CAS_CANVASKIT
recipe = "test_canvaskit"
} else if b.extraConfig("LottieWeb") {
// lottie_ci.isolate differs from lottie_web.isolate in that it includes more of the files,
// especially those brought in via DEPS in the lottie-ci repo. The main difference between
// Perf.+LottieWeb and Test.+LottieWeb is that the former pulls in the lottie build via
// npm and the latter always tests at lottie's ToT.
isolate = "lottie_ci.isolate"
// CAS_LOTTIE_CI differs from CAS_LOTTIE_WEB in that it includes
// more of the files, especially those brought in via DEPS in the
// lottie-ci repo. The main difference between Perf.+LottieWeb and
// Test.+LottieWeb is that the former pulls in the lottie build via
// npm and the latter always tests at lottie's
// ToT.
cas = CAS_LOTTIE_CI
recipe = "test_lottie_web"
}
b.recipeProp("gold_hashes_url", b.cfg.GoldHashesURL)
@ -1327,7 +1478,7 @@ func (b *jobBuilder) dm() {
b.dmFlags(iidStr)
}
b.kitchenTask(recipe, OUTPUT_TEST)
b.isolate(isolate)
b.cas(cas)
b.swarmDimensions()
if b.extraConfig("CanvasKit", "Docker", "LottieWeb", "PathKit", "SKQP") {
b.usesDocker()
@ -1399,7 +1550,7 @@ func (b *jobBuilder) fm() {
b.addTask(b.Name, func(b *taskBuilder) {
b.asset("skimage", "skp", "svg")
b.isolate("test_skia_bundled.isolate")
b.cas(CAS_TEST)
b.dep(b.buildTaskDrivers(goos, "amd64"), b.compile())
b.cmd("./fm_driver${EXECUTABLE_SUFFIX}",
"--local=false",
@ -1442,7 +1593,7 @@ func (b *jobBuilder) fm() {
// Canary-G3 does not use this path because it is very different from other autorollers.
func (b *jobBuilder) canary(rollerName string) {
b.addTask(b.Name, func(b *taskBuilder) {
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.cmd("./canary",
"--local=false",
@ -1475,7 +1626,7 @@ func (b *jobBuilder) puppeteer() {
b.dep(b.buildTaskDrivers("linux", "amd64"), compileTaskName)
b.output(OUTPUT_PERF)
b.timeout(60 * time.Minute)
b.isolate("perf_puppeteer.isolate")
b.cas(CAS_PUPPETEER)
b.serviceAccount(b.cfg.ServiceAccountCompile)
webglversion := "2"
@ -1586,7 +1737,7 @@ func (b *jobBuilder) cifuzz() {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.output("cifuzz_out")
b.timeout(60 * time.Minute)
b.isolate("whole_repo.isolate")
b.cas(CAS_WHOLE_REPO)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cmd(
"./cifuzz",
@ -1612,31 +1763,31 @@ func (b *jobBuilder) perf() {
doUpload := b.release() && b.doUpload()
b.addTask(b.Name, func(b *taskBuilder) {
recipe := "perf"
isolate := "perf_skia_bundled.isolate"
cas := CAS_PERF
if b.extraConfig("Skpbench") {
recipe = "skpbench"
isolate = "skpbench_skia_bundled.isolate"
cas = CAS_SKPBENCH
} else if b.extraConfig("PathKit") {
isolate = "pathkit.isolate"
cas = CAS_PATHKIT
recipe = "perf_pathkit"
} else if b.extraConfig("CanvasKit") {
isolate = "canvaskit.isolate"
cas = CAS_CANVASKIT
recipe = "perf_canvaskit"
} else if b.extraConfig("SkottieTracing") {
recipe = "perf_skottietrace"
} else if b.extraConfig("SkottieWASM") {
recipe = "perf_skottiewasm_lottieweb"
isolate = "skottie_wasm.isolate"
cas = CAS_SKOTTIE_WASM
} else if b.extraConfig("LottieWeb") {
recipe = "perf_skottiewasm_lottieweb"
isolate = "lottie_web.isolate"
cas = CAS_LOTTIE_WEB
}
b.recipeProps(EXTRA_PROPS)
if recipe == "perf" {
b.nanobenchFlags(doUpload)
}
b.kitchenTask(recipe, OUTPUT_PERF)
b.isolate(isolate)
b.cas(cas)
b.swarmDimensions()
if b.extraConfig("CanvasKit", "Docker", "PathKit") {
b.usesDocker()
@ -1711,7 +1862,7 @@ func (b *jobBuilder) presubmit() {
})
b.recipeProps(EXTRA_PROPS)
b.kitchenTaskNoBundle("run_presubmit", OUTPUT_NONE)
b.isolate("run_recipe.isolate")
b.cas(CAS_RUN_RECIPE)
b.serviceAccount(b.cfg.ServiceAccountCompile)
// Use MACHINE_TYPE_LARGE because it seems to save time versus
// MEDIUM and we want presubmit to be fast.
@ -1737,7 +1888,7 @@ func (b *jobBuilder) compileWasmGMTests(compileName string) {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.output("wasm_out")
b.timeout(60 * time.Minute)
b.isolate("compile.isolate")
b.cas(CAS_COMPILE)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.cache(CACHES_DOCKER...)
// For now, we only have one compile mode - a GPU release mode. This should be sufficient to
@ -1772,7 +1923,7 @@ func (b *jobBuilder) runWasmGMTests() {
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.dep(compileTaskName)
b.timeout(60 * time.Minute)
b.isolate("wasm_gm_tests.isolate")
b.cas(CAS_WASM_GM)
b.serviceAccount(b.cfg.ServiceAccountUploadGM)
b.cmd(
"./run_wasm_gm_tests",

14
infra/bots/gen_tasks_logic/job_builder.go
View File

@ -59,21 +59,21 @@ func (b *jobBuilder) addTask(name string, fn func(*taskBuilder)) {
b.Spec.TaskSpecs = newSpecs
}
// isolateCIPDAsset generates a task to isolate the given CIPD asset. Returns
// uploadCIPDAssetToCAS generates a task to isolate the given CIPD asset. Returns
// the name of the task.
func (b *jobBuilder) isolateCIPDAsset(asset string) string {
func (b *jobBuilder) uploadCIPDAssetToCAS(asset string) string {
cfg, ok := ISOLATE_ASSET_MAPPING[asset]
if !ok {
log.Fatalf("No isolate task for asset %q", asset)
}
b.addTask(cfg.isolateTaskName, func(b *taskBuilder) {
b.addTask(cfg.uploadTaskName, func(b *taskBuilder) {
b.cipd(b.MustGetCipdPackageFromAsset(asset))
b.cmd("/bin/cp", "-rL", cfg.path, "${ISOLATED_OUTDIR}")
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
b.idempotent()
b.isolate("empty.isolate")
b.cas(CAS_EMPTY)
})
return cfg.isolateTaskName
return cfg.uploadTaskName
}
// genTasksForJob generates the tasks needed by this job.
@ -92,8 +92,8 @@ func (b *jobBuilder) genTasksForJob() {
// Isolate CIPD assets.
if b.matchExtraConfig("Isolate") {
for asset, cfg := range ISOLATE_ASSET_MAPPING {
if cfg.isolateTaskName == b.Name {
b.isolateCIPDAsset(asset)
if cfg.uploadTaskName == b.Name {
b.uploadCIPDAssetToCAS(asset)
return
}
}

18
infra/bots/gen_tasks_logic/task_builder.go
View File

@ -84,9 +84,9 @@ func (b *taskBuilder) idempotent() {
b.Spec.Idempotent = true
}
// isolate sets the isolate file used by the task.
func (b *taskBuilder) isolate(i string) {
b.Spec.Isolate = b.relpath(i)
// cas sets the CasSpec used by the task.
func (b *taskBuilder) cas(casSpec string) {
b.Spec.CasSpec = casSpec
}
// env appends the given values to the given environment variable for the task.
@ -166,12 +166,12 @@ func (b *taskBuilder) useIsolatedAssets() bool {
return false
}
// isolateAssetConfig represents a task which copies a CIPD package into
// uploadAssetCASCfg represents a task which copies a CIPD package into
// isolate.
type isolateAssetCfg struct {
alwaysIsolate bool
isolateTaskName string
path string
type uploadAssetCASCfg struct {
alwaysIsolate bool
uploadTaskName string
path string
}
// asset adds the given assets to the task as CIPD packages.
@ -180,7 +180,7 @@ func (b *taskBuilder) asset(assets ...string) {
pkgs := make([]*specs.CipdPackage, 0, len(assets))
for _, asset := range assets {
if cfg, ok := ISOLATE_ASSET_MAPPING[asset]; ok && (cfg.alwaysIsolate || shouldIsolate) {
b.dep(b.isolateCIPDAsset(asset))
b.dep(b.uploadCIPDAssetToCAS(asset))
} else {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset(asset))
}

13
infra/bots/infra_tests.isolate
View File

@ -1,13 +0,0 @@
{
'includes': [
'infrabots.isolate',
'recipes.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../go.mod',
'../../go.sum',
],
},
}

7
infra/bots/infrabots.isolate
View File

@ -1,7 +0,0 @@
{
'variables': {
'files': [
'./',
],
},
}

7
infra/bots/ios_bin.isolate
View File

@ -1,7 +0,0 @@
{
'variables': {
'files': [
'../../platform_tools/ios/bin/',
],
},
}

13
infra/bots/lottie_ci.isolate
View File

@ -1,13 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../../lottie',
'../../tools/lottie-web-perf',
'../../tools/lottiecap',
'../lottiecap',
],
},
}

10
infra/bots/lottie_web.isolate
View File

@ -1,10 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/lottie-web-perf',
],
},
}

11
infra/bots/pathkit.isolate
View File

@ -1,11 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../modules/pathkit',
'../pathkit',
],
},
}

7
infra/bots/perf_puppeteer.isolate
View File

@ -1,7 +0,0 @@
{
'variables': {
'files': [
'../../tools/perf-canvaskit-puppeteer/',
],
},
}

13
infra/bots/perf_skia_bundled.isolate
View File

@ -1,13 +0,0 @@
{
'includes': [
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
],
},
}

13
infra/bots/recipes.isolate
View File

@ -1,13 +0,0 @@
{
'variables': {
'files': [
'../../../.gclient',
'../config/recipes.cfg',
'bundle_recipes.sh',
'README.recipes.md',
'recipe_modules/',
'recipes/',
'recipes.py',
],
},
}

7
infra/bots/resources.isolate
View File

@ -1,7 +0,0 @@
{
'variables': {
'files': [
'../../resources/',
],
},
}

8
infra/bots/run_recipe.isolate
View File

@ -1,8 +0,0 @@
{
'variables': {
'files': [
'../../../.gclient',
'run_recipe.py',
],
},
}

10
infra/bots/skottie_wasm.isolate
View File

@ -1,10 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/skottie-wasm-perf',
],
},
}

12
infra/bots/skpbench_skia_bundled.isolate
View File

@ -1,12 +0,0 @@
{
'includes': [
'assets.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
'../../tools/skpbench/',
],
},
}

10
infra/bots/skqp.isolate
View File

@ -1,10 +0,0 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../skqp',
],
},
}

8
infra/bots/swarm_recipe.isolate
View File

@ -1,8 +0,0 @@
{
'variables': {
'files': [
'../../../.gclient',
'run_recipe.py',
],
},
}

11
infra/bots/task_drivers.isolate
View File

@ -1,11 +0,0 @@
{
'variables': {
'files': [
'../../../.gclient',
'../../go.mod',
'../../go.sum',
'./build_task_drivers.sh',
'./task_drivers/',
],
},
}

2279
infra/bots/tasks.json
View File

File diff suppressed because it is too large Load Diff

13
infra/bots/test_skia_bundled.isolate
View File

@ -1,13 +0,0 @@
{
'includes': [
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../tools/valgrind.supp',
],
},
}

8
infra/bots/wasm_gm_tests.isolate
View File

@ -1,8 +0,0 @@
{
'variables': {
'files': [
'../../resources',
'../../tools/run-wasm-gm-tests',
],
},
}

8
infra/bots/whole_repo.isolate
View File

@ -1,8 +0,0 @@
{
'variables': {
'files': [
'../..',
'../../../.gclient',
],
},
}