141cfdfe98
DMSAA is complete enough now that we can just add gldmsaa configs to the normal test bots. Bug: skia:11396 Change-Id: I4f65777388e2e4ddbe10271dd419a71e8fa7820b Reviewed-on: https://skia-review.googlesource.com/c/skia/+/421376 Commit-Queue: Chris Dalton <csmartdalton@google.com> Reviewed-by: Brian Salomon <bsalomon@google.com>
1959 lines
62 KiB
Go
1959 lines
62 KiB
Go
// Copyright 2016 The Chromium Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
package gen_tasks_logic
|
|
|
|
/*
|
|
Generate the tasks.json file.
|
|
*/
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.skia.org/infra/go/cas/rbe"
|
|
"go.skia.org/infra/go/cipd"
|
|
"go.skia.org/infra/task_scheduler/go/specs"
|
|
)
|
|
|
|
const (
|
|
CAS_CANVASKIT = "canvaskit"
|
|
CAS_COMPILE = "compile"
|
|
CAS_EMPTY = "empty" // TODO(borenet): It'd be nice if this wasn't necessary.
|
|
CAS_LOTTIE_CI = "lottie-ci"
|
|
CAS_LOTTIE_WEB = "lottie-web"
|
|
CAS_PATHKIT = "pathkit"
|
|
CAS_PERF = "perf"
|
|
CAS_PUPPETEER = "puppeteer"
|
|
CAS_RUN_RECIPE = "run-recipe"
|
|
CAS_RECIPES = "recipes"
|
|
CAS_SKOTTIE_WASM = "skottie-wasm"
|
|
CAS_SKPBENCH = "skpbench"
|
|
CAS_TASK_DRIVERS = "task-drivers"
|
|
CAS_TEST = "test"
|
|
CAS_WASM_GM = "wasm-gm"
|
|
CAS_WHOLE_REPO = "whole-repo"
|
|
|
|
BUILD_TASK_DRIVERS_PREFIX = "Housekeeper-PerCommit-BuildTaskDrivers"
|
|
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
|
|
ISOLATE_GCLOUD_LINUX_NAME = "Housekeeper-PerCommit-IsolateGCloudLinux"
|
|
ISOLATE_SKIMAGE_NAME = "Housekeeper-PerCommit-IsolateSkImage"
|
|
ISOLATE_SKP_NAME = "Housekeeper-PerCommit-IsolateSKP"
|
|
ISOLATE_MSKP_NAME = "Housekeeper-PerCommit-IsolateMSKP"
|
|
ISOLATE_SVG_NAME = "Housekeeper-PerCommit-IsolateSVG"
|
|
ISOLATE_NDK_LINUX_NAME = "Housekeeper-PerCommit-IsolateAndroidNDKLinux"
|
|
ISOLATE_SDK_LINUX_NAME = "Housekeeper-PerCommit-IsolateAndroidSDKLinux"
|
|
ISOLATE_WIN_TOOLCHAIN_NAME = "Housekeeper-PerCommit-IsolateWinToolchain"
|
|
|
|
DEFAULT_OS_DEBIAN = "Debian-10.3"
|
|
DEFAULT_OS_LINUX_GCE = "Debian-10.3"
|
|
OLD_OS_LINUX_GCE = "Debian-9.8"
|
|
COMPILE_TASK_NAME_OS_LINUX = "Debian10"
|
|
COMPILE_TASK_NAME_OS_LINUX_OLD = "Debian9"
|
|
DEFAULT_OS_MAC = "Mac-10.15.7"
|
|
DEFAULT_OS_WIN = "Windows-Server-17763"
|
|
|
|
// Small is a 2-core machine.
|
|
// TODO(dogben): Would n1-standard-1 or n1-standard-2 be sufficient?
|
|
MACHINE_TYPE_SMALL = "n1-highmem-2"
|
|
// Medium is a 16-core machine
|
|
MACHINE_TYPE_MEDIUM = "n1-standard-16"
|
|
// Large is a 64-core machine. (We use "highcpu" because we don't need more than 57GB memory for
|
|
// any of our tasks.)
|
|
MACHINE_TYPE_LARGE = "n1-highcpu-64"
|
|
|
|
// Swarming output dirs.
|
|
OUTPUT_NONE = "output_ignored" // This will result in outputs not being isolated.
|
|
OUTPUT_BUILD = "build"
|
|
OUTPUT_TEST = "test"
|
|
OUTPUT_PERF = "perf"
|
|
|
|
// Name prefix for upload jobs.
|
|
PREFIX_UPLOAD = "Upload"
|
|
)
|
|
|
|
var (
|
|
// "Constants"
|
|
|
|
// Named caches used by tasks.
|
|
CACHES_GIT = []*specs.Cache{
|
|
{
|
|
Name: "git",
|
|
Path: "cache/git",
|
|
},
|
|
{
|
|
Name: "git_cache",
|
|
Path: "cache/git_cache",
|
|
},
|
|
}
|
|
CACHES_GO = []*specs.Cache{
|
|
{
|
|
Name: "go_cache",
|
|
Path: "cache/go_cache",
|
|
},
|
|
{
|
|
Name: "gopath",
|
|
Path: "cache/gopath",
|
|
},
|
|
}
|
|
CACHES_WORKDIR = []*specs.Cache{
|
|
{
|
|
Name: "work",
|
|
Path: "cache/work",
|
|
},
|
|
}
|
|
CACHES_CCACHE = []*specs.Cache{
|
|
{
|
|
Name: "ccache",
|
|
Path: "cache/ccache",
|
|
},
|
|
}
|
|
// The "docker" cache is used as a persistent working directory for
|
|
// tasks which use Docker. It is not to be confused with Docker's own
|
|
// cache, which stores images. We do not currently use a named Swarming
|
|
// cache for the latter.
|
|
// TODO(borenet): We should ensure that any task which uses Docker does
|
|
// not also use the normal "work" cache, to prevent issues like
|
|
// https://bugs.chromium.org/p/skia/issues/detail?id=9749.
|
|
CACHES_DOCKER = []*specs.Cache{
|
|
{
|
|
Name: "docker",
|
|
Path: "cache/docker",
|
|
},
|
|
}
|
|
|
|
// CAS_SPEC_LOTTIE_CI is a CasSpec which includes the files needed for
|
|
// lottie-ci. This is global so that it can be overridden by other
|
|
// repositories which import this file.
|
|
CAS_SPEC_LOTTIE_CI = &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/infra/lottiecap",
|
|
"skia/tools/lottie-web-perf",
|
|
"skia/tools/lottiecap",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
}
|
|
|
|
// CAS_SPEC_WHOLE_REPO is a CasSpec which includes the entire repo. This is
|
|
// global so that it can be overridden by other repositories which import
|
|
// this file.
|
|
CAS_SPEC_WHOLE_REPO = &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{"skia"},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
}
|
|
|
|
// TODO(borenet): This hacky and bad.
|
|
CIPD_PKG_LUCI_AUTH = cipd.MustGetPackage("infra/tools/luci-auth/${platform}")
|
|
|
|
CIPD_PKGS_GOLDCTL = []*specs.CipdPackage{cipd.MustGetPackage("skia/tools/goldctl/${platform}")}
|
|
|
|
CIPD_PKGS_XCODE = []*specs.CipdPackage{
|
|
// https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#317
|
|
// This package is really just an installer for XCode.
|
|
{
|
|
Name: "infra/tools/mac_toolchain/${platform}",
|
|
Path: "mac_toolchain",
|
|
// When this is updated, also update
|
|
// https://skia.googlesource.com/skcms.git/+/f1e2b45d18facbae2dece3aca673fe1603077846/infra/bots/gen_tasks.go#56
|
|
Version: "git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a",
|
|
},
|
|
}
|
|
|
|
// These properties are required by some tasks, eg. for running
|
|
// bot_update, but they prevent de-duplication, so they should only be
|
|
// used where necessary.
|
|
EXTRA_PROPS = map[string]string{
|
|
"buildbucket_build_id": specs.PLACEHOLDER_BUILDBUCKET_BUILD_ID,
|
|
"patch_issue": specs.PLACEHOLDER_ISSUE_INT,
|
|
"patch_ref": specs.PLACEHOLDER_PATCH_REF,
|
|
"patch_repo": specs.PLACEHOLDER_PATCH_REPO,
|
|
"patch_set": specs.PLACEHOLDER_PATCHSET_INT,
|
|
"patch_storage": specs.PLACEHOLDER_PATCH_STORAGE,
|
|
"repository": specs.PLACEHOLDER_REPO,
|
|
"revision": specs.PLACEHOLDER_REVISION,
|
|
"task_id": specs.PLACEHOLDER_TASK_ID,
|
|
}
|
|
|
|
// ISOLATE_ASSET_MAPPING maps the name of an asset to the configuration
|
|
// for how the CIPD package should be installed for a given task.
|
|
ISOLATE_ASSET_MAPPING = map[string]uploadAssetCASCfg{
|
|
"gcloud_linux": {
|
|
uploadTaskName: ISOLATE_GCLOUD_LINUX_NAME,
|
|
path: "gcloud_linux",
|
|
},
|
|
"skimage": {
|
|
uploadTaskName: ISOLATE_SKIMAGE_NAME,
|
|
path: "skimage",
|
|
},
|
|
"skp": {
|
|
uploadTaskName: ISOLATE_SKP_NAME,
|
|
path: "skp",
|
|
},
|
|
"svg": {
|
|
uploadTaskName: ISOLATE_SVG_NAME,
|
|
path: "svg",
|
|
},
|
|
"mskp": {
|
|
uploadTaskName: ISOLATE_MSKP_NAME,
|
|
path: "mskp",
|
|
},
|
|
"android_ndk_linux": {
|
|
uploadTaskName: ISOLATE_NDK_LINUX_NAME,
|
|
path: "android_ndk_linux",
|
|
},
|
|
"android_sdk_linux": {
|
|
uploadTaskName: ISOLATE_SDK_LINUX_NAME,
|
|
path: "android_sdk_linux",
|
|
},
|
|
"win_toolchain": {
|
|
alwaysIsolate: true,
|
|
uploadTaskName: ISOLATE_WIN_TOOLCHAIN_NAME,
|
|
path: "win_toolchain",
|
|
},
|
|
}
|
|
|
|
// Set dontReduceOpsTaskSplitting option on these models
|
|
DONT_REDUCE_OPS_TASK_SPLITTING_MODELS = []string{
|
|
"NUC5PPYH",
|
|
}
|
|
)
|
|
|
|
// Config contains general configuration information.
|
|
type Config struct {
|
|
// Directory containing assets. Assumed to be relative to the directory
|
|
// which contains the calling gen_tasks.go file. If not specified, uses
|
|
// the infra/bots/assets from this repo.
|
|
AssetsDir string `json:"assets_dir"`
|
|
|
|
// Path to the builder name schema JSON file. Assumed to be relative to
|
|
// the directory which contains the calling gen_tasks.go file. If not
|
|
// specified, uses infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
|
|
// from this repo.
|
|
BuilderNameSchemaFile string `json:"builder_name_schema"`
|
|
|
|
// URL of the Skia Gold known hashes endpoint.
|
|
GoldHashesURL string `json:"gold_hashes_url"`
|
|
|
|
// GCS bucket used for GM results.
|
|
GsBucketGm string `json:"gs_bucket_gm"`
|
|
|
|
// GCS bucket used for Nanobench results.
|
|
GsBucketNano string `json:"gs_bucket_nano"`
|
|
|
|
// Optional function which returns a bot ID for internal devices.
|
|
InternalHardwareLabel func(parts map[string]string) *int `json:"-"`
|
|
|
|
// List of task names for which we'll never upload results.
|
|
NoUpload []string `json:"no_upload"`
|
|
|
|
// PathToSkia is the relative path from the root of the current checkout to
|
|
// the root of the Skia checkout.
|
|
PathToSkia string `json:"path_to_skia"`
|
|
|
|
// Swarming pool used for triggering tasks.
|
|
Pool string `json:"pool"`
|
|
|
|
// LUCI project associated with this repo.
|
|
Project string `json:"project"`
|
|
|
|
// Service accounts.
|
|
ServiceAccountCanary string `json:"service_account_canary"`
|
|
ServiceAccountCompile string `json:"service_account_compile"`
|
|
ServiceAccountHousekeeper string `json:"service_account_housekeeper"`
|
|
ServiceAccountRecreateSKPs string `json:"service_account_recreate_skps"`
|
|
ServiceAccountUploadBinary string `json:"service_account_upload_binary"`
|
|
ServiceAccountUploadGM string `json:"service_account_upload_gm"`
|
|
ServiceAccountUploadNano string `json:"service_account_upload_nano"`
|
|
|
|
// Optional override function which derives Swarming bot dimensions
|
|
// from parts of task names.
|
|
SwarmDimensions func(parts map[string]string) []string `json:"-"`
|
|
}
|
|
|
|
// LoadConfig loads the Config from a cfg.json file which is the sibling of the
|
|
// calling gen_tasks.go file.
|
|
func LoadConfig() *Config {
|
|
cfgDir := getCallingDirName()
|
|
var cfg Config
|
|
LoadJson(filepath.Join(cfgDir, "cfg.json"), &cfg)
|
|
return &cfg
|
|
}
|
|
|
|
// CheckoutRoot is a wrapper around specs.GetCheckoutRoot which prevents the
|
|
// caller from needing a dependency on the specs package.
|
|
func CheckoutRoot() string {
|
|
root, err := specs.GetCheckoutRoot()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return root
|
|
}
|
|
|
|
// LoadJson loads JSON from the given file and unmarshals it into the given
|
|
// destination.
|
|
func LoadJson(filename string, dest interface{}) {
|
|
b, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
log.Fatalf("Unable to read %q: %s", filename, err)
|
|
}
|
|
if err := json.Unmarshal(b, dest); err != nil {
|
|
log.Fatalf("Unable to parse %q: %s", filename, err)
|
|
}
|
|
}
|
|
|
|
// In returns true if |s| is *in* |a| slice.
|
|
// TODO(borenet): This is copied from go.skia.org/infra/go/util to avoid the
|
|
// huge set of additional dependencies added by that package.
|
|
func In(s string, a []string) bool {
|
|
for _, x := range a {
|
|
if x == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GenTasks regenerates the tasks.json file. Loads the job list from a jobs.json
|
|
// file which is the sibling of the calling gen_tasks.go file. If cfg is nil, it
|
|
// is similarly loaded from a cfg.json file which is the sibling of the calling
|
|
// gen_tasks.go file.
|
|
func GenTasks(cfg *Config) {
|
|
b := specs.MustNewTasksCfgBuilder()
|
|
|
|
// Find the paths to the infra/bots directories in this repo and the
|
|
// repo of the calling file.
|
|
relpathTargetDir := getThisDirName()
|
|
relpathBaseDir := getCallingDirName()
|
|
|
|
var jobs []string
|
|
LoadJson(filepath.Join(relpathBaseDir, "jobs.json"), &jobs)
|
|
|
|
if cfg == nil {
|
|
cfg = new(Config)
|
|
LoadJson(filepath.Join(relpathBaseDir, "cfg.json"), cfg)
|
|
}
|
|
|
|
// Create the JobNameSchema.
|
|
builderNameSchemaFile := filepath.Join(relpathTargetDir, "recipe_modules", "builder_name_schema", "builder_name_schema.json")
|
|
if cfg.BuilderNameSchemaFile != "" {
|
|
builderNameSchemaFile = filepath.Join(relpathBaseDir, cfg.BuilderNameSchemaFile)
|
|
}
|
|
schema, err := NewJobNameSchema(builderNameSchemaFile)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// Set the assets dir.
|
|
assetsDir := filepath.Join(relpathTargetDir, "assets")
|
|
if cfg.AssetsDir != "" {
|
|
assetsDir = filepath.Join(relpathBaseDir, cfg.AssetsDir)
|
|
}
|
|
b.SetAssetsDir(assetsDir)
|
|
|
|
// Create Tasks and Jobs.
|
|
builder := &builder{
|
|
TasksCfgBuilder: b,
|
|
cfg: cfg,
|
|
jobNameSchema: schema,
|
|
jobs: jobs,
|
|
}
|
|
for _, name := range jobs {
|
|
jb := newJobBuilder(builder, name)
|
|
jb.genTasksForJob()
|
|
jb.finish()
|
|
}
|
|
|
|
// Create CasSpecs.
|
|
b.MustAddCasSpec(CAS_CANVASKIT, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/infra/canvaskit",
|
|
"skia/modules/canvaskit",
|
|
"skia/modules/pathkit/perf/perfReporter.js",
|
|
"skia/modules/pathkit/tests/testReporter.js",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_EMPTY, specs.EmptyCasSpec)
|
|
b.MustAddCasSpec(CAS_LOTTIE_CI, CAS_SPEC_LOTTIE_CI)
|
|
b.MustAddCasSpec(CAS_LOTTIE_WEB, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/tools/lottie-web-perf",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_PATHKIT, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/infra/pathkit",
|
|
"skia/modules/pathkit",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_PERF, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/assets",
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/platform_tools/ios/bin",
|
|
"skia/resources",
|
|
"skia/tools/valgrind.supp",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_PUPPETEER, &specs.CasSpec{
|
|
Root: "../skia", // Needed for other repos.
|
|
Paths: []string{
|
|
"tools/perf-canvaskit-puppeteer",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_RECIPES, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/config/recipes.cfg",
|
|
"skia/infra/bots/bundle_recipes.sh",
|
|
"skia/infra/bots/README.recipes.md",
|
|
"skia/infra/bots/recipe_modules",
|
|
"skia/infra/bots/recipes",
|
|
"skia/infra/bots/recipes.py",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_RUN_RECIPE, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_SKOTTIE_WASM, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/tools/skottie-wasm-perf",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_SKPBENCH, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/assets",
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/tools/skpbench",
|
|
"skia/tools/valgrind.supp",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_TASK_DRIVERS, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/go.mod",
|
|
"skia/go.sum",
|
|
"skia/infra/bots/build_task_drivers.sh",
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/infra/bots/task_drivers",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_TEST, &specs.CasSpec{
|
|
Root: "..",
|
|
Paths: []string{
|
|
"skia/infra/bots/assets",
|
|
"skia/infra/bots/run_recipe.py",
|
|
"skia/platform_tools/ios/bin",
|
|
"skia/resources",
|
|
"skia/tools/valgrind.supp",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_WASM_GM, &specs.CasSpec{
|
|
Root: "../skia", // Needed for other repos.
|
|
Paths: []string{
|
|
"resources",
|
|
"tools/run-wasm-gm-tests",
|
|
},
|
|
Excludes: []string{rbe.ExcludeGitDir},
|
|
})
|
|
b.MustAddCasSpec(CAS_WHOLE_REPO, CAS_SPEC_WHOLE_REPO)
|
|
generateCompileCAS(b, cfg)
|
|
|
|
builder.MustFinish()
|
|
}
|
|
|
|
// getThisDirName returns the infra/bots directory which is an ancestor of this
|
|
// file.
|
|
func getThisDirName() string {
|
|
_, thisFileName, _, ok := runtime.Caller(0)
|
|
if !ok {
|
|
log.Fatal("Unable to find path to current file.")
|
|
}
|
|
return filepath.Dir(filepath.Dir(thisFileName))
|
|
}
|
|
|
|
// getCallingDirName returns the infra/bots directory which is an ancestor of
|
|
// the calling gen_tasks.go file. WARNING: assumes that the calling gen_tasks.go
|
|
// file appears two steps up the stack; do not call from a function which is not
|
|
// directly called by gen_tasks.go.
|
|
func getCallingDirName() string {
|
|
_, callingFileName, _, ok := runtime.Caller(2)
|
|
if !ok {
|
|
log.Fatal("Unable to find path to calling file.")
|
|
}
|
|
return filepath.Dir(callingFileName)
|
|
}
|
|
|
|
// builder is a wrapper for specs.TasksCfgBuilder.
|
|
type builder struct {
|
|
*specs.TasksCfgBuilder
|
|
cfg *Config
|
|
jobNameSchema *JobNameSchema
|
|
jobs []string
|
|
}
|
|
|
|
// marshalJson encodes the given data as JSON and fixes escaping of '<' which Go
|
|
// does by default.
|
|
func marshalJson(data interface{}) string {
|
|
j, err := json.Marshal(data)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return strings.Replace(string(j), "\\u003c", "<", -1)
|
|
}
|
|
|
|
// kitchenTaskNoBundle sets up the task to run a recipe via Kitchen, without the
|
|
// recipe bundle.
|
|
func (b *taskBuilder) kitchenTaskNoBundle(recipe string, outputDir string) {
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.cipd(cipd.MustGetPackage("infra/tools/luci/kitchen/${platform}"))
|
|
b.usesPython()
|
|
b.recipeProp("swarm_out_dir", outputDir)
|
|
if outputDir != OUTPUT_NONE {
|
|
b.output(outputDir)
|
|
}
|
|
python := "cipd_bin_packages/vpython3${EXECUTABLE_SUFFIX}"
|
|
if b.role("Test", "Perf") && b.matchOs("Win7") && b.matchModel("Golo") {
|
|
// TODO(borenet): Win7 machines in the Golo seem to be missing a
|
|
// necessary DLL to make python3 work.
|
|
python = "cipd_bin_packages/vpython"
|
|
}
|
|
b.cmd(python, "-u", "skia/infra/bots/run_recipe.py", "${ISOLATED_OUTDIR}", recipe, b.getRecipeProps(), b.cfg.Project)
|
|
// Most recipes want this isolate; they can override if necessary.
|
|
b.cas(CAS_RUN_RECIPE)
|
|
b.timeout(time.Hour)
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
|
|
b.Spec.ExtraTags = map[string]string{
|
|
"log_location": fmt.Sprintf("logdog://logs.chromium.org/%s/${SWARMING_TASK_ID}/+/annotations", b.cfg.Project),
|
|
}
|
|
|
|
// Attempts.
|
|
if !b.role("Build", "Upload") && b.extraConfig("ASAN", "MSAN", "TSAN", "Valgrind") {
|
|
// Sanitizers often find non-deterministic issues that retries would hide.
|
|
b.attempts(1)
|
|
} else {
|
|
// Retry by default to hide random bot/hardware failures.
|
|
b.attempts(2)
|
|
}
|
|
}
|
|
|
|
// kitchenTask sets up the task to run a recipe via Kitchen.
|
|
func (b *taskBuilder) kitchenTask(recipe string, outputDir string) {
|
|
b.kitchenTaskNoBundle(recipe, outputDir)
|
|
b.dep(b.bundleRecipes())
|
|
}
|
|
|
|
// internalHardwareLabel returns the internal ID for the bot, if any.
|
|
func (b *taskBuilder) internalHardwareLabel() *int {
|
|
if b.cfg.InternalHardwareLabel != nil {
|
|
return b.cfg.InternalHardwareLabel(b.parts)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// linuxGceDimensions adds the Swarming bot dimensions for Linux GCE instances.
|
|
func (b *taskBuilder) linuxGceDimensions(machineType string) {
|
|
b.dimension(
|
|
// Specify CPU to avoid running builds on bots with a more unique CPU.
|
|
"cpu:x86-64-Haswell_GCE",
|
|
"gpu:none",
|
|
// Currently all Linux GCE tasks run on 16-CPU machines.
|
|
fmt.Sprintf("machine_type:%s", machineType),
|
|
fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE),
|
|
fmt.Sprintf("pool:%s", b.cfg.Pool),
|
|
)
|
|
}
|
|
|
|
// deriveCompileTaskName returns the name of a compile task based on the given
|
|
// job name.
|
|
func (b *jobBuilder) deriveCompileTaskName() string {
|
|
if b.role("Test", "Perf", "FM") {
|
|
task_os := b.parts["os"]
|
|
ec := []string{}
|
|
if val := b.parts["extra_config"]; val != "" {
|
|
ec = strings.Split(val, "_")
|
|
ignore := []string{
|
|
"Skpbench", "AbandonGpuContext", "PreAbandonGpuContext", "Valgrind",
|
|
"ReleaseAndAbandonGpuContext", "FSAA", "FAAA", "FDAA", "NativeFonts", "GDI",
|
|
"NoGPUThreads", "ProcDump", "DDL1", "DDL3", "OOPRDDL", "T8888",
|
|
"DDLTotal", "DDLRecord", "9x9", "BonusConfigs", "SkottieTracing", "SkottieWASM",
|
|
"GpuTess", "DMSAAStats", "Mskp", "Docker", "PDF", "SkVM", "Puppeteer",
|
|
"SkottieFrames", "RenderSKP", "CanvasPerf", "AllPathsVolatile", "WebGL2"}
|
|
keep := make([]string, 0, len(ec))
|
|
for _, part := range ec {
|
|
if !In(part, ignore) {
|
|
keep = append(keep, part)
|
|
}
|
|
}
|
|
ec = keep
|
|
}
|
|
if b.os("Android") {
|
|
if !In("Android", ec) {
|
|
ec = append([]string{"Android"}, ec...)
|
|
}
|
|
task_os = COMPILE_TASK_NAME_OS_LINUX
|
|
} else if b.os("ChromeOS") {
|
|
ec = append([]string{"Chromebook", "GLES"}, ec...)
|
|
task_os = COMPILE_TASK_NAME_OS_LINUX
|
|
} else if b.os("iOS") {
|
|
ec = append([]string{task_os}, ec...)
|
|
task_os = "Mac"
|
|
} else if b.matchOs("Win") {
|
|
task_os = "Win"
|
|
} else if b.compiler("GCC") {
|
|
// GCC compiles are now on a Docker container. We use the same OS and
|
|
// version to compile as to test.
|
|
ec = append(ec, "Docker")
|
|
} else if b.matchOs("Ubuntu", "Debian") {
|
|
task_os = COMPILE_TASK_NAME_OS_LINUX
|
|
} else if b.matchOs("Mac") {
|
|
task_os = "Mac"
|
|
}
|
|
jobNameMap := map[string]string{
|
|
"role": "Build",
|
|
"os": task_os,
|
|
"compiler": b.parts["compiler"],
|
|
"target_arch": b.parts["arch"],
|
|
"configuration": b.parts["configuration"],
|
|
}
|
|
if b.extraConfig("PathKit") {
|
|
ec = []string{"PathKit"}
|
|
}
|
|
if b.extraConfig("CanvasKit", "SkottieWASM", "Puppeteer") {
|
|
if b.cpu() {
|
|
ec = []string{"CanvasKit_CPU"}
|
|
} else {
|
|
ec = []string{"CanvasKit"}
|
|
}
|
|
|
|
}
|
|
if len(ec) > 0 {
|
|
jobNameMap["extra_config"] = strings.Join(ec, "_")
|
|
}
|
|
name, err := b.jobNameSchema.MakeJobName(jobNameMap)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return name
|
|
} else if b.parts["role"] == "BuildStats" {
|
|
return strings.Replace(b.Name, "BuildStats", "Build", 1)
|
|
} else {
|
|
return b.Name
|
|
}
|
|
}
|
|
|
|
// swarmDimensions generates swarming bot dimensions for the given task.
|
|
func (b *taskBuilder) swarmDimensions() {
|
|
if b.cfg.SwarmDimensions != nil {
|
|
dims := b.cfg.SwarmDimensions(b.parts)
|
|
if dims != nil {
|
|
b.dimension(dims...)
|
|
return
|
|
}
|
|
}
|
|
b.defaultSwarmDimensions()
|
|
}
|
|
|
|
// defaultSwarmDimensions generates default swarming bot dimensions for the given task.
|
|
func (b *taskBuilder) defaultSwarmDimensions() {
|
|
d := map[string]string{
|
|
"pool": b.cfg.Pool,
|
|
}
|
|
if os, ok := b.parts["os"]; ok {
|
|
d["os"], ok = map[string]string{
|
|
"Android": "Android",
|
|
"ChromeOS": "ChromeOS",
|
|
"Debian9": DEFAULT_OS_LINUX_GCE, // Runs in Deb9 Docker.
|
|
"Debian10": DEFAULT_OS_LINUX_GCE,
|
|
"Mac": DEFAULT_OS_MAC,
|
|
"Mac10.13": "Mac-10.13.6",
|
|
"Mac10.14": "Mac-10.14.3",
|
|
"Mac10.15.1": "Mac-10.15.1",
|
|
"Mac10.15.7": "Mac-10.15.7", // Same as 'Mac', but explicit.
|
|
"Mac11": "Mac-11.1",
|
|
"Ubuntu18": "Ubuntu-18.04",
|
|
"Win": DEFAULT_OS_WIN,
|
|
"Win10": "Windows-10-19041",
|
|
"Win2019": DEFAULT_OS_WIN,
|
|
"Win7": "Windows-7-SP1",
|
|
"Win8": "Windows-8.1-SP0",
|
|
"iOS": "iOS-13.3.1",
|
|
}[os]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in OS mapping.", os)
|
|
}
|
|
if os == "Win10" && b.parts["model"] == "Golo" {
|
|
// ChOps-owned machines have Windows 10 v1709.
|
|
d["os"] = "Windows-10-16299"
|
|
}
|
|
if os == "Mac10.14" && b.parts["model"] == "VMware7.1" {
|
|
// ChOps VMs are at a newer version of MacOS.
|
|
d["os"] = "Mac-10.14.6"
|
|
}
|
|
if os == "Mac10.15" && b.parts["model"] == "VMware7.1" {
|
|
// ChOps VMs are at a newer version of MacOS.
|
|
d["os"] = "Mac-10.15.7"
|
|
}
|
|
if b.parts["model"] == "iPhone6" {
|
|
// This is the latest iOS that supports iPhone6.
|
|
d["os"] = "iOS-12.4.5"
|
|
}
|
|
if b.parts["model"] == "iPhone11" {
|
|
d["os"] = "iOS-13.6"
|
|
}
|
|
if b.parts["model"] == "iPadPro" {
|
|
d["os"] = "iOS-13.6"
|
|
}
|
|
} else {
|
|
d["os"] = DEFAULT_OS_DEBIAN
|
|
}
|
|
if b.role("Test", "Perf") {
|
|
if b.os("Android") {
|
|
// For Android, the device type is a better dimension
|
|
// than CPU or GPU.
|
|
deviceInfo, ok := map[string][]string{
|
|
"AndroidOne": {"sprout", "MOB30Q"},
|
|
// S6 dimensions are more general than we would like. See skbug.com/11337 for context.
|
|
"GalaxyS6": {"universal7420", "NRD90M"},
|
|
"GalaxyS7_G930FD": {"herolte", "R16NW_G930FXXS2ERH6"}, // This is Oreo.
|
|
"GalaxyS9": {"starlte", "QP1A.190711.020"}, // This is Android10.
|
|
"GalaxyS20": {"exynos990", "QP1A.190711.020"},
|
|
"Nexus5": {"hammerhead", "M4B30Z_3437181"},
|
|
"Nexus5x": {"bullhead", "OPR6.170623.023"},
|
|
"Nexus7": {"grouper", "LMY47V_1836172"}, // 2012 Nexus 7
|
|
"P30": {"HWELE", "HUAWEIELE-L29"},
|
|
"Pixel2XL": {"taimen", "PPR1.180610.009"},
|
|
"Pixel3": {"blueline", "PQ1A.190105.004"},
|
|
"Pixel3a": {"sargo", "QP1A.190711.020"},
|
|
"Pixel4": {"flame", "RPB2.200611.009"}, // R Preview
|
|
"Pixel4XL": {"coral", "QD1A.190821.011.C4"},
|
|
"Pixel5": {"redfin", "RD1A.200810.022.A4"},
|
|
"TecnoSpark3Pro": {"TECNO-KB8", "PPR1.180610.011"},
|
|
}[b.parts["model"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in Android mapping.", b.parts["model"])
|
|
}
|
|
d["device_type"] = deviceInfo[0]
|
|
d["device_os"] = deviceInfo[1]
|
|
} else if b.os("iOS") {
|
|
device, ok := map[string]string{
|
|
"iPadMini4": "iPad5,1",
|
|
"iPhone6": "iPhone7,2",
|
|
"iPhone7": "iPhone9,1",
|
|
"iPhone8": "iPhone10,1",
|
|
"iPhone11": "iPhone12,1",
|
|
"iPadPro": "iPad6,3",
|
|
}[b.parts["model"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in iOS mapping.", b.parts["model"])
|
|
}
|
|
d["device_type"] = device
|
|
// Temporarily use this dimension to ensure we only use the new libimobiledevice, since the
|
|
// old version won't work with current recipes.
|
|
d["libimobiledevice"] = "1582155448"
|
|
} else if b.cpu() || b.extraConfig("CanvasKit", "Docker", "SwiftShader") {
|
|
modelMapping, ok := map[string]map[string]string{
|
|
"AppleM1": {
|
|
"MacMini9.1": "arm64-64-Apple_M1",
|
|
},
|
|
"AVX": {
|
|
"VMware7.1": "x86-64-E5-2697_v2",
|
|
},
|
|
"AVX2": {
|
|
"GCE": "x86-64-Haswell_GCE",
|
|
"MacBookAir7.2": "x86-64-i5-5350U",
|
|
"MacBookPro11.5": "x86-64-i7-4870HQ",
|
|
"NUC5i7RYH": "x86-64-i7-5557U",
|
|
},
|
|
"AVX512": {
|
|
"GCE": "x86-64-Skylake_GCE",
|
|
"Golo": "Intel64_Family_6_Model_85_Stepping_7__GenuineIntel",
|
|
},
|
|
"Rome": {
|
|
"GCE": "x86-64-AMD_Rome_GCE",
|
|
},
|
|
"SwiftShader": {
|
|
"GCE": "x86-64-Haswell_GCE",
|
|
},
|
|
}[b.parts["cpu_or_gpu_value"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in CPU mapping.", b.parts["cpu_or_gpu_value"])
|
|
}
|
|
cpu, ok := modelMapping[b.parts["model"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in %q model mapping.", b.parts["model"], b.parts["cpu_or_gpu_value"])
|
|
}
|
|
d["cpu"] = cpu
|
|
if b.model("GCE") && b.matchOs("Debian") {
|
|
d["os"] = DEFAULT_OS_LINUX_GCE
|
|
}
|
|
if b.model("GCE") && d["cpu"] == "x86-64-Haswell_GCE" {
|
|
d["machine_type"] = MACHINE_TYPE_MEDIUM
|
|
}
|
|
} else {
|
|
if b.matchOs("Win") {
|
|
gpu, ok := map[string]string{
|
|
// At some point this might use the device ID, but for now it's like Chromebooks.
|
|
"Adreno630": "Adreno630",
|
|
"GT610": "10de:104a-23.21.13.9101",
|
|
"GTX660": "10de:11c0-26.21.14.4120",
|
|
"GTX960": "10de:1401-27.21.14.5671",
|
|
"IntelHD4400": "8086:0a16-20.19.15.4963",
|
|
"IntelIris540": "8086:1926-26.20.100.7463",
|
|
"IntelIris6100": "8086:162b-20.19.15.4963",
|
|
"IntelIris655": "8086:3ea5-26.20.100.7463",
|
|
"RadeonHD7770": "1002:683d-26.20.13031.18002",
|
|
"RadeonR9M470X": "1002:6646-26.20.13031.18002",
|
|
"QuadroP400": "10de:1cb3-25.21.14.1678",
|
|
}[b.parts["cpu_or_gpu_value"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in Win GPU mapping.", b.parts["cpu_or_gpu_value"])
|
|
}
|
|
d["gpu"] = gpu
|
|
} else if b.isLinux() {
|
|
gpu, ok := map[string]string{
|
|
// Intel drivers come from CIPD, so no need to specify the version here.
|
|
"IntelBayTrail": "8086:0f31",
|
|
"IntelHD2000": "8086:0102",
|
|
"IntelHD405": "8086:22b1",
|
|
"IntelIris640": "8086:5926",
|
|
"QuadroP400": "10de:1cb3-430.14",
|
|
}[b.parts["cpu_or_gpu_value"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in Ubuntu GPU mapping.", b.parts["cpu_or_gpu_value"])
|
|
}
|
|
d["gpu"] = gpu
|
|
} else if b.matchOs("Mac") {
|
|
gpu, ok := map[string]string{
|
|
"AppleM1": "AppleM1",
|
|
"IntelHD6000": "8086:1626",
|
|
"IntelHD615": "8086:591e",
|
|
"IntelIris5100": "8086:0a2e",
|
|
"RadeonHD8870M": "1002:6821-4.0.20-3.2.8",
|
|
}[b.parts["cpu_or_gpu_value"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in Mac GPU mapping.", b.parts["cpu_or_gpu_value"])
|
|
}
|
|
if gpu == "AppleM1" {
|
|
// No GPU dimension yet, but we can constrain by CPU.
|
|
d["cpu"] = "arm64-64-Apple_M1"
|
|
} else {
|
|
d["gpu"] = gpu
|
|
}
|
|
// Yuck. We have two different types of MacMini7,1 with the same GPU but different CPUs.
|
|
if b.gpu("IntelIris5100") {
|
|
// Run all tasks on Golo machines for now.
|
|
d["cpu"] = "x86-64-i7-4578U"
|
|
}
|
|
} else if b.os("ChromeOS") {
|
|
version, ok := map[string]string{
|
|
"MaliT604": "10575.22.0",
|
|
"MaliT764": "10575.22.0",
|
|
"MaliT860": "10575.22.0",
|
|
"PowerVRGX6250": "10575.22.0",
|
|
"TegraK1": "10575.22.0",
|
|
"IntelHDGraphics615": "10575.22.0",
|
|
"IntelUHDGraphics605": "13729.56.0",
|
|
"RadeonVega3": "13729.56.0",
|
|
}[b.parts["cpu_or_gpu_value"]]
|
|
if !ok {
|
|
log.Fatalf("Entry %q not found in ChromeOS GPU mapping.", b.parts["cpu_or_gpu_value"])
|
|
}
|
|
d["gpu"] = b.parts["cpu_or_gpu_value"]
|
|
d["release_version"] = version
|
|
} else {
|
|
log.Fatalf("Unknown GPU mapping for OS %q.", b.parts["os"])
|
|
}
|
|
}
|
|
} else {
|
|
d["gpu"] = "none"
|
|
if d["os"] == DEFAULT_OS_LINUX_GCE {
|
|
if b.extraConfig("CanvasKit", "CMake", "Docker", "PathKit") || b.role("BuildStats") {
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
return
|
|
}
|
|
// Use many-core machines for Build tasks.
|
|
b.linuxGceDimensions(MACHINE_TYPE_LARGE)
|
|
return
|
|
} else if d["os"] == DEFAULT_OS_WIN {
|
|
// Windows CPU bots.
|
|
d["cpu"] = "x86-64-Haswell_GCE"
|
|
// Use many-core machines for Build tasks.
|
|
d["machine_type"] = MACHINE_TYPE_LARGE
|
|
} else if d["os"] == DEFAULT_OS_MAC {
|
|
// Mac CPU bots.
|
|
d["cpu"] = "x86-64-E5-2697_v2"
|
|
}
|
|
}
|
|
|
|
dims := make([]string, 0, len(d))
|
|
for k, v := range d {
|
|
dims = append(dims, fmt.Sprintf("%s:%s", k, v))
|
|
}
|
|
sort.Strings(dims)
|
|
b.dimension(dims...)
|
|
}
|
|
|
|
// bundleRecipes generates the task to bundle and isolate the recipes. Returns
|
|
// the name of the task, which may be added as a dependency.
|
|
func (b *jobBuilder) bundleRecipes() string {
|
|
b.addTask(BUNDLE_RECIPES_NAME, func(b *taskBuilder) {
|
|
b.cipd(specs.CIPD_PKGS_GIT_LINUX_AMD64...)
|
|
b.cipd(specs.CIPD_PKGS_PYTHON_LINUX_AMD64...)
|
|
b.cmd("/bin/bash", "skia/infra/bots/bundle_recipes.sh", specs.PLACEHOLDER_ISOLATED_OUTDIR)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
|
|
b.idempotent()
|
|
b.cas(CAS_RECIPES)
|
|
})
|
|
return BUNDLE_RECIPES_NAME
|
|
}
|
|
|
|
// buildTaskDrivers generates the task to compile the task driver code to run on
|
|
// all platforms. Returns the name of the task, which may be added as a
|
|
// dependency.
|
|
func (b *jobBuilder) buildTaskDrivers(goos, goarch string) string {
|
|
name := BUILD_TASK_DRIVERS_PREFIX + "_" + goos + "_" + goarch
|
|
b.addTask(name, func(b *taskBuilder) {
|
|
b.usesGo()
|
|
b.cmd("/bin/bash", "skia/infra/bots/build_task_drivers.sh",
|
|
specs.PLACEHOLDER_ISOLATED_OUTDIR,
|
|
goos,
|
|
goarch)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
|
|
b.idempotent()
|
|
b.cas(CAS_TASK_DRIVERS)
|
|
})
|
|
return name
|
|
}
|
|
|
|
// updateGoDeps generates the task to update Go dependencies.
|
|
func (b *jobBuilder) updateGoDeps() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.usesGo()
|
|
b.asset("protoc")
|
|
b.cmd(
|
|
"./update_go_deps",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--workdir", ".",
|
|
"--gerrit_project", "skia",
|
|
"--gerrit_url", "https://skia-review.googlesource.com",
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--alsologtostderr",
|
|
)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
|
|
b.cas(CAS_EMPTY)
|
|
b.serviceAccount(b.cfg.ServiceAccountRecreateSKPs)
|
|
})
|
|
}
|
|
|
|
// createDockerImage creates the specified docker image. Returns the name of the
|
|
// generated task.
|
|
func (b *jobBuilder) createDockerImage(wasm bool) string {
|
|
// First, derive the name of the task.
|
|
imageName := "skia-release"
|
|
taskName := "Housekeeper-PerCommit-CreateDockerImage_Skia_Release"
|
|
if wasm {
|
|
imageName = "skia-wasm-release"
|
|
taskName = "Housekeeper-PerCommit-CreateDockerImage_Skia_WASM_Release"
|
|
}
|
|
imageDir := path.Join("docker", imageName)
|
|
|
|
// Add the task.
|
|
b.addTask(taskName, func(b *taskBuilder) {
|
|
// TODO(borenet): Make this task not use Git.
|
|
b.usesGit()
|
|
b.cmd(
|
|
"./build_push_docker_image",
|
|
"--image_name", fmt.Sprintf("gcr.io/skia-public/%s", imageName),
|
|
"--dockerfile_dir", imageDir,
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--workdir", ".",
|
|
"--gerrit_project", "skia",
|
|
"--gerrit_url", "https://skia-review.googlesource.com",
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--swarm_out_dir", specs.PLACEHOLDER_ISOLATED_OUTDIR,
|
|
"--alsologtostderr",
|
|
)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
|
|
b.cas(CAS_EMPTY)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.usesDocker()
|
|
b.cache(CACHES_DOCKER...)
|
|
})
|
|
return taskName
|
|
}
|
|
|
|
// createPushAppsFromSkiaDockerImage creates and pushes docker images of some apps
|
|
// (eg: fiddler, debugger, api) using the skia-release docker image.
|
|
func (b *jobBuilder) createPushAppsFromSkiaDockerImage() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
// TODO(borenet): Make this task not use Git.
|
|
b.usesGit()
|
|
b.cmd(
|
|
"./push_apps_from_skia_image",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--workdir", ".",
|
|
"--gerrit_project", "buildbot",
|
|
"--gerrit_url", "https://skia-review.googlesource.com",
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--alsologtostderr",
|
|
)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.dep(b.createDockerImage(false))
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
|
|
b.cas(CAS_EMPTY)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.usesDocker()
|
|
b.cache(CACHES_DOCKER...)
|
|
})
|
|
}
|
|
|
|
// createPushAppsFromWASMDockerImage creates and pushes docker images of some apps
|
|
// (eg: jsfiddle, skottie, particles) using the skia-wasm-release docker image.
|
|
func (b *jobBuilder) createPushAppsFromWASMDockerImage() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
// TODO(borenet): Make this task not use Git.
|
|
b.usesGit()
|
|
b.cmd(
|
|
"./push_apps_from_wasm_image",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--workdir", ".",
|
|
"--gerrit_project", "buildbot",
|
|
"--gerrit_url", "https://skia-review.googlesource.com",
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--alsologtostderr",
|
|
)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.dep(b.createDockerImage(true))
|
|
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin", "go/go/bin")
|
|
b.cas(CAS_EMPTY)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.usesDocker()
|
|
b.cache(CACHES_DOCKER...)
|
|
})
|
|
}
|
|
|
|
var iosRegex = regexp.MustCompile(`os:iOS-(.*)`)
|
|
|
|
func (b *taskBuilder) maybeAddIosDevImage() {
|
|
for _, dim := range b.Spec.Dimensions {
|
|
if m := iosRegex.FindStringSubmatch(dim); len(m) >= 2 {
|
|
var asset string
|
|
switch m[1] {
|
|
// Other patch versions can be added to the same case.
|
|
case "11.4.1":
|
|
asset = "ios-dev-image-11.4"
|
|
case "12.4.5":
|
|
asset = "ios-dev-image-12.4"
|
|
case "13.3.1":
|
|
asset = "ios-dev-image-13.3"
|
|
case "13.4.1":
|
|
asset = "ios-dev-image-13.4"
|
|
case "13.5.1":
|
|
asset = "ios-dev-image-13.5"
|
|
case "13.6":
|
|
asset = "ios-dev-image-13.6"
|
|
default:
|
|
log.Fatalf("Unable to determine correct ios-dev-image asset for %s. If %s is a new iOS release, you must add a CIPD package containing the corresponding iOS dev image; see ios-dev-image-11.4 for an example.", b.Name, m[1])
|
|
}
|
|
b.asset(asset)
|
|
break
|
|
} else if strings.Contains(dim, "iOS") {
|
|
log.Fatalf("Must specify iOS version for %s to obtain correct dev image; os dimension is missing version: %s", b.Name, dim)
|
|
}
|
|
}
|
|
}
|
|
|
|
// compile generates a compile task. Returns the name of the compile task.
|
|
func (b *jobBuilder) compile() string {
|
|
name := b.deriveCompileTaskName()
|
|
if b.extraConfig("WasmGMTests") {
|
|
b.compileWasmGMTests(name)
|
|
} else {
|
|
b.addTask(name, func(b *taskBuilder) {
|
|
recipe := "compile"
|
|
casSpec := CAS_COMPILE
|
|
if b.extraConfig("NoDEPS", "CMake", "CommandBuffer", "Flutter") {
|
|
recipe = "sync_and_compile"
|
|
casSpec = CAS_RUN_RECIPE
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.usesGit()
|
|
if !b.extraConfig("NoDEPS") {
|
|
b.cache(CACHES_WORKDIR...)
|
|
}
|
|
} else {
|
|
b.idempotent()
|
|
}
|
|
b.kitchenTask(recipe, OUTPUT_BUILD)
|
|
b.cas(casSpec)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.swarmDimensions()
|
|
if b.extraConfig("Docker", "LottieWeb", "CMake") || b.compiler("EMCC") {
|
|
b.usesDocker()
|
|
b.cache(CACHES_DOCKER...)
|
|
}
|
|
|
|
// Android bots require a toolchain.
|
|
if b.extraConfig("Android") {
|
|
if b.matchOs("Mac") {
|
|
b.asset("android_ndk_darwin")
|
|
} else if b.matchOs("Win") {
|
|
pkg := b.MustGetCipdPackageFromAsset("android_ndk_windows")
|
|
pkg.Path = "n"
|
|
b.cipd(pkg)
|
|
} else {
|
|
b.asset("android_ndk_linux")
|
|
}
|
|
} else if b.extraConfig("Chromebook") {
|
|
b.asset("clang_linux")
|
|
if b.arch("x86_64") {
|
|
b.asset("chromebook_x86_64_gles")
|
|
} else if b.arch("arm") {
|
|
b.asset("armhf_sysroot")
|
|
b.asset("chromebook_arm_gles")
|
|
}
|
|
} else if b.isLinux() {
|
|
if b.compiler("Clang") {
|
|
b.asset("clang_linux")
|
|
}
|
|
if b.extraConfig("SwiftShader") {
|
|
b.asset("cmake_linux")
|
|
}
|
|
b.asset("ccache_linux")
|
|
b.usesCCache()
|
|
} else if b.matchOs("Win") {
|
|
b.asset("win_toolchain")
|
|
if b.compiler("Clang") {
|
|
b.asset("clang_win")
|
|
}
|
|
} else if b.matchOs("Mac") {
|
|
b.cipd(CIPD_PKGS_XCODE...)
|
|
b.Spec.Caches = append(b.Spec.Caches, &specs.Cache{
|
|
Name: "xcode",
|
|
Path: "cache/Xcode.app",
|
|
})
|
|
b.asset("ccache_mac")
|
|
b.usesCCache()
|
|
if b.extraConfig("CommandBuffer") {
|
|
b.timeout(2 * time.Hour)
|
|
}
|
|
if b.extraConfig("iOS") {
|
|
b.asset("provisioning_profile_ios")
|
|
}
|
|
// See skbug.com/11129 for more
|
|
if b.compiler("Xcode11.4.1") {
|
|
b.dimension("reserved_for_xcode_version:11.4.1")
|
|
} else {
|
|
b.dimension("reserved_for_xcode_version:newest")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// All compile tasks are runnable as their own Job. Assert that the Job
|
|
// is listed in jobs.
|
|
if !In(name, b.jobs) {
|
|
log.Fatalf("Job %q is missing from the jobs list! Derived from: %q", name, b.Name)
|
|
}
|
|
|
|
return name
|
|
}
|
|
|
|
// recreateSKPs generates a RecreateSKPs task.
|
|
func (b *jobBuilder) recreateSKPs() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTask("recreate_skps", OUTPUT_NONE)
|
|
b.serviceAccount(b.cfg.ServiceAccountRecreateSKPs)
|
|
b.dimension(
|
|
"pool:SkiaCT",
|
|
fmt.Sprintf("os:%s", DEFAULT_OS_LINUX_GCE),
|
|
)
|
|
b.usesGo()
|
|
b.cache(CACHES_WORKDIR...)
|
|
b.timeout(4 * time.Hour)
|
|
})
|
|
}
|
|
|
|
// checkGeneratedFiles verifies that no generated SKSL files have been edited
|
|
// by hand.
|
|
func (b *jobBuilder) checkGeneratedFiles() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTask("check_generated_files", OUTPUT_NONE)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.linuxGceDimensions(MACHINE_TYPE_LARGE)
|
|
b.usesGo()
|
|
b.asset("clang_linux")
|
|
b.asset("ccache_linux")
|
|
b.usesCCache()
|
|
b.cache(CACHES_WORKDIR...)
|
|
})
|
|
}
|
|
|
|
// checkGnToBp verifies that the gn_to_bp.py script continues to work.
|
|
func (b *jobBuilder) checkGnToBp() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.cas(CAS_COMPILE)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.cmd("./run_gn_to_bp",
|
|
"--local=false",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--alsologtostderr")
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.usesPython()
|
|
b.serviceAccount(b.cfg.ServiceAccountHousekeeper)
|
|
})
|
|
}
|
|
|
|
// housekeeper generates a Housekeeper task.
|
|
func (b *jobBuilder) housekeeper() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTask("housekeeper", OUTPUT_NONE)
|
|
b.serviceAccount(b.cfg.ServiceAccountHousekeeper)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.usesGit()
|
|
b.cache(CACHES_WORKDIR...)
|
|
})
|
|
}
|
|
|
|
// g3FrameworkCanary generates a G3 Framework Canary task. Returns
|
|
// the name of the last task in the generated chain of tasks, which the Job
|
|
// should add as a dependency.
|
|
func (b *jobBuilder) g3FrameworkCanary() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.cas(CAS_EMPTY)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.cmd("./g3_canary",
|
|
"--local=false",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--alsologtostderr")
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.serviceAccount("skia-g3-framework-compile@skia-swarming-bots.iam.gserviceaccount.com")
|
|
b.timeout(3 * time.Hour)
|
|
b.attempts(1)
|
|
})
|
|
}
|
|
|
|
// infra generates an infra_tests task.
|
|
func (b *jobBuilder) infra() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
if b.matchOs("Win") || b.matchExtraConfig("Win") {
|
|
b.dimension(
|
|
// Specify CPU to avoid running builds on bots with a more unique CPU.
|
|
"cpu:x86-64-Haswell_GCE",
|
|
"gpu:none",
|
|
fmt.Sprintf("machine_type:%s", MACHINE_TYPE_MEDIUM), // We don't have any small Windows instances.
|
|
fmt.Sprintf("os:%s", DEFAULT_OS_WIN),
|
|
fmt.Sprintf("pool:%s", b.cfg.Pool),
|
|
)
|
|
} else {
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
}
|
|
b.recipeProp("repository", specs.PLACEHOLDER_REPO)
|
|
b.kitchenTask("infra", OUTPUT_NONE)
|
|
b.cas(CAS_WHOLE_REPO)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
b.idempotent()
|
|
b.usesGo()
|
|
})
|
|
}
|
|
|
|
// buildstats generates a builtstats task, which compiles code and generates
|
|
// statistics about the build.
|
|
func (b *jobBuilder) buildstats() {
|
|
compileTaskName := b.compile()
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTask("compute_buildstats", OUTPUT_PERF)
|
|
b.dep(compileTaskName)
|
|
b.asset("bloaty")
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.usesDocker()
|
|
b.usesGit()
|
|
b.cache(CACHES_WORKDIR...)
|
|
})
|
|
// Upload release results (for tracking in perf)
|
|
// We have some jobs that are FYI (e.g. Debug-CanvasKit, tree-map generator)
|
|
if b.release() && !b.arch("x86_64") {
|
|
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, b.jobNameSchema.Sep, b.Name)
|
|
depName := b.Name
|
|
b.addTask(uploadName, func(b *taskBuilder) {
|
|
b.recipeProp("gs_bucket", b.cfg.GsBucketNano)
|
|
b.recipeProps(EXTRA_PROPS)
|
|
// TODO(borenet): I'm not sure why the upload task is
|
|
// using the BuildStats task name, but I've done this
|
|
// to maintain existing behavior.
|
|
b.Name = depName
|
|
b.kitchenTask("upload_buildstats_results", OUTPUT_NONE)
|
|
b.Name = uploadName
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadNano)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
b.dep(depName)
|
|
})
|
|
}
|
|
}
|
|
|
|
// doUpload indicates whether the given Job should upload its results.
|
|
func (b *jobBuilder) doUpload() bool {
|
|
for _, s := range b.cfg.NoUpload {
|
|
m, err := regexp.MatchString(s, b.Name)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if m {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// commonTestPerfAssets adds the assets needed by Test and Perf tasks.
|
|
func (b *taskBuilder) commonTestPerfAssets() {
|
|
// Docker-based tests don't need the standard CIPD assets
|
|
if b.extraConfig("CanvasKit", "PathKit") || (b.role("Test") && b.extraConfig("LottieWeb")) {
|
|
return
|
|
}
|
|
if b.extraConfig("Skpbench") {
|
|
// Skpbench only needs skps
|
|
b.asset("skp", "mskp")
|
|
} else if b.os("Android", "ChromeOS", "iOS") {
|
|
b.asset("skp", "svg", "skimage")
|
|
} else {
|
|
// for desktop machines
|
|
b.asset("skimage", "skp", "svg")
|
|
}
|
|
|
|
if b.isLinux() && b.matchExtraConfig("SAN") {
|
|
b.asset("clang_linux")
|
|
}
|
|
|
|
if b.isLinux() {
|
|
if b.extraConfig("Vulkan") {
|
|
b.asset("linux_vulkan_sdk")
|
|
}
|
|
if b.matchGpu("Intel") {
|
|
b.asset("mesa_intel_driver_linux")
|
|
}
|
|
}
|
|
if b.matchOs("Win") && b.extraConfig("ProcDump") {
|
|
b.asset("procdump_win")
|
|
}
|
|
}
|
|
|
|
// directUpload adds prerequisites for uploading to GCS.
|
|
func (b *taskBuilder) directUpload(gsBucket, serviceAccount string) {
|
|
b.recipeProp("gs_bucket", gsBucket)
|
|
b.serviceAccount(serviceAccount)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
}
|
|
|
|
// dm generates a Test task using dm.
|
|
func (b *jobBuilder) dm() {
|
|
compileTaskName := ""
|
|
// LottieWeb doesn't require anything in Skia to be compiled.
|
|
if !b.extraConfig("LottieWeb") {
|
|
compileTaskName = b.compile()
|
|
}
|
|
directUpload := false
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
cas := CAS_TEST
|
|
recipe := "test"
|
|
if b.extraConfig("PathKit") {
|
|
cas = CAS_PATHKIT
|
|
recipe = "test_pathkit"
|
|
if b.doUpload() {
|
|
b.directUpload(b.cfg.GsBucketGm, b.cfg.ServiceAccountUploadGM)
|
|
directUpload = true
|
|
}
|
|
} else if b.extraConfig("CanvasKit") {
|
|
cas = CAS_CANVASKIT
|
|
recipe = "test_canvaskit"
|
|
if b.doUpload() {
|
|
b.directUpload(b.cfg.GsBucketGm, b.cfg.ServiceAccountUploadGM)
|
|
directUpload = true
|
|
}
|
|
} else if b.extraConfig("LottieWeb") {
|
|
// CAS_LOTTIE_CI differs from CAS_LOTTIE_WEB in that it includes
|
|
// more of the files, especially those brought in via DEPS in the
|
|
// lottie-ci repo. The main difference between Perf.+LottieWeb and
|
|
// Test.+LottieWeb is that the former pulls in the lottie build via
|
|
// npm and the latter always tests at lottie's
|
|
// ToT.
|
|
cas = CAS_LOTTIE_CI
|
|
recipe = "test_lottie_web"
|
|
if b.doUpload() {
|
|
b.directUpload(b.cfg.GsBucketGm, b.cfg.ServiceAccountUploadGM)
|
|
directUpload = true
|
|
}
|
|
} else {
|
|
// Default recipe supports direct upload.
|
|
// TODO(http://skbug.com/11785): Windows jobs are unable to extract gsutil.
|
|
// https://bugs.chromium.org/p/chromium/issues/detail?id=1192611
|
|
if b.doUpload() && !b.matchOs("Win") {
|
|
b.directUpload(b.cfg.GsBucketGm, b.cfg.ServiceAccountUploadGM)
|
|
directUpload = true
|
|
}
|
|
}
|
|
b.recipeProp("gold_hashes_url", b.cfg.GoldHashesURL)
|
|
b.recipeProps(EXTRA_PROPS)
|
|
iid := b.internalHardwareLabel()
|
|
iidStr := ""
|
|
if iid != nil {
|
|
iidStr = strconv.Itoa(*iid)
|
|
}
|
|
if recipe == "test" {
|
|
b.dmFlags(iidStr)
|
|
}
|
|
b.kitchenTask(recipe, OUTPUT_TEST)
|
|
b.cas(cas)
|
|
b.swarmDimensions()
|
|
if b.extraConfig("CanvasKit", "Docker", "LottieWeb", "PathKit") {
|
|
b.usesDocker()
|
|
}
|
|
if compileTaskName != "" {
|
|
b.dep(compileTaskName)
|
|
}
|
|
if b.os("Android") && b.extraConfig("ASAN") {
|
|
b.asset("android_ndk_linux")
|
|
}
|
|
b.commonTestPerfAssets()
|
|
if b.matchExtraConfig("Lottie") {
|
|
b.asset("lottie-samples")
|
|
}
|
|
b.expiration(20 * time.Hour)
|
|
|
|
b.timeout(4 * time.Hour)
|
|
if b.extraConfig("Valgrind") {
|
|
b.timeout(9 * time.Hour)
|
|
b.expiration(48 * time.Hour)
|
|
b.asset("valgrind")
|
|
// Since Valgrind runs on the same bots as the CQ, we restrict Valgrind to a subset of the bots
|
|
// to ensure there are always bots free for CQ tasks.
|
|
b.dimension("valgrind:1")
|
|
} else if b.extraConfig("MSAN") {
|
|
b.timeout(9 * time.Hour)
|
|
} else if b.arch("x86") && b.debug() {
|
|
// skia:6737
|
|
b.timeout(6 * time.Hour)
|
|
}
|
|
b.maybeAddIosDevImage()
|
|
})
|
|
|
|
// Upload results if necessary. TODO(kjlubick): If we do coverage analysis at the same
|
|
// time as normal tests (which would be nice), cfg.json needs to have Coverage removed.
|
|
if b.doUpload() && !directUpload {
|
|
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, b.jobNameSchema.Sep, b.Name)
|
|
depName := b.Name
|
|
b.addTask(uploadName, func(b *taskBuilder) {
|
|
b.recipeProp("gs_bucket", b.cfg.GsBucketGm)
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTask("upload_dm_results", OUTPUT_NONE)
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadGM)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
b.dep(depName)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (b *jobBuilder) fm() {
|
|
goos := "linux"
|
|
if strings.Contains(b.parts["os"], "Win") {
|
|
goos = "windows"
|
|
}
|
|
if strings.Contains(b.parts["os"], "Mac") {
|
|
goos = "darwin"
|
|
}
|
|
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.asset("skimage", "skp", "svg")
|
|
b.cas(CAS_TEST)
|
|
b.dep(b.buildTaskDrivers(goos, "amd64"), b.compile())
|
|
b.cmd("./fm_driver${EXECUTABLE_SUFFIX}",
|
|
"--local=false",
|
|
"--resources=skia/resources",
|
|
"--imgs=skimage",
|
|
"--skps=skp",
|
|
"--svgs=svg",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--bot", b.Name,
|
|
"--gold="+strconv.FormatBool(!b.matchExtraConfig("SAN")),
|
|
"build/fm${EXECUTABLE_SUFFIX}")
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadGM)
|
|
b.swarmDimensions()
|
|
b.attempts(1)
|
|
|
|
if b.isLinux() && b.matchExtraConfig("SAN") {
|
|
b.asset("clang_linux")
|
|
// Sanitizers may want to run llvm-symbolizer for readable stack traces.
|
|
b.addToPATH("clang_linux/bin")
|
|
|
|
// Point sanitizer builds at our prebuilt libc++ for this sanitizer.
|
|
if b.extraConfig("MSAN") {
|
|
// We'd see false positives in std::basic_string<char> if this weren't set.
|
|
b.env("LD_LIBRARY_PATH", "clang_linux/msan")
|
|
} else if b.extraConfig("TSAN") {
|
|
// Occasional false positives may crop up in the standard library without this.
|
|
b.env("LD_LIBRARY_PATH", "clang_linux/tsan")
|
|
} else {
|
|
// This isn't strictly required, but we usually get better sanitizer
|
|
// diagnostics from libc++ than the default OS-provided libstdc++.
|
|
b.env("LD_LIBRARY_PATH", "clang_linux/lib")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// canary generates a task that uses TaskDrivers to trigger canary manual rolls on autorollers.
|
|
// Canary-G3 does not use this path because it is very different from other autorollers.
|
|
func (b *jobBuilder) canary(rollerName string) {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.cas(CAS_EMPTY)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.cmd("./canary",
|
|
"--local=false",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--roller_name", rollerName,
|
|
"--repo", specs.PLACEHOLDER_REPO,
|
|
"--revision", specs.PLACEHOLDER_REVISION,
|
|
"--patch_issue", specs.PLACEHOLDER_ISSUE,
|
|
"--patch_set", specs.PLACEHOLDER_PATCHSET,
|
|
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
|
|
"--alsologtostderr")
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.serviceAccount(b.cfg.ServiceAccountCanary)
|
|
b.timeout(3 * time.Hour)
|
|
b.attempts(1)
|
|
})
|
|
}
|
|
|
|
// puppeteer generates a task that uses TaskDrivers combined with a node script and puppeteer to
|
|
// benchmark something using Chromium (e.g. CanvasKit, LottieWeb).
|
|
func (b *jobBuilder) puppeteer() {
|
|
compileTaskName := b.compile()
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.defaultSwarmDimensions()
|
|
b.usesNode()
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"), compileTaskName)
|
|
b.output(OUTPUT_PERF)
|
|
b.timeout(60 * time.Minute)
|
|
b.cas(CAS_PUPPETEER)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
|
|
webglversion := "2"
|
|
if b.extraConfig("WebGL1") {
|
|
webglversion = "1"
|
|
}
|
|
|
|
if b.extraConfig("SkottieFrames") {
|
|
b.cmd(
|
|
"./perf_puppeteer_skottie_frames",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--git_hash", specs.PLACEHOLDER_REVISION,
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--canvaskit_bin_path", "./build",
|
|
"--lotties_path", "./lotties_with_assets",
|
|
"--node_bin_path", "./node/node/bin",
|
|
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
|
|
"--output_path", OUTPUT_PERF,
|
|
"--os_trace", b.parts["os"],
|
|
"--model_trace", b.parts["model"],
|
|
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
|
|
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
|
|
"--webgl_version", webglversion, // ignore when running with cpu backend
|
|
"--alsologtostderr",
|
|
)
|
|
// This CIPD package was made by hand with the following invocation:
|
|
// cipd create -name skia/internal/lotties_with_assets -in ./lotties/ -tag version:0
|
|
// cipd acl-edit skia/internal/lotties_with_assets -reader group:project-skia-external-task-accounts
|
|
// cipd acl-edit skia/internal/lotties_with_assets -reader user:pool-skia@chromium-swarm.iam.gserviceaccount.com
|
|
// Where lotties is a hand-selected set of lottie animations and (optionally) assets used in
|
|
// them (e.g. fonts, images).
|
|
b.cipd(&specs.CipdPackage{
|
|
Name: "skia/internal/lotties_with_assets",
|
|
Path: "lotties_with_assets",
|
|
Version: "version:1",
|
|
})
|
|
} else if b.extraConfig("RenderSKP") {
|
|
b.cmd(
|
|
"./perf_puppeteer_render_skps",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--git_hash", specs.PLACEHOLDER_REVISION,
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--canvaskit_bin_path", "./build",
|
|
"--skps_path", "./skp",
|
|
"--node_bin_path", "./node/node/bin",
|
|
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
|
|
"--output_path", OUTPUT_PERF,
|
|
"--os_trace", b.parts["os"],
|
|
"--model_trace", b.parts["model"],
|
|
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
|
|
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
|
|
"--webgl_version", webglversion,
|
|
"--alsologtostderr",
|
|
)
|
|
b.asset("skp")
|
|
} else if b.extraConfig("CanvasPerf") { // refers to the canvas_perf.js test suite
|
|
b.cmd(
|
|
"./perf_puppeteer_canvas",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--git_hash", specs.PLACEHOLDER_REVISION,
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--canvaskit_bin_path", "./build",
|
|
"--node_bin_path", "./node/node/bin",
|
|
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
|
|
"--output_path", OUTPUT_PERF,
|
|
"--os_trace", b.parts["os"],
|
|
"--model_trace", b.parts["model"],
|
|
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
|
|
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
|
|
"--webgl_version", webglversion,
|
|
"--alsologtostderr",
|
|
)
|
|
b.asset("skp")
|
|
}
|
|
|
|
})
|
|
|
|
// Upload results to Perf after.
|
|
// TODO(kjlubick,borenet) deduplicate this with the logic in perf().
|
|
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, b.jobNameSchema.Sep, b.Name)
|
|
depName := b.Name
|
|
b.addTask(uploadName, func(b *taskBuilder) {
|
|
b.recipeProp("gs_bucket", b.cfg.GsBucketNano)
|
|
b.recipeProps(EXTRA_PROPS)
|
|
// TODO(borenet): I'm not sure why the upload task is
|
|
// using the Perf task name, but I've done this to
|
|
// maintain existing behavior.
|
|
b.Name = depName
|
|
b.kitchenTask("upload_nano_results", OUTPUT_NONE)
|
|
b.Name = uploadName
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadNano)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
b.dep(depName)
|
|
})
|
|
}
|
|
|
|
func (b *jobBuilder) cifuzz() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.attempts(1)
|
|
b.usesDocker()
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.cipd(specs.CIPD_PKGS_GIT_LINUX_AMD64...)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.output("cifuzz_out")
|
|
b.timeout(60 * time.Minute)
|
|
b.cas(CAS_WHOLE_REPO)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.cmd(
|
|
"./cifuzz",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--git_exe_path", "./cipd_bin_packages/git",
|
|
"--out_path", "./cifuzz_out",
|
|
"--skia_path", "./skia",
|
|
"--work_path", "./cifuzz_work",
|
|
"--alsologtostderr",
|
|
)
|
|
})
|
|
}
|
|
|
|
// perf generates a Perf task.
|
|
func (b *jobBuilder) perf() {
|
|
compileTaskName := ""
|
|
// LottieWeb doesn't require anything in Skia to be compiled.
|
|
if !b.extraConfig("LottieWeb") {
|
|
compileTaskName = b.compile()
|
|
}
|
|
doUpload := b.release() && b.doUpload()
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
recipe := "perf"
|
|
cas := CAS_PERF
|
|
if b.extraConfig("Skpbench") {
|
|
recipe = "skpbench"
|
|
cas = CAS_SKPBENCH
|
|
} else if b.extraConfig("PathKit") {
|
|
cas = CAS_PATHKIT
|
|
recipe = "perf_pathkit"
|
|
} else if b.extraConfig("CanvasKit") {
|
|
cas = CAS_CANVASKIT
|
|
recipe = "perf_canvaskit"
|
|
} else if b.extraConfig("SkottieTracing") {
|
|
recipe = "perf_skottietrace"
|
|
} else if b.extraConfig("SkottieWASM") {
|
|
recipe = "perf_skottiewasm_lottieweb"
|
|
cas = CAS_SKOTTIE_WASM
|
|
} else if b.extraConfig("LottieWeb") {
|
|
recipe = "perf_skottiewasm_lottieweb"
|
|
cas = CAS_LOTTIE_WEB
|
|
}
|
|
b.recipeProps(EXTRA_PROPS)
|
|
if recipe == "perf" {
|
|
b.nanobenchFlags(doUpload)
|
|
} else if recipe == "skpbench" {
|
|
b.skpbenchFlags()
|
|
}
|
|
b.kitchenTask(recipe, OUTPUT_PERF)
|
|
b.cas(cas)
|
|
b.swarmDimensions()
|
|
if b.extraConfig("CanvasKit", "Docker", "PathKit") {
|
|
b.usesDocker()
|
|
}
|
|
if compileTaskName != "" {
|
|
b.dep(compileTaskName)
|
|
}
|
|
b.commonTestPerfAssets()
|
|
b.expiration(20 * time.Hour)
|
|
b.timeout(4 * time.Hour)
|
|
|
|
if b.extraConfig("Valgrind") {
|
|
b.timeout(9 * time.Hour)
|
|
b.expiration(48 * time.Hour)
|
|
b.asset("valgrind")
|
|
// Since Valgrind runs on the same bots as the CQ, we restrict Valgrind to a subset of the bots
|
|
// to ensure there are always bots free for CQ tasks.
|
|
b.dimension("valgrind:1")
|
|
} else if b.extraConfig("MSAN") {
|
|
b.timeout(9 * time.Hour)
|
|
} else if b.parts["arch"] == "x86" && b.parts["configuration"] == "Debug" {
|
|
// skia:6737
|
|
b.timeout(6 * time.Hour)
|
|
} else if b.extraConfig("LottieWeb", "SkottieWASM") {
|
|
b.asset("node", "lottie-samples")
|
|
} else if b.matchExtraConfig("Skottie") {
|
|
b.asset("lottie-samples")
|
|
}
|
|
|
|
if b.os("Android") && b.cpu() {
|
|
b.asset("text_blob_traces")
|
|
}
|
|
b.maybeAddIosDevImage()
|
|
|
|
iid := b.internalHardwareLabel()
|
|
if iid != nil {
|
|
b.Spec.Command = append(b.Spec.Command, fmt.Sprintf("internal_hardware_label=%d", *iid))
|
|
}
|
|
})
|
|
|
|
// Upload results if necessary.
|
|
if doUpload {
|
|
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, b.jobNameSchema.Sep, b.Name)
|
|
depName := b.Name
|
|
b.addTask(uploadName, func(b *taskBuilder) {
|
|
b.recipeProp("gs_bucket", b.cfg.GsBucketNano)
|
|
b.recipeProps(EXTRA_PROPS)
|
|
// TODO(borenet): I'm not sure why the upload task is
|
|
// using the Perf task name, but I've done this to
|
|
// maintain existing behavior.
|
|
b.Name = depName
|
|
b.kitchenTask("upload_nano_results", OUTPUT_NONE)
|
|
b.Name = uploadName
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadNano)
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.cipd(specs.CIPD_PKGS_GSUTIL...)
|
|
b.dep(depName)
|
|
})
|
|
}
|
|
}
|
|
|
|
// presubmit generates a task which runs the presubmit for this repo.
|
|
func (b *jobBuilder) presubmit() {
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.recipeProps(map[string]string{
|
|
"category": "cq",
|
|
"patch_gerrit_url": "https://skia-review.googlesource.com",
|
|
"patch_project": "skia",
|
|
"patch_ref": specs.PLACEHOLDER_PATCH_REF,
|
|
"reason": "CQ",
|
|
"repo_name": "skia",
|
|
})
|
|
b.recipeProps(EXTRA_PROPS)
|
|
b.kitchenTaskNoBundle("run_presubmit", OUTPUT_NONE)
|
|
b.cas(CAS_RUN_RECIPE)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
// Use MACHINE_TYPE_LARGE because it seems to save time versus
|
|
// MEDIUM and we want presubmit to be fast.
|
|
b.linuxGceDimensions(MACHINE_TYPE_LARGE)
|
|
b.usesGit()
|
|
b.cipd(&specs.CipdPackage{
|
|
Name: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build",
|
|
Path: "recipe_bundle",
|
|
Version: "git_revision:a8bcedad6768e206c4d2bd1718caa849f29cd42d",
|
|
})
|
|
})
|
|
}
|
|
|
|
// compileWasmGMTests uses a task driver to compile the GMs and unit tests for Web Assembly (WASM).
|
|
// We can use the same build for both CPU and GPU tests since the latter requires the code for the
|
|
// former anyway.
|
|
func (b *jobBuilder) compileWasmGMTests(compileName string) {
|
|
b.addTask(compileName, func(b *taskBuilder) {
|
|
b.attempts(1)
|
|
b.usesDocker()
|
|
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.output("wasm_out")
|
|
b.timeout(60 * time.Minute)
|
|
b.cas(CAS_COMPILE)
|
|
b.serviceAccount(b.cfg.ServiceAccountCompile)
|
|
b.cache(CACHES_DOCKER...)
|
|
// For now, we only have one compile mode - a GPU release mode. This should be sufficient to
|
|
// run CPU, WebGL1, and WebGL2 tests. Debug mode is not needed for the waterfall because
|
|
// when using puppeteer, stacktraces from exceptions are hard to get access to, so we do not
|
|
// even bother.
|
|
b.cmd(
|
|
"./compile_wasm_gm_tests",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", compileName,
|
|
"--out_path", "./wasm_out",
|
|
"--skia_path", "./skia",
|
|
"--work_path", "./cache/docker/wasm_gm",
|
|
"--alsologtostderr",
|
|
)
|
|
})
|
|
}
|
|
|
|
// compileWasmGMTests uses a task driver to compile the GMs and unit tests for Web Assembly (WASM).
|
|
// We can use the same build for both CPU and GPU tests since the latter requires the code for the
|
|
// former anyway.
|
|
func (b *jobBuilder) runWasmGMTests() {
|
|
compileTaskName := b.compile()
|
|
|
|
b.addTask(b.Name, func(b *taskBuilder) {
|
|
b.attempts(1)
|
|
b.usesNode()
|
|
b.swarmDimensions()
|
|
b.cipd(CIPD_PKG_LUCI_AUTH)
|
|
b.cipd(CIPD_PKGS_GOLDCTL...)
|
|
b.dep(b.buildTaskDrivers("linux", "amd64"))
|
|
b.dep(compileTaskName)
|
|
b.timeout(60 * time.Minute)
|
|
b.cas(CAS_WASM_GM)
|
|
b.serviceAccount(b.cfg.ServiceAccountUploadGM)
|
|
b.cmd(
|
|
"./run_wasm_gm_tests",
|
|
"--project_id", "skia-swarming-bots",
|
|
"--task_id", specs.PLACEHOLDER_TASK_ID,
|
|
"--task_name", b.Name,
|
|
"--test_harness_path", "./tools/run-wasm-gm-tests",
|
|
"--built_path", "./wasm_out",
|
|
"--node_bin_path", "./node/node/bin",
|
|
"--resource_path", "./resources",
|
|
"--work_path", "./wasm_gm/work",
|
|
"--gold_ctl_path", "./cipd_bin_packages/goldctl",
|
|
"--git_commit", specs.PLACEHOLDER_REVISION,
|
|
"--changelist_id", specs.PLACEHOLDER_ISSUE,
|
|
"--patchset_order", specs.PLACEHOLDER_PATCHSET,
|
|
"--tryjob_id", specs.PLACEHOLDER_BUILDBUCKET_BUILD_ID,
|
|
// TODO(kjlubick, nifong) Make these not hard coded if we change the configs we test on.
|
|
"--webgl_version", "2", // 0 means CPU ; this flag controls cpu_or_gpu and extra_config
|
|
"--gold_key", "alpha_type:Premul",
|
|
"--gold_key", "arch:wasm",
|
|
"--gold_key", "browser:Chrome",
|
|
"--gold_key", "color_depth:8888",
|
|
"--gold_key", "config:gles",
|
|
"--gold_key", "configuration:Release",
|
|
"--gold_key", "cpu_or_gpu_value:QuadroP400",
|
|
"--gold_key", "model:Golo",
|
|
"--gold_key", "os:Ubuntu18",
|
|
"--alsologtostderr",
|
|
)
|
|
})
|
|
}
|