skia2/infra/bots/gen_tasks_logic/task_builder.go

307 lines
8.4 KiB
Go
Raw Normal View History

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package gen_tasks_logic
import (
"log"
"reflect"
"time"
"go.skia.org/infra/go/cipd"
"go.skia.org/infra/task_scheduler/go/specs"
)
// taskBuilder is a helper for creating a task.
type taskBuilder struct {
*jobBuilder
parts
Name string
Spec *specs.TaskSpec
recipeProperties map[string]string
}
// newTaskBuilder returns a taskBuilder instance.
func newTaskBuilder(b *jobBuilder, name string) *taskBuilder {
parts, err := b.jobNameSchema.ParseJobName(name)
if err != nil {
log.Fatal(err)
}
return &taskBuilder{
jobBuilder: b,
parts: parts,
Name: name,
Spec: &specs.TaskSpec{},
recipeProperties: map[string]string{},
}
}
// attempts sets the desired MaxAttempts for this task.
func (b *taskBuilder) attempts(a int) {
b.Spec.MaxAttempts = a
}
// cache adds the given caches to the task.
func (b *taskBuilder) cache(caches ...*specs.Cache) {
for _, c := range caches {
alreadyHave := false
for _, exist := range b.Spec.Caches {
if c.Name == exist.Name {
if !reflect.DeepEqual(c, exist) {
log.Fatalf("Already have cache %s with a different definition!", c.Name)
}
alreadyHave = true
break
}
}
if !alreadyHave {
b.Spec.Caches = append(b.Spec.Caches, c)
}
}
}
// cmd sets the command for the task.
func (b *taskBuilder) cmd(c ...string) {
b.Spec.Command = c
}
// dimension adds the given dimensions to the task.
func (b *taskBuilder) dimension(dims ...string) {
for _, dim := range dims {
if !In(dim, b.Spec.Dimensions) {
b.Spec.Dimensions = append(b.Spec.Dimensions, dim)
}
}
}
// expiration sets the expiration of the task.
func (b *taskBuilder) expiration(e time.Duration) {
b.Spec.Expiration = e
}
// idempotent marks the task as idempotent.
func (b *taskBuilder) idempotent() {
b.Spec.Idempotent = true
}
// isolate sets the isolate file used by the task.
func (b *taskBuilder) isolate(i string) {
b.Spec.Isolate = b.relpath(i)
}
// env appends the given values to the given environment variable for the task.
func (b *taskBuilder) env(key string, values ...string) {
if b.Spec.EnvPrefixes == nil {
b.Spec.EnvPrefixes = map[string][]string{}
}
for _, value := range values {
if !In(value, b.Spec.EnvPrefixes[key]) {
b.Spec.EnvPrefixes[key] = append(b.Spec.EnvPrefixes[key], value)
}
}
}
// addToPATH adds the given locations to PATH for the task.
func (b *taskBuilder) addToPATH(loc ...string) {
b.env("PATH", loc...)
}
// output adds the given paths as outputs to the task, which results in their
// contents being uploaded to the isolate server.
func (b *taskBuilder) output(paths ...string) {
for _, path := range paths {
if !In(path, b.Spec.Outputs) {
b.Spec.Outputs = append(b.Spec.Outputs, path)
}
}
}
// serviceAccount sets the service account for this task.
func (b *taskBuilder) serviceAccount(sa string) {
b.Spec.ServiceAccount = sa
}
// timeout sets the timeout(s) for this task.
func (b *taskBuilder) timeout(timeout time.Duration) {
b.Spec.ExecutionTimeout = timeout
b.Spec.IoTimeout = timeout // With kitchen, step logs don't count toward IoTimeout.
}
// dep adds the given tasks as dependencies of this task.
func (b *taskBuilder) dep(tasks ...string) {
for _, task := range tasks {
if !In(task, b.Spec.Dependencies) {
b.Spec.Dependencies = append(b.Spec.Dependencies, task)
}
}
}
// cipd adds the given CIPD packages to the task.
func (b *taskBuilder) cipd(pkgs ...*specs.CipdPackage) {
for _, pkg := range pkgs {
alreadyHave := false
for _, exist := range b.Spec.CipdPackages {
if pkg.Name == exist.Name {
if !reflect.DeepEqual(pkg, exist) {
log.Fatalf("Already have package %s with a different definition!", pkg.Name)
}
alreadyHave = true
break
}
}
if !alreadyHave {
b.Spec.CipdPackages = append(b.Spec.CipdPackages, pkg)
}
}
}
// useIsolatedAssets returns true if this task should use assets which are
// isolated rather than downloading directly from CIPD.
func (b *taskBuilder) useIsolatedAssets() bool {
// Only do this on the RPIs for now. Other, faster machines shouldn't
// see much benefit and we don't need the extra complexity, for now.
if b.os("Android", "ChromeOS", "iOS") {
return true
}
return false
}
// isolateAssetConfig represents a task which copies a CIPD package into
// isolate.
type isolateAssetCfg struct {
alwaysIsolate bool
isolateTaskName string
path string
}
// asset adds the given assets to the task as CIPD packages.
func (b *taskBuilder) asset(assets ...string) {
shouldIsolate := b.useIsolatedAssets()
pkgs := make([]*specs.CipdPackage, 0, len(assets))
for _, asset := range assets {
if cfg, ok := ISOLATE_ASSET_MAPPING[asset]; ok && (cfg.alwaysIsolate || shouldIsolate) {
b.dep(b.isolateCIPDAsset(asset))
} else {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset(asset))
}
}
b.cipd(pkgs...)
}
// usesCCache adds attributes to tasks which use ccache.
func (b *taskBuilder) usesCCache() {
b.cache(CACHES_CCACHE...)
}
// usesGit adds attributes to tasks which use git.
func (b *taskBuilder) usesGit() {
b.cache(CACHES_GIT...)
if b.matchOs("Win") || b.matchExtraConfig("Win") {
b.cipd(specs.CIPD_PKGS_GIT_WINDOWS_AMD64...)
} else if b.matchOs("Mac") || b.matchExtraConfig("Mac") {
b.cipd(specs.CIPD_PKGS_GIT_MAC_AMD64...)
} else {
b.cipd(specs.CIPD_PKGS_GIT_LINUX_AMD64...)
}
}
// usesGo adds attributes to tasks which use go. Recipes should use
// "with api.context(env=api.infra.go_env)".
func (b *taskBuilder) usesGo() {
b.usesGit() // Go requires Git.
b.cache(CACHES_GO...)
pkg := b.MustGetCipdPackageFromAsset("go")
if b.matchOs("Win") || b.matchExtraConfig("Win") {
pkg = b.MustGetCipdPackageFromAsset("go_win")
pkg.Path = "go"
}
b.cipd(pkg)
}
// usesDocker adds attributes to tasks which use docker.
func (b *taskBuilder) usesDocker() {
b.dimension("docker_installed:true")
}
// recipeProp adds the given recipe property key/value pair. Panics if
// getRecipeProps() was already called.
func (b *taskBuilder) recipeProp(key, value string) {
if b.recipeProperties == nil {
log.Fatal("taskBuilder.recipeProp() cannot be called after taskBuilder.getRecipeProps()!")
}
b.recipeProperties[key] = value
}
// recipeProps calls recipeProp for every key/value pair in the given map.
// Panics if getRecipeProps() was already called.
func (b *taskBuilder) recipeProps(props map[string]string) {
for k, v := range props {
b.recipeProp(k, v)
}
}
// getRecipeProps returns JSON-encoded recipe properties. Subsequent calls to
// recipeProp[s] will panic, to prevent accidentally adding recipe properties
// after they have been added to the task.
func (b *taskBuilder) getRecipeProps() string {
props := make(map[string]interface{}, len(b.recipeProperties)+2)
props["buildername"] = b.Name
props["$kitchen"] = struct {
DevShell bool `json:"devshell"`
GitAuth bool `json:"git_auth"`
}{
DevShell: true,
GitAuth: true,
}
for k, v := range b.recipeProperties {
props[k] = v
}
b.recipeProperties = nil
return marshalJson(props)
}
// cipdPlatform returns the CIPD platform for this task.
func (b *taskBuilder) cipdPlatform() string {
if b.role("Upload") {
return cipd.PlatformLinuxAmd64
} else if b.matchOs("Win") || b.matchExtraConfig("Win") {
if b.matchArch("x86_64") {
return cipd.PlatformWindowsAmd64
} else {
return cipd.PlatformWindows386
}
} else if b.matchOs("Mac") {
return cipd.PlatformMacAmd64
} else if b.matchArch("Arm64") {
return cipd.PlatformLinuxArm64
} else {
return cipd.PlatformLinuxAmd64
}
}
// usesPython adds attributes to tasks which use python.
func (b *taskBuilder) usesPython() {
// TODO(borenet): This handling of the Python package is hacky and bad.
pythonPkgs := cipd.PkgsPython[b.cipdPlatform()]
b.cipd(pythonPkgs[1])
if b.os("Mac10.15") && b.model("VMware7.1") {
b.cipd(pythonPkgs[0])
}
if (b.matchOs("Win") || b.matchExtraConfig("Win")) && !b.model("LenovoYogaC630") {
b.cipd(pythonPkgs[0])
}
b.cache(&specs.Cache{
Name: "vpython",
Path: "cache/vpython",
})
b.env("VPYTHON_VIRTUALENV_ROOT", "cache/vpython")
}
[canvaskit] Start a generic puppeteer perfing system. IMPORTANT LESSON: when bringing in node (and possibly other executables) via CIPD, add them to the path in gen_tasks_logic so the parent executable (the task driver itself) has the right PATH set. Otherwise, the subprocesses it spawns might grab the wrong version because of how golang handles environments of subprocesses. This is starting as a fork of Skottie WASM. I hope to have a more unified system for creating and running benchmarks. Overall overview: gen_tasks_logic.go creates a task in task.json that compiles CanvasKit and the task drivers and then executes our task (i.e. perf_puppeteer.go) perf_puppeteer runs a node program (perf-with-puppeteer.js) that uses puppeteer to execute benchmarking code on an html page (canvaskit-skottie-frames-load.html). I needed to update the node package so npm could be updated from 3.x to 6.14.4 so it knew about `npm ci`. This may not have been entirely necessary, given the problems of executing the correct npm (see important lesson above), but it hasn't broken things further, so more up-to-date is probably a good thing. Suggested Review Order: - canvaskit-skottie-frames-load.html (note it is similar to skottie-wasm-perf.html, but it waits for a button click to start animating and records times from the main JS thread itself) - perf-with-puppeteer.js (similar to skottie-wasm-perf.js, but has some things made optional [e.g. tracing]) - perf_puppeteer_test.go (shows the inputs/outputs of various steps) - perf_puppeteer.go - Everything else. Change-Id: I380e81b825f36682c257664d488267edaf36369e Reviewed-on: https://skia-review.googlesource.com/c/skia/+/285783 Commit-Queue: Kevin Lubick <kjlubick@google.com> Reviewed-by: Eric Boren <borenet@google.com>
2020-05-01 18:16:27 +00:00
func (b *taskBuilder) usesNode() {
// It is very important when including node via CIPD to also add it to the PATH of the
// taskdriver or mysterious things can happen when subprocesses try to resolve node/npm.
b.asset("node")
b.addToPATH("node/node/bin")
}