b7eaeadf3f
This CL adds the task driver and plumbing necessary to run CI tasks that upload build statistics to https://codesize.skia.org. For now, a single CodeSize-dm-Debian10-Clang-x86_64-Release task is defined, which as the name suggests: - Depends on Build-Debian10-Clang-x86_64-Release. - Takes the "dm" binary built by the aforementioned job. - Runs Bloaty against said binary. - Uploads the Bloaty output, along with a JSON file with metadata, to the GCS bucket where https://codesize.skia.org gets its data from. If one wishes to upload codesize statistics for another binary, the steps are as follows: - Identify the target binary, for example "fm". - Identify a compile task that builds said binary with the desired settings, for example "Build-Debian10-Clang-x86_64-Debug". - Add a new job in //infra/bots/jobs.json named "CodeSize-<binary>-<compile task>", where <compile task> is the name of the compile task without the "Build-" prefix, for example "CodeSize-fm-Debian10-Clang-x86_64-Debug". - Run "make train" from the //infra/bots directory. Eventually the codesize.skia.org webserver will automatically pick up the output of any such CodeSize-* task and show it in the UI, with no additional steps needed. One caveat is that the binary file name, and therefore the "<binary>" part of the task name, cannot contain dashes. This is due to how //infra/bots/gen_tasks_logic.go works. See comments in said Go file for ideas on how to work around this if we ever need to. Bug: skia:12151 Change-Id: If406944ca7660c4dd15c8e6b8f34e48c65cbbe2f Reviewed-on: https://skia-review.googlesource.com/c/skia/+/502788 Reviewed-by: Kevin Lubick <kjlubick@google.com> Reviewed-by: Eric Boren <borenet@google.com> Commit-Queue: Leandro Lovisolo <lovisolo@google.com>
240 lines
5.6 KiB
Go
240 lines
5.6 KiB
Go
// Copyright 2020 The Chromium Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
package gen_tasks_logic
|
|
|
|
import (
|
|
"log"
|
|
"strings"
|
|
|
|
"go.skia.org/infra/task_scheduler/go/specs"
|
|
)
|
|
|
|
// jobBuilder provides helpers for creating a job.
|
|
type jobBuilder struct {
|
|
*builder
|
|
parts
|
|
Name string
|
|
Spec *specs.JobSpec
|
|
}
|
|
|
|
// newJobBuilder returns a jobBuilder for the given job name.
|
|
func newJobBuilder(b *builder, name string) *jobBuilder {
|
|
p, err := b.jobNameSchema.ParseJobName(name)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return &jobBuilder{
|
|
builder: b,
|
|
parts: p,
|
|
Name: name,
|
|
Spec: &specs.JobSpec{},
|
|
}
|
|
}
|
|
|
|
// priority sets the priority of the job.
|
|
func (b *jobBuilder) priority(p float64) {
|
|
b.Spec.Priority = p
|
|
}
|
|
|
|
// trigger dictates when the job should be triggered.
|
|
func (b *jobBuilder) trigger(trigger string) {
|
|
b.Spec.Trigger = trigger
|
|
}
|
|
|
|
// Create a taskBuilder and run the given function for it.
|
|
func (b *jobBuilder) addTask(name string, fn func(*taskBuilder)) {
|
|
tb := newTaskBuilder(b, name)
|
|
fn(tb)
|
|
b.MustAddTask(tb.Name, tb.Spec)
|
|
// Add the task to the Job's dependency set, removing any which are
|
|
// accounted for by the new task's dependencies.
|
|
b.Spec.TaskSpecs = append(b.Spec.TaskSpecs, tb.Name)
|
|
newSpecs := make([]string, 0, len(b.Spec.TaskSpecs))
|
|
for _, t := range b.Spec.TaskSpecs {
|
|
if !In(t, tb.Spec.Dependencies) {
|
|
newSpecs = append(newSpecs, t)
|
|
}
|
|
}
|
|
b.Spec.TaskSpecs = newSpecs
|
|
}
|
|
|
|
// uploadCIPDAssetToCAS generates a task to isolate the given CIPD asset. Returns
|
|
// the name of the task.
|
|
func (b *jobBuilder) uploadCIPDAssetToCAS(asset string) string {
|
|
cfg, ok := ISOLATE_ASSET_MAPPING[asset]
|
|
if !ok {
|
|
log.Fatalf("No isolate task for asset %q", asset)
|
|
}
|
|
b.addTask(cfg.uploadTaskName, func(b *taskBuilder) {
|
|
b.cipd(b.MustGetCipdPackageFromAsset(asset))
|
|
b.cmd("/bin/cp", "-rL", cfg.path, "${ISOLATED_OUTDIR}")
|
|
b.linuxGceDimensions(MACHINE_TYPE_SMALL)
|
|
b.idempotent()
|
|
b.cas(CAS_EMPTY)
|
|
})
|
|
return cfg.uploadTaskName
|
|
}
|
|
|
|
// genTasksForJob generates the tasks needed by this job.
|
|
func (b *jobBuilder) genTasksForJob() {
|
|
// Bundle Recipes.
|
|
if b.Name == BUNDLE_RECIPES_NAME {
|
|
b.bundleRecipes()
|
|
return
|
|
}
|
|
if strings.HasPrefix(b.Name, BUILD_TASK_DRIVERS_PREFIX) {
|
|
parts := strings.Split(b.Name, "_")
|
|
b.buildTaskDrivers(parts[1], parts[2])
|
|
return
|
|
}
|
|
|
|
// Isolate CIPD assets.
|
|
if b.matchExtraConfig("Isolate") {
|
|
for asset, cfg := range ISOLATE_ASSET_MAPPING {
|
|
if cfg.uploadTaskName == b.Name {
|
|
b.uploadCIPDAssetToCAS(asset)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// RecreateSKPs.
|
|
if b.extraConfig("RecreateSKPs") {
|
|
b.recreateSKPs()
|
|
return
|
|
}
|
|
|
|
// Update Go Dependencies.
|
|
if b.extraConfig("UpdateGoDeps") {
|
|
b.updateGoDeps()
|
|
return
|
|
}
|
|
|
|
// Create docker image.
|
|
if b.extraConfig("CreateDockerImage") {
|
|
b.createDockerImage(b.extraConfig("WASM"))
|
|
return
|
|
}
|
|
|
|
// Push apps from docker image.
|
|
if b.extraConfig("PushAppsFromSkiaDockerImage") {
|
|
b.createPushAppsFromSkiaDockerImage()
|
|
return
|
|
} else if b.extraConfig("PushBazelAppsFromWASMDockerImage") {
|
|
b.createPushBazelAppsFromWASMDockerImage()
|
|
return
|
|
}
|
|
|
|
// Infra tests.
|
|
if b.extraConfig("InfraTests") {
|
|
b.infra()
|
|
return
|
|
}
|
|
|
|
// Housekeepers.
|
|
if b.Name == "Housekeeper-PerCommit" {
|
|
b.housekeeper()
|
|
return
|
|
}
|
|
if b.Name == "Housekeeper-PerCommit-CheckGeneratedFiles" {
|
|
b.checkGeneratedFiles()
|
|
return
|
|
}
|
|
if b.Name == "Housekeeper-PerCommit-RunGnToBp" {
|
|
b.checkGnToBp()
|
|
return
|
|
}
|
|
if b.Name == "Housekeeper-OnDemand-Presubmit" {
|
|
b.priority(1)
|
|
b.presubmit()
|
|
return
|
|
}
|
|
|
|
// Compile bots.
|
|
if b.role("Build") {
|
|
b.compile()
|
|
return
|
|
}
|
|
|
|
// BuildStats bots. This computes things like binary size.
|
|
if b.role("BuildStats") {
|
|
b.buildstats()
|
|
return
|
|
}
|
|
|
|
if b.role("CodeSize") {
|
|
b.codesize()
|
|
return
|
|
}
|
|
|
|
// Valgrind runs at a low priority so that it doesn't occupy all the bots.
|
|
if b.extraConfig("Valgrind") {
|
|
// Priority of 0.085 should result in Valgrind tasks with a blamelist of ~10 commits having the
|
|
// same score as other tasks with a blamelist of 1 commit, when we have insufficient bot
|
|
// capacity to run more frequently.
|
|
b.priority(0.085)
|
|
}
|
|
|
|
// Test bots.
|
|
if b.role("Test") {
|
|
if b.extraConfig("WasmGMTests") {
|
|
b.runWasmGMTests()
|
|
return
|
|
}
|
|
b.dm()
|
|
return
|
|
}
|
|
if b.role("FM") {
|
|
b.fm()
|
|
return
|
|
}
|
|
|
|
// Canary bots.
|
|
if b.role("Canary") {
|
|
if b.project("G3") {
|
|
b.g3FrameworkCanary()
|
|
return
|
|
} else if b.project("Android") {
|
|
b.canary("android-master-autoroll", "Canary-Android-Topic", "https://googleplex-android-review.googlesource.com/q/topic:")
|
|
return
|
|
} else if b.project("Chromium") {
|
|
b.canary("skia-autoroll", "Canary-Chromium-CL", "https://chromium-review.googlesource.com/c/")
|
|
return
|
|
} else if b.project("Flutter") {
|
|
b.canary("skia-flutter-autoroll", "Canary-Flutter-PR", "https://github.com/flutter/engine/pull/")
|
|
return
|
|
}
|
|
}
|
|
|
|
if b.extraConfig("Puppeteer") {
|
|
// TODO(kjlubick) make this a new role
|
|
b.puppeteer()
|
|
return
|
|
}
|
|
|
|
// Perf bots.
|
|
if b.role("Perf") {
|
|
b.perf()
|
|
return
|
|
}
|
|
|
|
log.Fatalf("Don't know how to handle job %q", b.Name)
|
|
}
|
|
|
|
func (b *jobBuilder) finish() {
|
|
// Add the Job spec.
|
|
if b.frequency("Nightly") {
|
|
b.trigger(specs.TRIGGER_NIGHTLY)
|
|
} else if b.frequency("Weekly") {
|
|
b.trigger(specs.TRIGGER_WEEKLY)
|
|
} else if b.extraConfig("Flutter", "CommandBuffer", "CreateDockerImage", "PushAppsFromSkiaDockerImage", "PushBazelAppsFromWASMDockerImage") {
|
|
b.trigger(specs.TRIGGER_MAIN_ONLY)
|
|
} else if b.frequency("OnDemand") || b.role("Canary") {
|
|
b.trigger(specs.TRIGGER_ON_DEMAND)
|
|
} else {
|
|
b.trigger(specs.TRIGGER_ANY_BRANCH)
|
|
}
|
|
b.MustAddJob(b.Name, b.Spec)
|
|
}
|