skia2/infra/bots/gen_tasks.go

606 lines
16 KiB
Go
Raw Normal View History

// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
/*
Generate the tasks.json file.
*/
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"sort"
"strings"
"github.com/skia-dev/glog"
"go.skia.org/infra/go/common"
"go.skia.org/infra/go/util"
"go.skia.org/infra/task_scheduler/go/specs"
)
const (
DEFAULT_OS = "Ubuntu"
// Pool for Skia bots.
POOL_SKIA = "Skia"
// Name prefix for upload jobs.
PREFIX_UPLOAD = "Upload"
)
var (
// "Constants"
// Top-level list of all jobs to run at each commit.
JOBS = []string{
"Build-Ubuntu-GCC-x86_64-Release-GN",
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
}
// UPLOAD_DIMENSIONS are the Swarming dimensions for upload tasks.
UPLOAD_DIMENSIONS = []string{
"cpu:x86-64-avx2",
"gpu:none",
"os:Ubuntu",
fmt.Sprintf("pool:%s", POOL_SKIA),
}
// Defines the structure of job names.
jobNameSchema *JobNameSchema
// Caches CIPD package info so that we don't have to re-read VERSION
// files.
cipdPackages = map[string]*specs.CipdPackage{}
// Path to the infra/bots directory.
infrabotsDir = ""
// Flags.
testing = flag.Bool("test", false, "Run in test mode: verify that the output hasn't changed.")
)
// deriveCompileTaskName returns the name of a compile task based on the given
// job name.
func deriveCompileTaskName(jobName string, parts map[string]string) string {
if parts["role"] == "Housekeeper" {
return "Build-Ubuntu-GCC-x86_64-Release-Shared"
} else if parts["role"] == "Test" || parts["role"] == "Perf" {
task_os := parts["os"]
ec := parts["extra_config"]
if task_os == "Android" {
if ec == "Vulkan" {
ec = "Android_Vulkan"
} else if !strings.Contains(ec, "GN_Android") {
ec = task_os
}
task_os = "Android"
} else if task_os == "iOS" {
ec = task_os
task_os = "Mac"
} else if strings.Contains(task_os, "Win") {
task_os = "Win"
}
name, err := jobNameSchema.MakeJobName(map[string]string{
"role": "Build",
"os": task_os,
"compiler": parts["compiler"],
"target_arch": parts["arch"],
"configuration": parts["configuration"],
"extra_config": ec,
})
if err != nil {
glog.Fatal(err)
}
return name
} else {
return jobName
}
}
// swarmDimensions generates swarming bot dimensions for the given task.
func swarmDimensions(parts map[string]string) []string {
if parts["extra_config"] == "SkiaCT" {
return []string{
"pool:SkiaCT",
}
}
d := map[string]string{
"pool": POOL_SKIA,
}
if os, ok := parts["os"]; ok {
d["os"] = os
} else {
d["os"] = DEFAULT_OS
}
if strings.Contains(d["os"], "Win") {
d["os"] = "Windows"
}
if parts["role"] == "Test" || parts["role"] == "Perf" {
if strings.Contains(parts["os"], "Android") {
// For Android, the device type is a better dimension
// than CPU or GPU.
d["device_type"] = map[string]string{
"AndroidOne": "sprout",
"GalaxyS3": "m0", // "smdk4x12", Detected incorrectly by swarming?
"GalaxyS4": "", // TODO(borenet,kjlubick)
"GalaxyS7": "heroqlteatt",
"NVIDIA_Shield": "foster",
"Nexus10": "manta",
"Nexus5": "hammerhead",
"Nexus6": "shamu",
"Nexus6p": "angler",
"Nexus7": "grouper",
"Nexus7v2": "flo",
"Nexus9": "flounder",
"NexusPlayer": "fugu",
}[parts["model"]]
} else if strings.Contains(parts["os"], "iOS") {
d["device"] = map[string]string{
"iPad4": "iPad4,1",
}[parts["model"]]
// TODO(borenet): Replace this hack with something
// better.
d["os"] = "iOS-9.2"
} else if parts["cpu_or_gpu"] == "CPU" {
d["gpu"] = "none"
d["cpu"] = map[string]string{
"AVX": "x86-64",
"AVX2": "x86-64-avx2",
"SSE4": "x86-64",
}[parts["cpu_or_gpu_value"]]
if strings.Contains(parts["os"], "Win") && parts["cpu_or_gpu_value"] == "AVX2" {
// AVX2 is not correctly detected on Windows. Fall back on other
// dimensions to ensure that we correctly target machines which we know
// have AVX2 support.
d["cpu"] = "x86-64"
d["os"] = "Windows-2008ServerR2-SP1"
}
} else {
d["gpu"] = map[string]string{
"GeForce320M": "10de:08a4",
"GT610": "10de:104a",
"GTX550Ti": "10de:1244",
"GTX660": "10de:11c0",
"GTX960": "10de:1401",
"HD4000": "8086:0a2e",
"HD4600": "8086:0412",
"HD7770": "1002:683d",
"iHD530": "8086:1912",
}[parts["cpu_or_gpu_value"]]
}
} else {
d["gpu"] = "none"
}
rv := make([]string, 0, len(d))
for k, v := range d {
rv = append(rv, fmt.Sprintf("%s:%s", k, v))
}
sort.Strings(rv)
return rv
}
// getCipdPackage finds and returns the given CIPD package and version.
func getCipdPackage(assetName string) *specs.CipdPackage {
if pkg, ok := cipdPackages[assetName]; ok {
return pkg
}
versionFile := path.Join(infrabotsDir, "assets", assetName, "VERSION")
contents, err := ioutil.ReadFile(versionFile)
if err != nil {
glog.Fatal(err)
}
version := strings.TrimSpace(string(contents))
pkg := &specs.CipdPackage{
Name: fmt.Sprintf("skia/bots/%s", assetName),
Path: assetName,
Version: fmt.Sprintf("version:%s", version),
}
if assetName == "win_toolchain" {
pkg.Path = "t" // Workaround for path length limit on Windows.
}
cipdPackages[assetName] = pkg
return pkg
}
// compile generates a compile task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func compile(cfg *specs.TasksCfg, name string, parts map[string]string) string {
// Collect the necessary CIPD packages.
pkgs := []*specs.CipdPackage{}
// Android bots require a toolchain.
if strings.Contains(name, "Android") {
pkgs = append(pkgs, getCipdPackage("android_sdk"))
if strings.Contains(name, "Mac") {
pkgs = append(pkgs, getCipdPackage("android_ndk_darwin"))
} else {
pkgs = append(pkgs, getCipdPackage("android_ndk_linux"))
}
}
// Clang on Linux.
if strings.Contains(name, "Ubuntu") && strings.Contains(name, "Clang") {
pkgs = append(pkgs, getCipdPackage("clang_linux"))
}
// Windows toolchain.
if strings.Contains(name, "Win") {
pkgs = append(pkgs, getCipdPackage("win_toolchain"))
if strings.Contains(name, "Vulkan") {
pkgs = append(pkgs, getCipdPackage("win_vulkan_sdk"))
}
}
// Add the task.
cfg.Tasks[name] = &specs.TaskSpec{
CipdPackages: pkgs,
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_compile",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
},
Isolate: "compile_skia.isolate",
Priority: 0.8,
}
return name
}
// recreateSKPs generates a RecreateSKPs task. Returns the name of the last
// task in the generated chain of tasks, which the Job should add as a
// dependency.
func recreateSKPs(cfg *specs.TasksCfg, name string) string {
// TODO
return name
}
// ctSKPs generates a CT SKPs task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func ctSKPs(cfg *specs.TasksCfg, name string) string {
// TODO
return name
}
// housekeeper generates a Housekeeper task. Returns the name of the last task
// in the generated chain of tasks, which the Job should add as a dependency.
func housekeeper(cfg *specs.TasksCfg, name, compileTaskName string) string {
// TODO
return name
}
// doUpload indicates whether the given Job should upload its results.
func doUpload(name string) bool {
skipUploadBots := []string{
"ASAN",
"Coverage",
"MSAN",
"TSAN",
"UBSAN",
"Valgrind",
}
for _, s := range skipUploadBots {
if strings.Contains(name, s) {
return false
}
}
return true
}
// test generates a Test task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
cfg.Tasks[name] = &specs.TaskSpec{
CipdPackages: pkgs,
Dependencies: []string{compileTaskName},
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_test",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
},
Isolate: "test_skia.isolate",
Priority: 0.8,
}
// Upload results if necessary.
if doUpload(name) {
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
cfg.Tasks[uploadName] = &specs.TaskSpec{
Dependencies: []string{name},
Dimensions: UPLOAD_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_dm_results",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
},
Isolate: "upload_dm_results.isolate",
Priority: 0.8,
}
return uploadName
}
return name
}
// perf generates a Perf task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
cfg.Tasks[name] = &specs.TaskSpec{
CipdPackages: pkgs,
Dependencies: []string{compileTaskName},
Dimensions: swarmDimensions(parts),
ExtraArgs: []string{
"--workdir", "../../..", "swarm_perf",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
},
Isolate: "perf_skia.isolate",
Priority: 0.8,
}
// Upload results if necessary.
if strings.Contains(name, "Release") && doUpload(name) {
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
cfg.Tasks[uploadName] = &specs.TaskSpec{
Dependencies: []string{name},
Dimensions: UPLOAD_DIMENSIONS,
ExtraArgs: []string{
"--workdir", "../../..", "upload_nano_results",
fmt.Sprintf("buildername=%s", name),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
},
Isolate: "upload_nano_results.isolate",
Priority: 0.8,
}
return uploadName
}
return name
}
// process generates tasks and jobs for the given job name.
func process(cfg *specs.TasksCfg, name string) {
if _, ok := cfg.Jobs[name]; ok {
glog.Fatalf("Duplicate job %q", name)
}
deps := []string{}
parts, err := jobNameSchema.ParseJobName(name)
if err != nil {
glog.Fatal(err)
}
// RecreateSKPs.
if strings.Contains(name, "RecreateSKPs") {
deps = append(deps, recreateSKPs(cfg, name))
}
// CT bots.
if strings.Contains(name, "-CT_") {
deps = append(deps, ctSKPs(cfg, name))
}
// Compile bots.
if parts["role"] == "Build" {
deps = append(deps, compile(cfg, name, parts))
}
// Any remaining bots need a compile task.
compileTaskName := deriveCompileTaskName(name, parts)
// Housekeeper.
if parts["role"] == "Housekeeper" {
deps = append(deps, housekeeper(cfg, name, compileTaskName))
}
// Common assets needed by the remaining bots.
pkgs := []*specs.CipdPackage{
getCipdPackage("skimage"),
getCipdPackage("skp"),
getCipdPackage("svg"),
}
// Test bots.
if parts["role"] == "Test" {
deps = append(deps, test(cfg, name, parts, compileTaskName, pkgs))
}
// Perf bots.
if parts["role"] == "Perf" {
deps = append(deps, perf(cfg, name, parts, compileTaskName, pkgs))
}
// Add the Job spec.
cfg.Jobs[name] = &specs.JobSpec{
Priority: 0.8,
TaskSpecs: deps,
}
}
// getCheckoutRoot returns the path of the root of the Skia checkout, or an
// error if it cannot be found.
func getCheckoutRoot() string {
cwd, err := os.Getwd()
if err != nil {
glog.Fatal(err)
}
for {
if _, err := os.Stat(cwd); err != nil {
glog.Fatal(err)
}
s, err := os.Stat(path.Join(cwd, ".git"))
if err == nil && s.IsDir() {
// TODO(borenet): Should we verify that this is a Skia
// checkout and not something else?
return cwd
}
cwd = filepath.Clean(path.Join(cwd, ".."))
}
}
// Regenerate the tasks.json file.
func main() {
common.Init()
defer common.LogPanic()
// Where are we?
root := getCheckoutRoot()
infrabotsDir = path.Join(root, "infra", "bots")
// Create the JobNameSchema.
schema, err := NewJobNameSchema(path.Join(infrabotsDir, "recipe_modules", "builder_name_schema", "builder_name_schema.json"))
if err != nil {
glog.Fatal(err)
}
jobNameSchema = schema
// Create the config.
cfg := &specs.TasksCfg{
Jobs: map[string]*specs.JobSpec{},
Tasks: map[string]*specs.TaskSpec{},
}
// Create Tasks and Jobs.
for _, j := range JOBS {
process(cfg, j)
}
// Validate the config.
if err := cfg.Validate(); err != nil {
glog.Fatal(err)
}
// Write the tasks.json file.
b, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
glog.Fatal(err)
}
// The json package escapes HTML characters, which makes our output
// much less readable. Replace the escape characters with the real
// character.
b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
outFile := path.Join(root, specs.TASKS_CFG_FILE)
if *testing {
// Don't write the file; read it and compare.
expect, err := ioutil.ReadFile(outFile)
if err != nil {
glog.Fatal(err)
}
if !bytes.Equal(expect, b) {
glog.Fatalf("Expected no changes, but changes were found!")
}
} else {
if err := ioutil.WriteFile(outFile, b, os.ModePerm); err != nil {
glog.Fatal(err)
}
}
}
// TODO(borenet): The below really belongs in its own file, probably next to the
// builder_name_schema.json file.
// JobNameSchema is a struct used for (de)constructing Job names in a
// predictable format.
type JobNameSchema struct {
Schema map[string][]string `json:"builder_name_schema"`
Sep string `json:"builder_name_sep"`
}
// NewJobNameSchema returns a JobNameSchema instance based on the given JSON
// file.
func NewJobNameSchema(jsonFile string) (*JobNameSchema, error) {
var rv JobNameSchema
f, err := os.Open(jsonFile)
if err != nil {
return nil, err
}
defer util.Close(f)
if err := json.NewDecoder(f).Decode(&rv); err != nil {
return nil, err
}
return &rv, nil
}
// ParseJobName splits the given Job name into its component parts, according
// to the schema.
func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error) {
split := strings.Split(n, s.Sep)
if len(split) < 2 {
return nil, fmt.Errorf("Invalid job name: %q", n)
}
role := split[0]
split = split[1:]
keys, ok := s.Schema[role]
if !ok {
return nil, fmt.Errorf("Invalid job name; %q is not a valid role.", role)
}
extraConfig := ""
if len(split) == len(keys)+1 {
extraConfig = split[len(split)-1]
split = split[:len(split)-1]
}
if len(split) != len(keys) {
return nil, fmt.Errorf("Invalid job name; %q has incorrect number of parts.", n)
}
rv := make(map[string]string, len(keys)+2)
rv["role"] = role
if extraConfig != "" {
rv["extra_config"] = extraConfig
}
for i, k := range keys {
rv[k] = split[i]
}
return rv, nil
}
// MakeJobName assembles the given parts of a Job name, according to the schema.
func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error) {
role, ok := parts["role"]
if !ok {
return "", fmt.Errorf("Invalid job parts; jobs must have a role.")
}
keys, ok := s.Schema[role]
if !ok {
return "", fmt.Errorf("Invalid job parts; unknown role %q", role)
}
rvParts := make([]string, 0, len(parts))
rvParts = append(rvParts, role)
for _, k := range keys {
v, ok := parts[k]
if !ok {
return "", fmt.Errorf("Invalid job parts; missing %q", k)
}
rvParts = append(rvParts, v)
}
if _, ok := parts["extra_config"]; ok {
rvParts = append(rvParts, parts["extra_config"])
}
return strings.Join(rvParts, s.Sep), nil
}