[task scheduler] Add gen_tasks.go
This mimics the flow of infra/bots/recipes/swarm_trigger.py but it builds the tasks.json file instead of triggering the actual tasks. BUG=skia: GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2386663002 Review-Url: https://codereview.chromium.org/2386663002
This commit is contained in:
borenet
Commit bot
parent
476e6c9a7f
commit
db182c770f
589
infra/bots/gen_tasks.go
Normal file
589
infra/bots/gen_tasks.go
Normal file
@ -0,0 +1,589 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
package main
|
||||
|
||||
/*
|
||||
Generate the tasks.json file.
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/skia-dev/glog"
|
||||
"go.skia.org/infra/go/common"
|
||||
"go.skia.org/infra/go/util"
|
||||
"go.skia.org/infra/task_scheduler/go/specs"
|
||||
)
|
||||
|
||||
const (
|
||||
DEFAULT_OS = "Ubuntu"
|
||||
|
||||
// Pool for Skia bots.
|
||||
POOL_SKIA = "Skia"
|
||||
|
||||
// Name prefix for upload jobs.
|
||||
PREFIX_UPLOAD = "Upload"
|
||||
)
|
||||
|
||||
var (
|
||||
// "Constants"
|
||||
|
||||
// Top-level list of all jobs to run at each commit.
|
||||
JOBS = []string{
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN",
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
}
|
||||
|
||||
// UPLOAD_DIMENSIONS are the Swarming dimensions for upload tasks.
|
||||
UPLOAD_DIMENSIONS = []string{
|
||||
"cpu:x86-64-avx2",
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
fmt.Sprintf("pool:%s", POOL_SKIA),
|
||||
}
|
||||
|
||||
// Defines the structure of job names.
|
||||
jobNameSchema *JobNameSchema
|
||||
|
||||
// Caches CIPD package info so that we don't have to re-read VERSION
|
||||
// files.
|
||||
cipdPackages = map[string]*specs.CipdPackage{}
|
||||
|
||||
// Path to the infra/bots directory.
|
||||
infrabotsDir = ""
|
||||
)
|
||||
|
||||
// deriveCompileTaskName returns the name of a compile task based on the given
|
||||
// job name.
|
||||
func deriveCompileTaskName(jobName string, parts map[string]string) string {
|
||||
if parts["role"] == "Housekeeper" {
|
||||
return "Build-Ubuntu-GCC-x86_64-Release-Shared"
|
||||
} else if parts["role"] == "Test" || parts["role"] == "Perf" {
|
||||
task_os := parts["os"]
|
||||
ec := parts["extra_config"]
|
||||
if task_os == "Android" {
|
||||
if ec == "Vulkan" {
|
||||
ec = "Android_Vulkan"
|
||||
} else if !strings.Contains(ec, "GN_Android") {
|
||||
ec = task_os
|
||||
}
|
||||
task_os = "Android"
|
||||
} else if task_os == "iOS" {
|
||||
ec = task_os
|
||||
task_os = "Mac"
|
||||
} else if strings.Contains(task_os, "Win") {
|
||||
task_os = "Win"
|
||||
}
|
||||
name, err := jobNameSchema.MakeJobName(map[string]string{
|
||||
"role": "Build",
|
||||
"os": task_os,
|
||||
"compiler": parts["compiler"],
|
||||
"target_arch": parts["arch"],
|
||||
"configuration": parts["configuration"],
|
||||
"extra_config": ec,
|
||||
})
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
return name
|
||||
} else {
|
||||
return jobName
|
||||
}
|
||||
}
|
||||
|
||||
// swarmDimensions generates swarming bot dimensions for the given task.
|
||||
func swarmDimensions(parts map[string]string) []string {
|
||||
if parts["extra_config"] == "SkiaCT" {
|
||||
return []string{
|
||||
"pool:SkiaCT",
|
||||
}
|
||||
}
|
||||
d := map[string]string{
|
||||
"pool": POOL_SKIA,
|
||||
}
|
||||
if os, ok := parts["os"]; ok {
|
||||
d["os"] = os
|
||||
} else {
|
||||
d["os"] = DEFAULT_OS
|
||||
}
|
||||
if strings.Contains(d["os"], "Win") {
|
||||
d["os"] = "Windows"
|
||||
}
|
||||
if parts["role"] == "Test" || parts["role"] == "Perf" {
|
||||
if strings.Contains(parts["os"], "Android") {
|
||||
// For Android, the device type is a better dimension
|
||||
// than CPU or GPU.
|
||||
d["device_type"] = map[string]string{
|
||||
"AndroidOne": "sprout",
|
||||
"GalaxyS3": "m0", // "smdk4x12", Detected incorrectly by swarming?
|
||||
"GalaxyS4": "", // TODO(borenet,kjlubick)
|
||||
"GalaxyS7": "heroqlteatt",
|
||||
"NVIDIA_Shield": "foster",
|
||||
"Nexus10": "manta",
|
||||
"Nexus5": "hammerhead",
|
||||
"Nexus6": "shamu",
|
||||
"Nexus6p": "angler",
|
||||
"Nexus7": "grouper",
|
||||
"Nexus7v2": "flo",
|
||||
"Nexus9": "flounder",
|
||||
"NexusPlayer": "fugu",
|
||||
}[parts["model"]]
|
||||
} else if strings.Contains(parts["os"], "iOS") {
|
||||
d["device"] = map[string]string{
|
||||
"iPad4": "iPad4,1",
|
||||
}[parts["model"]]
|
||||
// TODO(borenet): Replace this hack with something
|
||||
// better.
|
||||
d["os"] = "iOS-9.2"
|
||||
} else if parts["cpu_or_gpu"] == "CPU" {
|
||||
d["gpu"] = "none"
|
||||
d["cpu"] = map[string]string{
|
||||
"AVX": "x86-64",
|
||||
"AVX2": "x86-64-avx2",
|
||||
"SSE4": "x86-64",
|
||||
}[parts["cpu_or_gpu_value"]]
|
||||
if strings.Contains(parts["os"], "Win") && parts["cpu_or_gpu_value"] == "AVX2" {
|
||||
// AVX2 is not correctly detected on Windows. Fall back on other
|
||||
// dimensions to ensure that we correctly target machines which we know
|
||||
// have AVX2 support.
|
||||
d["cpu"] = "x86-64"
|
||||
d["os"] = "Windows-2008ServerR2-SP1"
|
||||
}
|
||||
} else {
|
||||
d["gpu"] = map[string]string{
|
||||
"GeForce320M": "10de:08a4",
|
||||
"GT610": "10de:104a",
|
||||
"GTX550Ti": "10de:1244",
|
||||
"GTX660": "10de:11c0",
|
||||
"GTX960": "10de:1401",
|
||||
"HD4000": "8086:0a2e",
|
||||
"HD4600": "8086:0412",
|
||||
"HD7770": "1002:683d",
|
||||
"iHD530": "8086:1912",
|
||||
}[parts["cpu_or_gpu_value"]]
|
||||
}
|
||||
} else {
|
||||
d["gpu"] = "none"
|
||||
}
|
||||
rv := make([]string, 0, len(d))
|
||||
for k, v := range d {
|
||||
rv = append(rv, fmt.Sprintf("%s:%s", k, v))
|
||||
}
|
||||
sort.Strings(rv)
|
||||
return rv
|
||||
}
|
||||
|
||||
// getCipdPackage finds and returns the given CIPD package and version.
|
||||
func getCipdPackage(assetName string) *specs.CipdPackage {
|
||||
if pkg, ok := cipdPackages[assetName]; ok {
|
||||
return pkg
|
||||
}
|
||||
versionFile := path.Join(infrabotsDir, "assets", assetName, "VERSION")
|
||||
contents, err := ioutil.ReadFile(versionFile)
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
version := strings.TrimSpace(string(contents))
|
||||
pkg := &specs.CipdPackage{
|
||||
Name: fmt.Sprintf("skia/bots/%s", assetName),
|
||||
Path: assetName,
|
||||
Version: fmt.Sprintf("version:%s", version),
|
||||
}
|
||||
if assetName == "win_toolchain" {
|
||||
pkg.Path = "t" // Workaround for path length limit on Windows.
|
||||
}
|
||||
cipdPackages[assetName] = pkg
|
||||
return pkg
|
||||
}
|
||||
|
||||
// compile generates a compile task. Returns the name of the last task in the
|
||||
// generated chain of tasks, which the Job should add as a dependency.
|
||||
func compile(cfg *specs.TasksCfg, name string, parts map[string]string) string {
|
||||
// Collect the necessary CIPD packages.
|
||||
pkgs := []*specs.CipdPackage{}
|
||||
|
||||
// Android bots require a toolchain.
|
||||
if strings.Contains(name, "Android") {
|
||||
pkgs = append(pkgs, getCipdPackage("android_sdk"))
|
||||
if strings.Contains(name, "Mac") {
|
||||
pkgs = append(pkgs, getCipdPackage("android_ndk_darwin"))
|
||||
} else {
|
||||
pkgs = append(pkgs, getCipdPackage("android_ndk_linux"))
|
||||
}
|
||||
}
|
||||
|
||||
// Clang on Linux.
|
||||
if strings.Contains(name, "Ubuntu") && strings.Contains(name, "Clang") {
|
||||
pkgs = append(pkgs, getCipdPackage("clang_linux"))
|
||||
}
|
||||
|
||||
// Windows toolchain.
|
||||
if strings.Contains(name, "Win") {
|
||||
pkgs = append(pkgs, getCipdPackage("win_toolchain"))
|
||||
if strings.Contains(name, "Vulkan") {
|
||||
pkgs = append(pkgs, getCipdPackage("win_vulkan_sdk"))
|
||||
}
|
||||
}
|
||||
|
||||
// Add the task.
|
||||
cfg.Tasks[name] = &specs.TaskSpec{
|
||||
CipdPackages: pkgs,
|
||||
Dimensions: swarmDimensions(parts),
|
||||
ExtraArgs: []string{
|
||||
"--workdir", "../../..", "swarm_compile",
|
||||
fmt.Sprintf("buildername=%s", name),
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
|
||||
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
|
||||
},
|
||||
Isolate: "compile_skia.isolate",
|
||||
Priority: 0.8,
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// recreateSKPs generates a RecreateSKPs task. Returns the name of the last
|
||||
// task in the generated chain of tasks, which the Job should add as a
|
||||
// dependency.
|
||||
func recreateSKPs(cfg *specs.TasksCfg, name string) string {
|
||||
// TODO
|
||||
return name
|
||||
}
|
||||
|
||||
// ctSKPs generates a CT SKPs task. Returns the name of the last task in the
|
||||
// generated chain of tasks, which the Job should add as a dependency.
|
||||
func ctSKPs(cfg *specs.TasksCfg, name string) string {
|
||||
// TODO
|
||||
return name
|
||||
}
|
||||
|
||||
// housekeeper generates a Housekeeper task. Returns the name of the last task
|
||||
// in the generated chain of tasks, which the Job should add as a dependency.
|
||||
func housekeeper(cfg *specs.TasksCfg, name, compileTaskName string) string {
|
||||
// TODO
|
||||
return name
|
||||
}
|
||||
|
||||
// doUpload indicates whether the given Job should upload its results.
|
||||
func doUpload(name string) bool {
|
||||
skipUploadBots := []string{
|
||||
"ASAN",
|
||||
"Coverage",
|
||||
"MSAN",
|
||||
"TSAN",
|
||||
"UBSAN",
|
||||
"Valgrind",
|
||||
}
|
||||
for _, s := range skipUploadBots {
|
||||
if strings.Contains(name, s) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// test generates a Test task. Returns the name of the last task in the
|
||||
// generated chain of tasks, which the Job should add as a dependency.
|
||||
func test(cfg *specs.TasksCfg, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
|
||||
cfg.Tasks[name] = &specs.TaskSpec{
|
||||
CipdPackages: pkgs,
|
||||
Dependencies: []string{compileTaskName},
|
||||
Dimensions: swarmDimensions(parts),
|
||||
ExtraArgs: []string{
|
||||
"--workdir", "../../..", "swarm_test",
|
||||
fmt.Sprintf("buildername=%s", name),
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
|
||||
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
|
||||
},
|
||||
Isolate: "test_skia.isolate",
|
||||
Priority: 0.8,
|
||||
}
|
||||
// Upload results if necessary.
|
||||
if doUpload(name) {
|
||||
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
|
||||
cfg.Tasks[uploadName] = &specs.TaskSpec{
|
||||
Dependencies: []string{name},
|
||||
Dimensions: UPLOAD_DIMENSIONS,
|
||||
ExtraArgs: []string{
|
||||
"--workdir", "../../..", "upload_dm_results",
|
||||
fmt.Sprintf("buildername=%s", name),
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
|
||||
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
|
||||
},
|
||||
Isolate: "upload_dm_results.isolate",
|
||||
Priority: 0.8,
|
||||
}
|
||||
return uploadName
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// perf generates a Perf task. Returns the name of the last task in the
|
||||
// generated chain of tasks, which the Job should add as a dependency.
|
||||
func perf(cfg *specs.TasksCfg, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string {
|
||||
cfg.Tasks[name] = &specs.TaskSpec{
|
||||
CipdPackages: pkgs,
|
||||
Dependencies: []string{compileTaskName},
|
||||
Dimensions: swarmDimensions(parts),
|
||||
ExtraArgs: []string{
|
||||
"--workdir", "../../..", "swarm_perf",
|
||||
fmt.Sprintf("buildername=%s", name),
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
|
||||
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
|
||||
},
|
||||
Isolate: "perf_skia.isolate",
|
||||
Priority: 0.8,
|
||||
}
|
||||
// Upload results if necessary.
|
||||
if strings.Contains(name, "Release") && doUpload(name) {
|
||||
uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name)
|
||||
cfg.Tasks[uploadName] = &specs.TaskSpec{
|
||||
Dependencies: []string{name},
|
||||
Dimensions: UPLOAD_DIMENSIONS,
|
||||
ExtraArgs: []string{
|
||||
"--workdir", "../../..", "upload_nano_results",
|
||||
fmt.Sprintf("buildername=%s", name),
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
|
||||
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
|
||||
},
|
||||
Isolate: "upload_nano_results.isolate",
|
||||
Priority: 0.8,
|
||||
}
|
||||
return uploadName
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// process generates tasks and jobs for the given job name.
|
||||
func process(cfg *specs.TasksCfg, name string) {
|
||||
if _, ok := cfg.Jobs[name]; ok {
|
||||
glog.Fatalf("Duplicate job %q", name)
|
||||
}
|
||||
deps := []string{}
|
||||
|
||||
parts, err := jobNameSchema.ParseJobName(name)
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
|
||||
// RecreateSKPs.
|
||||
if strings.Contains(name, "RecreateSKPs") {
|
||||
deps = append(deps, recreateSKPs(cfg, name))
|
||||
}
|
||||
|
||||
// CT bots.
|
||||
if strings.Contains(name, "-CT_") {
|
||||
deps = append(deps, ctSKPs(cfg, name))
|
||||
}
|
||||
|
||||
// Compile bots.
|
||||
if parts["role"] == "Build" {
|
||||
deps = append(deps, compile(cfg, name, parts))
|
||||
}
|
||||
|
||||
// Any remaining bots need a compile task.
|
||||
compileTaskName := deriveCompileTaskName(name, parts)
|
||||
|
||||
// Housekeeper.
|
||||
if parts["role"] == "Housekeeper" {
|
||||
deps = append(deps, housekeeper(cfg, name, compileTaskName))
|
||||
}
|
||||
|
||||
// Common assets needed by the remaining bots.
|
||||
pkgs := []*specs.CipdPackage{
|
||||
getCipdPackage("skimage"),
|
||||
getCipdPackage("skp"),
|
||||
getCipdPackage("svg"),
|
||||
}
|
||||
|
||||
// Test bots.
|
||||
if parts["role"] == "Test" {
|
||||
deps = append(deps, test(cfg, name, parts, compileTaskName, pkgs))
|
||||
}
|
||||
|
||||
// Perf bots.
|
||||
if parts["role"] == "Perf" {
|
||||
deps = append(deps, perf(cfg, name, parts, compileTaskName, pkgs))
|
||||
}
|
||||
|
||||
// Add the Job spec.
|
||||
cfg.Jobs[name] = &specs.JobSpec{
|
||||
Priority: 0.8,
|
||||
TaskSpecs: deps,
|
||||
}
|
||||
}
|
||||
|
||||
// getCheckoutRoot returns the path of the root of the Skia checkout, or an
|
||||
// error if it cannot be found.
|
||||
func getCheckoutRoot() string {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(cwd); err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
s, err := os.Stat(path.Join(cwd, ".git"))
|
||||
if err == nil && s.IsDir() {
|
||||
// TODO(borenet): Should we verify that this is a Skia
|
||||
// checkout and not something else?
|
||||
return cwd
|
||||
}
|
||||
cwd = filepath.Clean(path.Join(cwd, ".."))
|
||||
}
|
||||
}
|
||||
|
||||
// Regenerate the tasks.json file.
|
||||
func main() {
|
||||
common.Init()
|
||||
defer common.LogPanic()
|
||||
|
||||
// Where are we?
|
||||
root := getCheckoutRoot()
|
||||
infrabotsDir = path.Join(root, "infra", "bots")
|
||||
|
||||
// Create the JobNameSchema.
|
||||
schema, err := NewJobNameSchema(path.Join(infrabotsDir, "recipe_modules", "builder_name_schema", "builder_name_schema.json"))
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
jobNameSchema = schema
|
||||
|
||||
// Create the config.
|
||||
cfg := &specs.TasksCfg{
|
||||
Jobs: map[string]*specs.JobSpec{},
|
||||
Tasks: map[string]*specs.TaskSpec{},
|
||||
}
|
||||
|
||||
// Create Tasks and Jobs.
|
||||
for _, j := range JOBS {
|
||||
process(cfg, j)
|
||||
}
|
||||
|
||||
// Validate the config.
|
||||
if err := cfg.Validate(); err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
|
||||
// Write the tasks.json file.
|
||||
outFile := path.Join(root, specs.TASKS_CFG_FILE)
|
||||
b, err := json.MarshalIndent(cfg, "", " ")
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
// The json package escapes HTML characters, which makes our output
|
||||
// much less readable. Replace the escape characters with the real
|
||||
// character.
|
||||
b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1)
|
||||
if err := ioutil.WriteFile(outFile, b, os.ModePerm); err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(borenet): The below really belongs in its own file, probably next to the
|
||||
// builder_name_schema.json file.
|
||||
|
||||
// JobNameSchema is a struct used for (de)constructing Job names in a
|
||||
// predictable format.
|
||||
type JobNameSchema struct {
|
||||
Schema map[string][]string `json:"builder_name_schema"`
|
||||
Sep string `json:"builder_name_sep"`
|
||||
}
|
||||
|
||||
// NewJobNameSchema returns a JobNameSchema instance based on the given JSON
|
||||
// file.
|
||||
func NewJobNameSchema(jsonFile string) (*JobNameSchema, error) {
|
||||
var rv JobNameSchema
|
||||
f, err := os.Open(jsonFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer util.Close(f)
|
||||
if err := json.NewDecoder(f).Decode(&rv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
// ParseJobName splits the given Job name into its component parts, according
|
||||
// to the schema.
|
||||
func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error) {
|
||||
split := strings.Split(n, s.Sep)
|
||||
if len(split) < 2 {
|
||||
return nil, fmt.Errorf("Invalid job name: %q", n)
|
||||
}
|
||||
role := split[0]
|
||||
split = split[1:]
|
||||
keys, ok := s.Schema[role]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Invalid job name; %q is not a valid role.", role)
|
||||
}
|
||||
extraConfig := ""
|
||||
if len(split) == len(keys)+1 {
|
||||
extraConfig = split[len(split)-1]
|
||||
split = split[:len(split)-1]
|
||||
}
|
||||
if len(split) != len(keys) {
|
||||
return nil, fmt.Errorf("Invalid job name; %q has incorrect number of parts.", n)
|
||||
}
|
||||
rv := make(map[string]string, len(keys)+2)
|
||||
rv["role"] = role
|
||||
if extraConfig != "" {
|
||||
rv["extra_config"] = extraConfig
|
||||
}
|
||||
for i, k := range keys {
|
||||
rv[k] = split[i]
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
// MakeJobName assembles the given parts of a Job name, according to the schema.
|
||||
func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error) {
|
||||
role, ok := parts["role"]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("Invalid job parts; jobs must have a role.")
|
||||
}
|
||||
keys, ok := s.Schema[role]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("Invalid job parts; unknown role %q", role)
|
||||
}
|
||||
rvParts := make([]string, 0, len(parts))
|
||||
rvParts = append(rvParts, role)
|
||||
for _, k := range keys {
|
||||
v, ok := parts[k]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("Invalid job parts; missing %q", k)
|
||||
}
|
||||
rvParts = append(rvParts, v)
|
||||
}
|
||||
if _, ok := parts["extra_config"]; ok {
|
||||
rvParts = append(rvParts, parts["extra_config"])
|
||||
}
|
||||
return strings.Join(rvParts, s.Sep), nil
|
||||
}
|
||||
@ -1,12 +1,36 @@
|
||||
{
|
||||
"jobs": {
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": [
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN"
|
||||
]
|
||||
},
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": [
|
||||
"Upload-Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"
|
||||
]
|
||||
},
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": [
|
||||
"Upload-Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"
|
||||
]
|
||||
}
|
||||
},
|
||||
"tasks": {
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN": {
|
||||
"cipd_packages": [],
|
||||
"dependencies": [],
|
||||
"dimensions": ["pool:Skia", "os:Ubuntu", "gpu:none"],
|
||||
"dimensions": [
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir", "../../..", "swarm_compile",
|
||||
"buildername=<(TASK_NAME)",
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"swarm_compile",
|
||||
"buildername=Build-Ubuntu-GCC-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
@ -16,27 +40,79 @@
|
||||
"isolate": "compile_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"cipd_packages": [{
|
||||
"name": "skia/bots/skimage",
|
||||
"path": "skimage",
|
||||
"version": "version:4"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/skp",
|
||||
"path": "skp",
|
||||
"version": "version:6"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/svg",
|
||||
"path": "svg",
|
||||
"version": "version:1"
|
||||
}],
|
||||
"dependencies": ["Build-Ubuntu-GCC-x86_64-Release-GN"],
|
||||
"dimensions": ["pool:Skia", "os:Ubuntu", "cpu:x86-64-avx2", "gpu:none"],
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"cipd_packages": [
|
||||
{
|
||||
"name": "skia/bots/skimage",
|
||||
"path": "skimage",
|
||||
"version": "version:8"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/skp",
|
||||
"path": "skp",
|
||||
"version": "version:10"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/svg",
|
||||
"path": "svg",
|
||||
"version": "version:3"
|
||||
}
|
||||
],
|
||||
"dependencies": [
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN"
|
||||
],
|
||||
"dimensions": [
|
||||
"cpu:x86-64-avx2",
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir", "../../..", "swarm_test",
|
||||
"buildername=<(TASK_NAME)",
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"swarm_perf",
|
||||
"buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}",
|
||||
"revision=<(REVISION)"
|
||||
],
|
||||
"isolate": "perf_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"cipd_packages": [
|
||||
{
|
||||
"name": "skia/bots/skimage",
|
||||
"path": "skimage",
|
||||
"version": "version:8"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/skp",
|
||||
"path": "skp",
|
||||
"version": "version:10"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/svg",
|
||||
"path": "svg",
|
||||
"version": "version:3"
|
||||
}
|
||||
],
|
||||
"dependencies": [
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN"
|
||||
],
|
||||
"dimensions": [
|
||||
"cpu:x86-64-avx2",
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"swarm_test",
|
||||
"buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
@ -46,80 +122,53 @@
|
||||
"isolate": "test_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Upload-Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"cipd_packages": [],
|
||||
"dependencies": ["Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"],
|
||||
"dimensions": ["pool:Skia", "os:Ubuntu", "cpu:x86-64-avx2", "gpu:none"],
|
||||
"extra_args": [
|
||||
"--workdir", "../../..", "upload_dm_results",
|
||||
"buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"revision=<(REVISION)",
|
||||
"slavename=fake-buildslave",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}"
|
||||
],
|
||||
"isolate": "upload_dm_results.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"cipd_packages": [{
|
||||
"name": "skia/bots/skimage",
|
||||
"path": "skimage",
|
||||
"version": "version:4"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/skp",
|
||||
"path": "skp",
|
||||
"version": "version:6"
|
||||
},
|
||||
{
|
||||
"name": "skia/bots/svg",
|
||||
"path": "svg",
|
||||
"version": "version:1"
|
||||
}],
|
||||
"dependencies": ["Build-Ubuntu-GCC-x86_64-Release-GN"],
|
||||
"dimensions": ["pool:Skia", "os:Ubuntu", "cpu:x86-64-avx2", "gpu:none"],
|
||||
"extra_args": [
|
||||
"--workdir", "../../..", "swarm_perf",
|
||||
"buildername=<(TASK_NAME)",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}",
|
||||
"revision=<(REVISION)"
|
||||
],
|
||||
"isolate": "perf_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Upload-Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"dependencies": ["Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"],
|
||||
"dimensions": ["pool:Skia", "os:Ubuntu", "cpu:x86-64-avx2", "gpu:none"],
|
||||
"extra_args": [
|
||||
"--workdir", "../../..", "upload_nano_results",
|
||||
"dependencies": [
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"
|
||||
],
|
||||
"dimensions": [
|
||||
"cpu:x86-64-avx2",
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"upload_nano_results",
|
||||
"buildername=Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"revision=<(REVISION)",
|
||||
"slavename=fake-buildslave",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}"
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}",
|
||||
"revision=<(REVISION)"
|
||||
],
|
||||
"isolate": "upload_nano_results.isolate",
|
||||
"priority": 0.8
|
||||
}
|
||||
},
|
||||
"jobs": {
|
||||
"Build-Ubuntu-GCC-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": ["Build-Ubuntu-GCC-x86_64-Release-GN"]
|
||||
},
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": ["Upload-Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"]
|
||||
},
|
||||
"Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"priority": 0.8,
|
||||
"tasks": ["Upload-Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"]
|
||||
"Upload-Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN": {
|
||||
"dependencies": [
|
||||
"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN"
|
||||
],
|
||||
"dimensions": [
|
||||
"cpu:x86-64-avx2",
|
||||
"gpu:none",
|
||||
"os:Ubuntu",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"upload_dm_results",
|
||||
"buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-GN",
|
||||
"mastername=fake-master",
|
||||
"buildnumber=2",
|
||||
"slavename=fake-buildslave",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}",
|
||||
"revision=<(REVISION)"
|
||||
],
|
||||
"isolate": "upload_dm_results.isolate",
|
||||
"priority": 0.8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user