AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add fm driver

Browse Source 
PS 6 trying StartStep/EndStep/FailStep()
PS 7 better usage?
PS 8 goes back to td.Fatal* for top-level failures

Failures seem to be working ok as of PS 8,
but I am puzzled why PS 7 wasn't correct... much prefer it.

Also set max_attempts to 1... this driver will handle flakiness itself.

Change-Id: I7de6809920bfaf1d878d654c9cf5b7861a64d23f
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/286118
Reviewed-by: Mike Klein <mtklein@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Mike Klein <mtklein@google.com>
This commit is contained in:
Mike Klein 2020-04-29 12:49:28 -05:00 committed by Skia Commit-Bot
parent f59a961dc9
commit 298bda131c
7 changed files with 161 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -433,7 +433,7 @@ func (b *taskBuilder) linuxGceDimensions(machineType string) {
// deriveCompileTaskName returns the name of a compile task based on the given
// job name.
func (b *jobBuilder) deriveCompileTaskName() string {
if b.role("Test", "Perf") {
if b.role("Test", "Perf", "FM") {
task_os := b.parts["os"]
ec := []string{}
if val := b.parts["extra_config"]; val != "" {
@ -1272,7 +1272,7 @@ func (b *taskBuilder) commonTestPerfAssets() {
}
// test generates a Test task.
func (b *jobBuilder) test() {
func (b *jobBuilder) dm() {
compileTaskName := ""
// LottieWeb doesn't require anything in Skia to be compiled.
if !b.extraConfig("LottieWeb") {
@ -1373,6 +1373,23 @@ func (b *jobBuilder) test() {
}
}
func (b *jobBuilder) fm() {
b.addTask(b.Name, func(b *taskBuilder) {
b.isolate("test_skia_bundled.isolate")
b.dep(b.buildTaskDrivers(), b.compile())
b.cmd("./fm_driver",
"--local=false",
"--resources=skia/resources",
"--project_id", "skia-swarming-bots",
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.swarmDimensions()
b.expiration(15 * time.Minute)
b.attempts(1)
})
}
// perf generates a Perf task.
func (b *jobBuilder) perf() {
compileTaskName := ""

6
infra/bots/gen_tasks_logic/job_builder.go
View File

@ -180,7 +180,11 @@ func (b *jobBuilder) genTasksForJob() {
// Test bots.
if b.role("Test") {
b.test()
b.dm()
return
}
if b.role("FM") {
b.fm()
return
}

1
infra/bots/jobs.json
View File

@ -143,6 +143,7 @@
"BuildStats-Debian10-EMCC-wasm-Release-CanvasKit",
"BuildStats-Debian10-EMCC-wasm-Release-CanvasKit_CPU",
"BuildStats-Debian10-EMCC-wasm-Release-PathKit",
"FM-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All",
"Housekeeper-Nightly-RecreateSKPs_Canary",
"Housekeeper-Nightly-UpdateGoDeps",
"Housekeeper-OnDemand-Presubmit",

15
infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
View File

@ -68,6 +68,21 @@
"extra_config"
]
},
"FM": {
"keys": [
"os",
"compiler",
"model",
"cpu_or_gpu",
"cpu_or_gpu_value",
"arch",
"configuration",
"test_filter"
],
"optional_keys": [
"extra_config"
]
},
"Upload": {
"recurse_roles": [
"Build",

2
infra/bots/recipe_modules/builder_name_schema/builder_name_schema.py
View File

@ -26,6 +26,7 @@ BUILDER_ROLE_HOUSEKEEPER = 'Housekeeper'
BUILDER_ROLE_INFRA = 'Infra'
BUILDER_ROLE_PERF = 'Perf'
BUILDER_ROLE_TEST = 'Test'
BUILDER_ROLE_FM = 'FM'
BUILDER_ROLE_UPLOAD = 'Upload'
BUILDER_ROLES = (BUILDER_ROLE_BUILD,
BUILDER_ROLE_BUILDSTATS,
@ -33,6 +34,7 @@ BUILDER_ROLES = (BUILDER_ROLE_BUILD,
BUILDER_ROLE_INFRA,
BUILDER_ROLE_PERF,
BUILDER_ROLE_TEST,
BUILDER_ROLE_FM,
BUILDER_ROLE_UPLOAD)

86
infra/bots/task_drivers/fm_driver/fm_driver.go Normal file
View File

@ -0,0 +1,86 @@
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"bufio"
"flag"
"math/rand"
"runtime"
"strings"
"sync"
"sync/atomic"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/util"
"go.skia.org/infra/task_driver/go/td"
)
func main() {
var (
projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
taskId = flag.String("task_id", "", "ID of this task.")
taskName = flag.String("task_name", "", "Name of the task.")
local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)")
output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
workdir = flag.String("workdir", ".", "Working directory")
fm = flag.String("fm", "build/fm", "FM binary to run, relative to -workdir")
resources = flag.String("resources", "resources", "Passed to fm -i, relative to -workdir")
)
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
// Run fm --listTests to find the names of all linked GMs.
stdout, err := exec.RunCwd(ctx, *workdir, *fm, "--listGMs", "-i", *resources)
if err != nil {
td.Fatal(ctx, err)
}
gms := []string{}
scanner := bufio.NewScanner(strings.NewReader(stdout))
for scanner.Scan() {
gms = append(gms, scanner.Text())
}
if err := scanner.Err(); err != nil {
td.Fatal(ctx, err)
}
// Shuffle the GMs randomly as a cheap way to approximate evenly expensive batches.
rand.Shuffle(len(gms), func(i, j int) {
gms[i], gms[j] = gms[j], gms[i]
})
// Round up so there's at least one GM per batch.
limit := runtime.NumCPU()
batch := (len(gms) + limit - 1) / limit
var failures int32 = 0
wg := &sync.WaitGroup{}
util.ChunkIter(len(gms), batch, func(start, end int) error {
cmd := []string{}
cmd = append(cmd, *fm)
cmd = append(cmd, "-i", *resources)
cmd = append(cmd, "-b", "cpu")
cmd = append(cmd, "-s")
cmd = append(cmd, gms[start:end]...)
wg.Add(1)
go func(cmd []string) {
if _, err := exec.RunCwd(ctx, *workdir, cmd...); err != nil {
atomic.AddInt32(&failures, 1);
td.FailStep(ctx, err)
}
wg.Done()
}(cmd)
return nil
})
wg.Wait()
if failures > 0 {
td.Fatalf(ctx, "%v runs of %v failed.", failures, *fm)
}
}

33
infra/bots/tasks.json
View File

@ -727,6 +727,11 @@
],
"trigger": "master"
},
"FM-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All": {
"tasks": [
"FM-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All"
]
},
"Housekeeper-Nightly-RecreateSKPs_Canary": {
"tasks": [
"Housekeeper-Nightly-RecreateSKPs_Canary"
@ -14619,6 +14624,34 @@
"perf"
]
},
"FM-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All": {
"command": [
"./fm_driver",
"--local=false",
"--resources=skia/resources",
"--project_id",
"skia-swarming-bots",
"--task_id",
"<(TASK_ID)",
"--task_name",
"FM-Debian10-Clang-GCE-CPU-AVX2-x86_64-Debug-All"
],
"dependencies": [
"Build-Debian10-Clang-x86_64-Debug",
"Housekeeper-PerCommit-BuildTaskDrivers"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-highcpu-64",
"os:Debian-10.3",
"pool:Skia"
],
"expiration_ns": 900000000000,
"isolate": "test_skia_bundled.isolate",
"max_attempts": 1,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Housekeeper-Nightly-RecreateSKPs_Canary": {
"caches": [
{