AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[infra] Move other push task drivers from infra repo

Browse Source 
AFAIK, they aren't being used from the infra repo, so
putting them here will make it easier to work on them
(e.g. as we continue migrating the apps to Bazel).

Change-Id: I983a85a4934c8d849f2b59055554689216983b4c
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/482077
Reviewed-by: Eric Boren <borenet@google.com>
This commit is contained in:
Kevin Lubick 2021-12-09 11:08:26 -05:00
parent c95c53ed0f
commit 3374bcb68a
5 changed files with 410 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
infra/bots/build_task_drivers.sh
View File

@ -22,8 +22,6 @@ export GOFLAGS="-mod=readonly"
go mod download
go install -v go.skia.org/infra/infra/bots/task_drivers/build_push_docker_image
go install -v go.skia.org/infra/infra/bots/task_drivers/canary
go install -v go.skia.org/infra/infra/bots/task_drivers/push_apps_from_skia_image
go install -v go.skia.org/infra/infra/bots/task_drivers/push_apps_from_wasm_image
go install -v go.skia.org/infra/infra/bots/task_drivers/update_go_deps
goos=$2

4
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -1103,8 +1103,6 @@ func (b *jobBuilder) createPushAppsFromSkiaDockerImage() {
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--workdir", ".",
"--gerrit_project", "buildbot",
"--gerrit_url", "https://skia-review.googlesource.com",
"--repo", specs.PLACEHOLDER_REPO,
"--revision", specs.PLACEHOLDER_REVISION,
"--patch_issue", specs.PLACEHOLDER_ISSUE,
@ -1134,8 +1132,6 @@ func (b *jobBuilder) createPushAppsFromWASMDockerImage() {
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--workdir", ".",
"--gerrit_project", "buildbot",
"--gerrit_url", "https://skia-review.googlesource.com",
"--repo", specs.PLACEHOLDER_REPO,
"--revision", specs.PLACEHOLDER_REVISION,
"--patch_issue", specs.PLACEHOLDER_ISSUE,

188
infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go Normal file
View File

@ -0,0 +1,188 @@
// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This executable builds the Docker images based off the Skia executables in the
// gcr.io/skia-public/skia-release image. It then issues a PubSub notification to have those apps
// tagged and deployed by docker_pushes_watcher.
// See //docker_pushes_watcher/README.md in the infra repo for more.
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"os"
"path"
"cloud.google.com/go/pubsub"
"google.golang.org/api/option"
"go.skia.org/infra/go/auth"
docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
"go.skia.org/infra/go/util"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/checkout"
"go.skia.org/infra/task_driver/go/lib/docker"
"go.skia.org/infra/task_driver/go/lib/golang"
"go.skia.org/infra/task_driver/go/lib/os_steps"
"go.skia.org/infra/task_driver/go/td"
)
var (
// Required properties for this task.
projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
taskId = flag.String("task_id", "", "ID of this task.")
taskName = flag.String("task_name", "", "Name of the task.")
workdir = flag.String("workdir", ".", "Working directory")
checkoutFlags = checkout.SetupFlags(nil)
// Optional flags.
local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)
const (
fiddlerImageName = "fiddler"
apiImageName = "api"
)
var (
infraCommonEnv = []string{
"SKIP_BUILD=1",
"ROOT=/OUT",
}
infraCommonBuildArgs = map[string]string{
"SKIA_IMAGE_NAME": "skia-release",
}
)
func buildPushFiddlerImage(ctx context.Context, tag, repo, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", fiddlerImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/fiddlek && ./build_fiddler_release"}
volumes := []string{fmt.Sprintf("%s:/OUT", tempDir)}
err = docker.BuildPushImageFromInfraImage(ctx, "Fiddler", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, infraCommonBuildArgs)
if err != nil {
return err
}
return cleanupTempFiles(ctx, configDir, volumes)
}
func cleanupTempFiles(ctx context.Context, configDir string, volumes []string) error {
// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
// files it cannot clean up.
const infraImageWithTag = "gcr.io/skia-public/infra:prod"
cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
return docker.Run(ctx, infraImageWithTag, configDir, cleanupCmd, volumes, nil)
}
func buildPushApiImage(ctx context.Context, tag, repo, configDir, checkoutDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
// Change perms of the directory for doxygen to be able to write to it.
if err := os.Chmod(tempDir, 0777); err != nil {
return err
}
// Run Doxygen pointing to the location of the checkout and the out dir.
volumes := []string{
fmt.Sprintf("%s:/OUT", tempDir),
fmt.Sprintf("%s:/CHECKOUT", checkoutDir),
}
env := []string{
"OUTPUT_DIRECTORY=/OUT",
}
doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"}
if err := docker.Run(ctx, "gcr.io/skia-public/doxygen:testing-slim", configDir, doxygenCmd, volumes, env); err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", apiImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/api && make release_ci"}
infraEnv := util.CopyStringSlice(infraCommonEnv)
infraEnv = append(infraEnv, "DOXYGEN_HTML=/OUT/html")
infraVolumes := []string{fmt.Sprintf("%s:/OUT", tempDir)}
err = docker.BuildPushImageFromInfraImage(ctx, "Api", image, tag, repo, configDir, tempDir, "prod", topic, cmd, infraVolumes, infraEnv, infraCommonBuildArgs)
if err != nil {
return err
}
return cleanupTempFiles(ctx, configDir, volumes)
}
func main() {
// Setup.
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
rs, err := checkout.GetRepoState(checkoutFlags)
if err != nil {
td.Fatal(ctx, err)
}
wd, err := os_steps.Abs(ctx, *workdir)
if err != nil {
td.Fatal(ctx, err)
}
// Check out the code.
co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs)
if err != nil {
td.Fatal(ctx, err)
}
// Setup go.
ctx = golang.WithEnv(ctx, wd)
// Create token source with scope for cloud registry (storage) and pubsub.
ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
if err != nil {
td.Fatal(ctx, err)
}
// Create pubsub client.
client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts))
if err != nil {
td.Fatal(ctx, err)
}
topic := client.Topic(docker_pubsub.TOPIC)
// Figure out which tag to use for docker build and push.
tag := rs.Revision
if rs.Issue != "" && rs.Patchset != "" {
tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
}
// Add the tag to infraCommonBuildArgs.
infraCommonBuildArgs["SKIA_IMAGE_TAG"] = tag
// Create a temporary config dir for Docker.
configDir, err := ioutil.TempDir("", "")
if err != nil {
td.Fatal(ctx, err)
}
defer util.RemoveAll(configDir)
// Login to docker (required to push to docker).
token, err := ts.Token()
if err != nil {
td.Fatal(ctx, err)
}
if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil {
td.Fatal(ctx, err)
}
// Build and push all apps of interest below.
if err := buildPushFiddlerImage(ctx, tag, rs.Repo, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPushApiImage(ctx, tag, rs.Repo, configDir, co.Dir(), topic); err != nil {
td.Fatal(ctx, err)
}
}

222
infra/bots/task_drivers/push_apps_from_wasm_image/push_apps_from_wasm_image.go Normal file
View File

@ -0,0 +1,222 @@
// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This executable builds the Docker images based off the WASM executables in the
// gcr.io/skia-public/skia-wasm-release image. It then issues a PubSub notification to have those apps
// tagged and deployed by docker_pushes_watcher.
// See //docker_pushes_watcher/README.md in the infra repo for more.
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"cloud.google.com/go/pubsub"
"google.golang.org/api/option"
"go.skia.org/infra/go/auth"
docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
"go.skia.org/infra/go/util"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/checkout"
"go.skia.org/infra/task_driver/go/lib/docker"
"go.skia.org/infra/task_driver/go/lib/golang"
"go.skia.org/infra/task_driver/go/lib/os_steps"
"go.skia.org/infra/task_driver/go/td"
)
var (
// Required properties for this task.
projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
taskId = flag.String("task_id", "", "ID of this task.")
taskName = flag.String("task_name", "", "Name of the task.")
workdir = flag.String("workdir", ".", "Working directory")
checkoutFlags = checkout.SetupFlags(nil)
// Optional flags.
local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)
const (
debuggerImageName = "debugger-app"
jsfiddleImageName = "jsfiddle"
particlesImageName = "particles"
shaderImageName = "shaders"
skottieImageName = "skottie"
)
var (
infraCommonEnv = []string{
"SKIP_BUILD=1",
"ROOT=/WORKSPACE",
}
)
func buildPushJsFiddleImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", jsfiddleImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/jsfiddle && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "JsFiddle", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func buildPushSkottieImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", skottieImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/skottie && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Skottie", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func buildPushParticlesImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", particlesImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/particles && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Particles", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func buildPushDebuggerImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", debuggerImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/debugger-app && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Debugger-App", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func buildPushShadersImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", shaderImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/shaders && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Shaders", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func main() {
// Setup.
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
rs, err := checkout.GetRepoState(checkoutFlags)
if err != nil {
td.Fatal(ctx, err)
}
wd, err := os_steps.Abs(ctx, *workdir)
if err != nil {
td.Fatal(ctx, err)
}
// Setup go.
ctx = golang.WithEnv(ctx, wd)
// Create token source with scope for cloud registry (storage) and pubsub.
ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
if err != nil {
td.Fatal(ctx, err)
}
// Create pubsub client.
client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts))
if err != nil {
td.Fatal(ctx, err)
}
topic := client.Topic(docker_pubsub.TOPIC)
// Figure out which tag to use for docker build and push.
tag := rs.Revision
if rs.Issue != "" && rs.Patchset != "" {
tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
}
// Create a temporary config dir for Docker.
configDir, err := ioutil.TempDir("", "")
if err != nil {
td.Fatal(ctx, err)
}
defer util.RemoveAll(configDir)
// Login to docker (required to push to docker).
token, err := ts.Token()
if err != nil {
td.Fatal(ctx, err)
}
if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil {
td.Fatal(ctx, err)
}
// Run skia-wasm-release image and extract wasm products out of it.
wasmProductsDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
td.Fatal(ctx, err)
}
// Run Doxygen pointing to the location of the checkout and the out dir.
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
}
wasmCopyCmd := []string{"/bin/sh", "-c", "cp -r /tmp/* /OUT"}
releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-wasm-release:%s", tag)
if err := docker.Run(ctx, releaseImg, configDir, wasmCopyCmd, volumes, nil); err != nil {
td.Fatal(ctx, err)
}
// Build and push all apps of interest below.
if err := buildPushJsFiddleImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPushSkottieImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPushParticlesImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPushDebuggerImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPushShadersImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
// files it cannot clean up.
cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
if err := docker.Run(ctx, releaseImg, configDir, cleanupCmd, volumes, nil); err != nil {
td.Fatal(ctx, err)
}
}

8
infra/bots/tasks.json
View File

@ -20106,10 +20106,6 @@
"Housekeeper-PerCommit-PushAppsFromSkiaDockerImage",
"--workdir",
".",
"--gerrit_project",
"buildbot",
"--gerrit_url",
"https://skia-review.googlesource.com",
"--repo",
"<(REPO)",
"--revision",
@ -20184,10 +20180,6 @@
"Housekeeper-PerCommit-PushAppsFromWASMDockerImage",
"--workdir",
".",
"--gerrit_project",
"buildbot",
"--gerrit_url",
"https://skia-review.googlesource.com",
"--repo",
"<(REPO)",
"--revision",