diff --git a/infra/bots/build_task_drivers.sh b/infra/bots/build_task_drivers.sh index d10c7a712e..b757945f47 100755 --- a/infra/bots/build_task_drivers.sh +++ b/infra/bots/build_task_drivers.sh @@ -22,8 +22,6 @@ export GOFLAGS="-mod=readonly" go mod download go install -v go.skia.org/infra/infra/bots/task_drivers/build_push_docker_image go install -v go.skia.org/infra/infra/bots/task_drivers/canary -go install -v go.skia.org/infra/infra/bots/task_drivers/push_apps_from_skia_image -go install -v go.skia.org/infra/infra/bots/task_drivers/push_apps_from_wasm_image go install -v go.skia.org/infra/infra/bots/task_drivers/update_go_deps goos=$2 diff --git a/infra/bots/gen_tasks_logic/gen_tasks_logic.go b/infra/bots/gen_tasks_logic/gen_tasks_logic.go index 5dcc8e6faa..aaa004b6fc 100644 --- a/infra/bots/gen_tasks_logic/gen_tasks_logic.go +++ b/infra/bots/gen_tasks_logic/gen_tasks_logic.go @@ -1103,8 +1103,6 @@ func (b *jobBuilder) createPushAppsFromSkiaDockerImage() { "--task_id", specs.PLACEHOLDER_TASK_ID, "--task_name", b.Name, "--workdir", ".", - "--gerrit_project", "buildbot", - "--gerrit_url", "https://skia-review.googlesource.com", "--repo", specs.PLACEHOLDER_REPO, "--revision", specs.PLACEHOLDER_REVISION, "--patch_issue", specs.PLACEHOLDER_ISSUE, @@ -1134,8 +1132,6 @@ func (b *jobBuilder) createPushAppsFromWASMDockerImage() { "--task_id", specs.PLACEHOLDER_TASK_ID, "--task_name", b.Name, "--workdir", ".", - "--gerrit_project", "buildbot", - "--gerrit_url", "https://skia-review.googlesource.com", "--repo", specs.PLACEHOLDER_REPO, "--revision", specs.PLACEHOLDER_REVISION, "--patch_issue", specs.PLACEHOLDER_ISSUE, diff --git a/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go b/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go new file mode 100644 index 0000000000..3459214fcd --- /dev/null +++ b/infra/bots/task_drivers/push_apps_from_skia_image/push_apps_from_skia_image.go @@ -0,0 +1,188 @@ +// Copyright 2021 Google Inc. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// This executable builds the Docker images based off the Skia executables in the +// gcr.io/skia-public/skia-release image. It then issues a PubSub notification to have those apps +// tagged and deployed by docker_pushes_watcher. +// See //docker_pushes_watcher/README.md in the infra repo for more. +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "os" + "path" + + "cloud.google.com/go/pubsub" + "google.golang.org/api/option" + + "go.skia.org/infra/go/auth" + docker_pubsub "go.skia.org/infra/go/docker/build/pubsub" + "go.skia.org/infra/go/util" + "go.skia.org/infra/task_driver/go/lib/auth_steps" + "go.skia.org/infra/task_driver/go/lib/checkout" + "go.skia.org/infra/task_driver/go/lib/docker" + "go.skia.org/infra/task_driver/go/lib/golang" + "go.skia.org/infra/task_driver/go/lib/os_steps" + "go.skia.org/infra/task_driver/go/td" +) + +var ( + // Required properties for this task. + projectId = flag.String("project_id", "", "ID of the Google Cloud project.") + taskId = flag.String("task_id", "", "ID of this task.") + taskName = flag.String("task_name", "", "Name of the task.") + workdir = flag.String("workdir", ".", "Working directory") + + checkoutFlags = checkout.SetupFlags(nil) + + // Optional flags. + local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") + output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") +) + +const ( + fiddlerImageName = "fiddler" + apiImageName = "api" +) + +var ( + infraCommonEnv = []string{ + "SKIP_BUILD=1", + "ROOT=/OUT", + } + infraCommonBuildArgs = map[string]string{ + "SKIA_IMAGE_NAME": "skia-release", + } +) + +func buildPushFiddlerImage(ctx context.Context, tag, repo, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", fiddlerImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/fiddlek && ./build_fiddler_release"} + volumes := []string{fmt.Sprintf("%s:/OUT", tempDir)} + err = docker.BuildPushImageFromInfraImage(ctx, "Fiddler", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, infraCommonBuildArgs) + if err != nil { + return err + } + return cleanupTempFiles(ctx, configDir, volumes) +} + +func cleanupTempFiles(ctx context.Context, configDir string, volumes []string) error { + // Remove all temporary files from the host machine. Swarming gets upset if there are root-owned + // files it cannot clean up. + const infraImageWithTag = "gcr.io/skia-public/infra:prod" + cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"} + return docker.Run(ctx, infraImageWithTag, configDir, cleanupCmd, volumes, nil) +} + +func buildPushApiImage(ctx context.Context, tag, repo, configDir, checkoutDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + // Change perms of the directory for doxygen to be able to write to it. + if err := os.Chmod(tempDir, 0777); err != nil { + return err + } + // Run Doxygen pointing to the location of the checkout and the out dir. + volumes := []string{ + fmt.Sprintf("%s:/OUT", tempDir), + fmt.Sprintf("%s:/CHECKOUT", checkoutDir), + } + env := []string{ + "OUTPUT_DIRECTORY=/OUT", + } + doxygenCmd := []string{"/bin/sh", "-c", "cd /CHECKOUT/tools/doxygen && doxygen ProdDoxyfile"} + if err := docker.Run(ctx, "gcr.io/skia-public/doxygen:testing-slim", configDir, doxygenCmd, volumes, env); err != nil { + return err + } + + image := fmt.Sprintf("gcr.io/skia-public/%s", apiImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/api && make release_ci"} + infraEnv := util.CopyStringSlice(infraCommonEnv) + infraEnv = append(infraEnv, "DOXYGEN_HTML=/OUT/html") + infraVolumes := []string{fmt.Sprintf("%s:/OUT", tempDir)} + err = docker.BuildPushImageFromInfraImage(ctx, "Api", image, tag, repo, configDir, tempDir, "prod", topic, cmd, infraVolumes, infraEnv, infraCommonBuildArgs) + if err != nil { + return err + } + return cleanupTempFiles(ctx, configDir, volumes) +} + +func main() { + // Setup. + ctx := td.StartRun(projectId, taskId, taskName, output, local) + defer td.EndRun(ctx) + + rs, err := checkout.GetRepoState(checkoutFlags) + if err != nil { + td.Fatal(ctx, err) + } + + wd, err := os_steps.Abs(ctx, *workdir) + if err != nil { + td.Fatal(ctx, err) + } + + // Check out the code. + co, err := checkout.EnsureGitCheckout(ctx, path.Join(wd, "repo"), rs) + if err != nil { + td.Fatal(ctx, err) + } + + // Setup go. + ctx = golang.WithEnv(ctx, wd) + + // Create token source with scope for cloud registry (storage) and pubsub. + ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub) + if err != nil { + td.Fatal(ctx, err) + } + + // Create pubsub client. + client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts)) + if err != nil { + td.Fatal(ctx, err) + } + topic := client.Topic(docker_pubsub.TOPIC) + + // Figure out which tag to use for docker build and push. + tag := rs.Revision + if rs.Issue != "" && rs.Patchset != "" { + tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset) + } + // Add the tag to infraCommonBuildArgs. + infraCommonBuildArgs["SKIA_IMAGE_TAG"] = tag + + // Create a temporary config dir for Docker. + configDir, err := ioutil.TempDir("", "") + if err != nil { + td.Fatal(ctx, err) + } + defer util.RemoveAll(configDir) + + // Login to docker (required to push to docker). + token, err := ts.Token() + if err != nil { + td.Fatal(ctx, err) + } + if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil { + td.Fatal(ctx, err) + } + + // Build and push all apps of interest below. + if err := buildPushFiddlerImage(ctx, tag, rs.Repo, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + if err := buildPushApiImage(ctx, tag, rs.Repo, configDir, co.Dir(), topic); err != nil { + td.Fatal(ctx, err) + } +} diff --git a/infra/bots/task_drivers/push_apps_from_wasm_image/push_apps_from_wasm_image.go b/infra/bots/task_drivers/push_apps_from_wasm_image/push_apps_from_wasm_image.go new file mode 100644 index 0000000000..8a4281a48a --- /dev/null +++ b/infra/bots/task_drivers/push_apps_from_wasm_image/push_apps_from_wasm_image.go @@ -0,0 +1,222 @@ +// Copyright 2021 Google Inc. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// This executable builds the Docker images based off the WASM executables in the +// gcr.io/skia-public/skia-wasm-release image. It then issues a PubSub notification to have those apps +// tagged and deployed by docker_pushes_watcher. +// See //docker_pushes_watcher/README.md in the infra repo for more. +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + + "cloud.google.com/go/pubsub" + "google.golang.org/api/option" + + "go.skia.org/infra/go/auth" + docker_pubsub "go.skia.org/infra/go/docker/build/pubsub" + "go.skia.org/infra/go/util" + "go.skia.org/infra/task_driver/go/lib/auth_steps" + "go.skia.org/infra/task_driver/go/lib/checkout" + "go.skia.org/infra/task_driver/go/lib/docker" + "go.skia.org/infra/task_driver/go/lib/golang" + "go.skia.org/infra/task_driver/go/lib/os_steps" + "go.skia.org/infra/task_driver/go/td" +) + +var ( + // Required properties for this task. + projectId = flag.String("project_id", "", "ID of the Google Cloud project.") + taskId = flag.String("task_id", "", "ID of this task.") + taskName = flag.String("task_name", "", "Name of the task.") + workdir = flag.String("workdir", ".", "Working directory") + + checkoutFlags = checkout.SetupFlags(nil) + + // Optional flags. + local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") + output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") +) + +const ( + debuggerImageName = "debugger-app" + jsfiddleImageName = "jsfiddle" + particlesImageName = "particles" + shaderImageName = "shaders" + skottieImageName = "skottie" +) + +var ( + infraCommonEnv = []string{ + "SKIP_BUILD=1", + "ROOT=/WORKSPACE", + } +) + +func buildPushJsFiddleImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", jsfiddleImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/jsfiddle && make release_ci"} + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + fmt.Sprintf("%s:/WORKSPACE", tempDir), + } + return docker.BuildPushImageFromInfraImage(ctx, "JsFiddle", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil) +} + +func buildPushSkottieImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", skottieImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/skottie && make release_ci"} + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + fmt.Sprintf("%s:/WORKSPACE", tempDir), + } + return docker.BuildPushImageFromInfraImage(ctx, "Skottie", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil) +} + +func buildPushParticlesImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", particlesImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/particles && make release_ci"} + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + fmt.Sprintf("%s:/WORKSPACE", tempDir), + } + return docker.BuildPushImageFromInfraImage(ctx, "Particles", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil) +} + +func buildPushDebuggerImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", debuggerImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/debugger-app && make release_ci"} + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + fmt.Sprintf("%s:/WORKSPACE", tempDir), + } + return docker.BuildPushImageFromInfraImage(ctx, "Debugger-App", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil) +} + +func buildPushShadersImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error { + tempDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + return err + } + image := fmt.Sprintf("gcr.io/skia-public/%s", shaderImageName) + cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/shaders && make release_ci"} + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + fmt.Sprintf("%s:/WORKSPACE", tempDir), + } + return docker.BuildPushImageFromInfraImage(ctx, "Shaders", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil) +} + +func main() { + // Setup. + ctx := td.StartRun(projectId, taskId, taskName, output, local) + defer td.EndRun(ctx) + + rs, err := checkout.GetRepoState(checkoutFlags) + if err != nil { + td.Fatal(ctx, err) + } + + wd, err := os_steps.Abs(ctx, *workdir) + if err != nil { + td.Fatal(ctx, err) + } + + // Setup go. + ctx = golang.WithEnv(ctx, wd) + + // Create token source with scope for cloud registry (storage) and pubsub. + ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub) + if err != nil { + td.Fatal(ctx, err) + } + + // Create pubsub client. + client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts)) + if err != nil { + td.Fatal(ctx, err) + } + topic := client.Topic(docker_pubsub.TOPIC) + + // Figure out which tag to use for docker build and push. + tag := rs.Revision + if rs.Issue != "" && rs.Patchset != "" { + tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset) + } + + // Create a temporary config dir for Docker. + configDir, err := ioutil.TempDir("", "") + if err != nil { + td.Fatal(ctx, err) + } + defer util.RemoveAll(configDir) + + // Login to docker (required to push to docker). + token, err := ts.Token() + if err != nil { + td.Fatal(ctx, err) + } + if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil { + td.Fatal(ctx, err) + } + + // Run skia-wasm-release image and extract wasm products out of it. + wasmProductsDir, err := os_steps.TempDir(ctx, "", "") + if err != nil { + td.Fatal(ctx, err) + } + // Run Doxygen pointing to the location of the checkout and the out dir. + volumes := []string{ + fmt.Sprintf("%s:/OUT", wasmProductsDir), + } + wasmCopyCmd := []string{"/bin/sh", "-c", "cp -r /tmp/* /OUT"} + releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-wasm-release:%s", tag) + if err := docker.Run(ctx, releaseImg, configDir, wasmCopyCmd, volumes, nil); err != nil { + td.Fatal(ctx, err) + } + + // Build and push all apps of interest below. + if err := buildPushJsFiddleImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + if err := buildPushSkottieImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + if err := buildPushParticlesImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + if err := buildPushDebuggerImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + if err := buildPushShadersImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil { + td.Fatal(ctx, err) + } + + // Remove all temporary files from the host machine. Swarming gets upset if there are root-owned + // files it cannot clean up. + cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"} + if err := docker.Run(ctx, releaseImg, configDir, cleanupCmd, volumes, nil); err != nil { + td.Fatal(ctx, err) + } +} diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 8e375fde99..be1f55749d 100755 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -20106,10 +20106,6 @@ "Housekeeper-PerCommit-PushAppsFromSkiaDockerImage", "--workdir", ".", - "--gerrit_project", - "buildbot", - "--gerrit_url", - "https://skia-review.googlesource.com", "--repo", "<(REPO)", "--revision", @@ -20184,10 +20180,6 @@ "Housekeeper-PerCommit-PushAppsFromWASMDockerImage", "--workdir", ".", - "--gerrit_project", - "buildbot", - "--gerrit_url", - "https://skia-review.googlesource.com", "--repo", "<(REPO)", "--revision",