AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[infra] Deploy debugger with Bazel

Browse Source 
Housekeeper-PerCommit-PushAppsFromWASMDockerImage would be a no-op
so it has been deleted.

Requires https://skia-review.googlesource.com/c/buildbot/+/490898/
to land first

Change-Id: I7db91363bba714fb0c1685fc332cd424f72786a9
Bug: skia:10614
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/491038
Reviewed-by: Ravi Mistry <rmistry@google.com>
This commit is contained in:
Kevin Lubick 2022-01-04 15:55:08 -05:00
parent d7771857e9
commit ec2e8f11b9
6 changed files with 6 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -1116,35 +1116,6 @@ func (b *jobBuilder) createPushAppsFromSkiaDockerImage() {
})
}
// createPushAppsFromWASMDockerImage creates and pushes docker images of some apps
// (eg: jsfiddle, skottie, particles) using the skia-wasm-release docker image.
func (b *jobBuilder) createPushAppsFromWASMDockerImage() {
b.addTask(b.Name, func(b *taskBuilder) {
// TODO(borenet): Make this task not use Git.
b.usesGit()
b.cmd(
"./push_apps_from_wasm_image",
"--project_id", "skia-swarming-bots",
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--workdir", ".",
"--repo", specs.PLACEHOLDER_REPO,
"--revision", specs.PLACEHOLDER_REVISION,
"--patch_issue", specs.PLACEHOLDER_ISSUE,
"--patch_set", specs.PLACEHOLDER_PATCHSET,
"--patch_server", specs.PLACEHOLDER_CODEREVIEW_SERVER,
)
b.dep(b.buildTaskDrivers("linux", "amd64"))
b.dep(b.createDockerImage(true))
b.addToPATH("cipd_bin_packages", "cipd_bin_packages/bin")
b.cas(CAS_EMPTY)
b.serviceAccount(b.cfg.ServiceAccountCompile)
b.linuxGceDimensions(MACHINE_TYPE_MEDIUM)
b.usesDocker()
b.cache(CACHES_DOCKER...)
})
}
// createPushBazelAppsFromWASMDockerImage pushes those infra apps that have been ported to Bazel
// and require assets built in the WASM docker image.
// TODO(kjlubick) The inputs to this job should not be the docker build, but a Bazel build.

3
infra/bots/gen_tasks_logic/job_builder.go
View File

@ -121,9 +121,6 @@ func (b *jobBuilder) genTasksForJob() {
if b.extraConfig("PushAppsFromSkiaDockerImage") {
b.createPushAppsFromSkiaDockerImage()
return
} else if b.extraConfig("PushAppsFromWASMDockerImage") {
b.createPushAppsFromWASMDockerImage()
return
} else if b.extraConfig("PushBazelAppsFromWASMDockerImage") {
b.createPushBazelAppsFromWASMDockerImage()
return

1
infra/bots/jobs.json
View File

@ -222,7 +222,6 @@
{"name": "Housekeeper-PerCommit-IsolateSVG"},
{"name": "Housekeeper-PerCommit-IsolateSkImage"},
{"name": "Housekeeper-PerCommit-PushAppsFromSkiaDockerImage"},
{"name": "Housekeeper-PerCommit-PushAppsFromWASMDockerImage"},
{"name": "Housekeeper-PerCommit-PushBazelAppsFromWASMDockerImage"},
{"name": "Housekeeper-PerCommit-RunGnToBp",
"cq_config": {}

165
infra/bots/task_drivers/push_apps_from_wasm_image/push_apps_from_wasm_image.go
View File

@ -1,165 +0,0 @@
// Copyright 2021 Google Inc.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This executable builds the Docker images based off the WASM executables in the
// gcr.io/skia-public/skia-wasm-release image. It then issues a PubSub notification to have those apps
// tagged and deployed by docker_pushes_watcher.
// See //docker_pushes_watcher/README.md in the infra repo for more.
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"cloud.google.com/go/pubsub"
"google.golang.org/api/option"
"go.skia.org/infra/go/auth"
docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
"go.skia.org/infra/go/util"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/checkout"
"go.skia.org/infra/task_driver/go/lib/docker"
"go.skia.org/infra/task_driver/go/lib/golang"
"go.skia.org/infra/task_driver/go/lib/os_steps"
"go.skia.org/infra/task_driver/go/td"
)
var (
// Required properties for this task.
projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
taskId = flag.String("task_id", "", "ID of this task.")
taskName = flag.String("task_name", "", "Name of the task.")
workdir = flag.String("workdir", ".", "Working directory")
checkoutFlags = checkout.SetupFlags(nil)
// Optional flags.
local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)
const (
debuggerImageName = "debugger-app"
skottieImageName = "skottie"
)
var (
infraCommonEnv = []string{
"SKIP_BUILD=1",
"ROOT=/WORKSPACE",
}
)
func buildPushSkottieImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", skottieImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/skottie && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Skottie", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func buildPushDebuggerImage(ctx context.Context, tag, repo, wasmProductsDir, configDir string, topic *pubsub.Topic) error {
tempDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
return err
}
image := fmt.Sprintf("gcr.io/skia-public/%s", debuggerImageName)
cmd := []string{"/bin/sh", "-c", "cd /home/skia/golib/src/go.skia.org/infra/debugger-app && make release_ci"}
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
fmt.Sprintf("%s:/WORKSPACE", tempDir),
}
return docker.BuildPushImageFromInfraImage(ctx, "Debugger-App", image, tag, repo, configDir, tempDir, "prod", topic, cmd, volumes, infraCommonEnv, nil)
}
func main() {
// Setup.
ctx := td.StartRun(projectId, taskId, taskName, output, local)
defer td.EndRun(ctx)
rs, err := checkout.GetRepoState(checkoutFlags)
if err != nil {
td.Fatal(ctx, err)
}
wd, err := os_steps.Abs(ctx, *workdir)
if err != nil {
td.Fatal(ctx, err)
}
// Setup go.
ctx = golang.WithEnv(ctx, wd)
// Create token source with scope for cloud registry (storage) and pubsub.
ts, err := auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail, auth.ScopeFullControl, pubsub.ScopePubSub)
if err != nil {
td.Fatal(ctx, err)
}
// Create pubsub client.
client, err := pubsub.NewClient(ctx, docker_pubsub.TOPIC_PROJECT_ID, option.WithTokenSource(ts))
if err != nil {
td.Fatal(ctx, err)
}
topic := client.Topic(docker_pubsub.TOPIC)
// Figure out which tag to use for docker build and push.
tag := rs.Revision
if rs.Issue != "" && rs.Patchset != "" {
tag = fmt.Sprintf("%s_%s", rs.Issue, rs.Patchset)
}
// Create a temporary config dir for Docker.
configDir, err := ioutil.TempDir("", "")
if err != nil {
td.Fatal(ctx, err)
}
defer util.RemoveAll(configDir)
// Login to docker (required to push to docker).
token, err := ts.Token()
if err != nil {
td.Fatal(ctx, err)
}
if err := docker.Login(ctx, token.AccessToken, "gcr.io/skia-public/", configDir); err != nil {
td.Fatal(ctx, err)
}
// Run skia-wasm-release image and extract wasm products out of it.
wasmProductsDir, err := os_steps.TempDir(ctx, "", "")
if err != nil {
td.Fatal(ctx, err)
}
// Run Doxygen pointing to the location of the checkout and the out dir.
volumes := []string{
fmt.Sprintf("%s:/OUT", wasmProductsDir),
}
wasmCopyCmd := []string{"/bin/sh", "-c", "cp -r /tmp/* /OUT"}
releaseImg := fmt.Sprintf("gcr.io/skia-public/skia-wasm-release:%s", tag)
if err := docker.Run(ctx, releaseImg, configDir, wasmCopyCmd, volumes, nil); err != nil {
td.Fatal(ctx, err)
}
// Build and push all apps of interest below.
if err := buildPushDebuggerImage(ctx, tag, rs.Repo, wasmProductsDir, configDir, topic); err != nil {
td.Fatal(ctx, err)
}
// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
// files it cannot clean up.
cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}
if err := docker.Run(ctx, releaseImg, configDir, cleanupCmd, volumes, nil); err != nil {
td.Fatal(ctx, err)
}
}

9
infra/bots/task_drivers/push_bazel_apps_from_wasm_image/push_bazel_apps_from_wasm_image.go
View File

@ -16,8 +16,6 @@ import (
"fmt"
"path/filepath"
"go.skia.org/infra/task_driver/go/lib/bazel"
"cloud.google.com/go/pubsub"
"google.golang.org/api/option"
@ -25,6 +23,7 @@ import (
docker_pubsub "go.skia.org/infra/go/docker/build/pubsub"
sk_exec "go.skia.org/infra/go/exec"
"go.skia.org/infra/task_driver/go/lib/auth_steps"
"go.skia.org/infra/task_driver/go/lib/bazel"
"go.skia.org/infra/task_driver/go/lib/checkout"
"go.skia.org/infra/task_driver/go/lib/docker"
"go.skia.org/infra/task_driver/go/lib/golang"
@ -124,7 +123,10 @@ func main() {
td.Fatal(ctx, err)
}
// TODO(kjlubick) Build and push all apps of interest as they are ported.
if err := buildPush(ctx, "debugger-app", wasmProductsDir, checkoutDir, *skiaRevision, topic); err != nil {
td.Fatal(ctx, err)
}
if err := buildPush(ctx, "jsfiddle", wasmProductsDir, checkoutDir, *skiaRevision, topic); err != nil {
td.Fatal(ctx, err)
}
@ -140,6 +142,7 @@ func main() {
if err := buildPush(ctx, "skottie", wasmProductsDir, checkoutDir, *skiaRevision, topic); err != nil {
td.Fatal(ctx, err)
}
// Remove all temporary files from the host machine. Swarming gets upset if there are root-owned
// files it cannot clean up.
cleanupCmd := []string{"/bin/sh", "-c", "rm -rf /OUT/*"}

79
infra/bots/tasks.json
View File

@ -889,11 +889,6 @@
"Housekeeper-PerCommit-PushAppsFromSkiaDockerImage"
]
},
"Housekeeper-PerCommit-PushAppsFromWASMDockerImage": {
"tasks": [
"Housekeeper-PerCommit-PushAppsFromWASMDockerImage"
]
},
"Housekeeper-PerCommit-PushBazelAppsFromWASMDockerImage": {
"tasks": [
"Housekeeper-PerCommit-PushBazelAppsFromWASMDockerImage"
@ -20117,80 +20112,6 @@
},
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Housekeeper-PerCommit-PushAppsFromWASMDockerImage": {
"caches": [
{
"name": "docker",
"path": "cache/docker"
},
{
"name": "git",
"path": "cache/git"
},
{
"name": "git_cache",
"path": "cache/git_cache"
}
],
"casSpec": "empty",
"cipd_packages": [
{
"name": "infra/3pp/tools/git/linux-amd64",
"path": "cipd_bin_packages",
"version": "version:2@2.34.1.chromium.6"
},
{
"name": "infra/tools/git/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:749deb20c96c279223a5325eaf327d41ef1716f9"
},
{
"name": "infra/tools/luci/git-credential-luci/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:749deb20c96c279223a5325eaf327d41ef1716f9"
}
],
"command": [
"./push_apps_from_wasm_image",
"--project_id",
"skia-swarming-bots",
"--task_id",
"<(TASK_ID)",
"--task_name",
"Housekeeper-PerCommit-PushAppsFromWASMDockerImage",
"--workdir",
".",
"--repo",
"<(REPO)",
"--revision",
"<(REVISION)",
"--patch_issue",
"<(ISSUE)",
"--patch_set",
"<(PATCHSET)",
"--patch_server",
"<(CODEREVIEW_SERVER)"
],
"dependencies": [
"Housekeeper-PerCommit-BuildTaskDrivers_linux_amd64",
"Housekeeper-PerCommit-CreateDockerImage_Skia_WASM_Release"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-standard-16",
"os:Debian-10.3",
"pool:Skia",
"docker_installed:true"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
]
},
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Housekeeper-PerCommit-PushBazelAppsFromWASMDockerImage": {
"caches": [
{