From e33845317bf21e47713bdf819387a1c6e66ab787 Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Mon, 26 Jul 2021 08:14:12 -0400 Subject: [PATCH] [infra] Remove CIFuzz task Don't have the cycles to maintain the build. Change-Id: Ia06cd78f5920091fcad250391a4d91b039e97f5d Reviewed-on: https://skia-review.googlesource.com/c/skia/+/432777 Reviewed-by: Kevin Lubick --- infra/bots/gen_tasks_logic/gen_tasks_logic.go | 26 -- infra/bots/gen_tasks_logic/job_builder.go | 7 - infra/bots/jobs.json | 1 - .../builder_name_schema.json | 9 - .../builder_name_schema.py | 2 - infra/bots/task_drivers/cifuzz/cifuzz.go | 254 ------------------ infra/bots/task_drivers/cifuzz/cifuzz_test.go | 84 ------ infra/bots/tasks.json | 66 ----- 8 files changed, 449 deletions(-) delete mode 100644 infra/bots/task_drivers/cifuzz/cifuzz.go delete mode 100644 infra/bots/task_drivers/cifuzz/cifuzz_test.go diff --git a/infra/bots/gen_tasks_logic/gen_tasks_logic.go b/infra/bots/gen_tasks_logic/gen_tasks_logic.go index e6fe3881dd..1b566f218e 100644 --- a/infra/bots/gen_tasks_logic/gen_tasks_logic.go +++ b/infra/bots/gen_tasks_logic/gen_tasks_logic.go @@ -1784,32 +1784,6 @@ func (b *jobBuilder) puppeteer() { }) } -func (b *jobBuilder) cifuzz() { - b.addTask(b.Name, func(b *taskBuilder) { - b.attempts(1) - b.usesDocker() - b.linuxGceDimensions(MACHINE_TYPE_MEDIUM) - b.cipd(CIPD_PKG_LUCI_AUTH) - b.cipd(specs.CIPD_PKGS_GIT_LINUX_AMD64...) - b.dep(b.buildTaskDrivers("linux", "amd64")) - b.output("cifuzz_out") - b.timeout(60 * time.Minute) - b.cas(CAS_WHOLE_REPO) - b.serviceAccount(b.cfg.ServiceAccountCompile) - b.cmd( - "./cifuzz", - "--project_id", "skia-swarming-bots", - "--task_id", specs.PLACEHOLDER_TASK_ID, - "--task_name", b.Name, - "--git_exe_path", "./cipd_bin_packages/git", - "--out_path", "./cifuzz_out", - "--skia_path", "./skia", - "--work_path", "./cifuzz_work", - "--alsologtostderr", - ) - }) -} - // perf generates a Perf task. func (b *jobBuilder) perf() { compileTaskName := "" diff --git a/infra/bots/gen_tasks_logic/job_builder.go b/infra/bots/gen_tasks_logic/job_builder.go index c8dec7f9bc..0ffc609757 100644 --- a/infra/bots/gen_tasks_logic/job_builder.go +++ b/infra/bots/gen_tasks_logic/job_builder.go @@ -214,13 +214,6 @@ func (b *jobBuilder) genTasksForJob() { return } - // Fuzz bots (aka CIFuzz). See - // https://google.github.io/oss-fuzz/getting-started/continuous-integration/ for more. - if b.role("Fuzz") { - b.cifuzz() - return - } - log.Fatalf("Don't know how to handle job %q", b.Name) } diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index 5daa853ed5..d317ae9fd0 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -195,7 +195,6 @@ {"name": "FM-Win-Clang-GCE-CPU-AVX2-x86_64-Debug-All"}, {"name": "FM-Win-Clang-GCE-CPU-AVX2-x86_64-Debug-All-ASAN"}, {"name": "FM-Win-MSVC-GCE-CPU-AVX2-x86_64-Debug-All"}, - {"name": "Fuzz-Debian10-Clang"}, {"name": "Housekeeper-Nightly-RecreateSKPs_DryRun"}, {"name": "Housekeeper-Nightly-UpdateGoDeps"}, {"name": "Housekeeper-OnDemand-Presubmit", diff --git a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json index e3c1892ef0..e648ca3df5 100644 --- a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json +++ b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json @@ -88,15 +88,6 @@ "extra_config" ] }, - "Fuzz": { - "keys": [ - "os", - "compiler" - ], - "optional_keys": [ - "extra_config" - ] - }, "Upload": { "recurse_roles": [ "Build", diff --git a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.py b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.py index bca502714b..e94391e9f1 100644 --- a/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.py +++ b/infra/bots/recipe_modules/builder_name_schema/builder_name_schema.py @@ -28,7 +28,6 @@ BUILDER_ROLE_INFRA = 'Infra' BUILDER_ROLE_PERF = 'Perf' BUILDER_ROLE_TEST = 'Test' BUILDER_ROLE_FM = 'FM' -BUILDER_ROLE_FUZZ = 'Fuzz' BUILDER_ROLE_UPLOAD = 'Upload' BUILDER_ROLES = (BUILDER_ROLE_BUILD, BUILDER_ROLE_BUILDSTATS, @@ -38,7 +37,6 @@ BUILDER_ROLES = (BUILDER_ROLE_BUILD, BUILDER_ROLE_PERF, BUILDER_ROLE_TEST, BUILDER_ROLE_FM, - BUILDER_ROLE_FUZZ, BUILDER_ROLE_UPLOAD) diff --git a/infra/bots/task_drivers/cifuzz/cifuzz.go b/infra/bots/task_drivers/cifuzz/cifuzz.go deleted file mode 100644 index 5e2128e2e7..0000000000 --- a/infra/bots/task_drivers/cifuzz/cifuzz.go +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package main - -import ( - "context" - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "go.skia.org/infra/go/exec" - "go.skia.org/infra/go/git/git_common" - "go.skia.org/infra/go/skerr" - "go.skia.org/infra/go/sklog" - "go.skia.org/infra/task_driver/go/lib/os_steps" - "go.skia.org/infra/task_driver/go/td" -) - -var sleepOnFail = flag.Bool("sleep_on_fail", false, "True if we should sleep for 30 minutes on failure instead of exiting (for inspection via SSH)") - -func main() { - var ( - // Required properties for this task. - fuzzDuration = flag.Duration("fuzz_duration", 600*time.Second, "The total time that the fuzzers run. Divided up between all fuzzers.") - gitExePath = flag.String("git_exe_path", "", "Path to a git exe. Used to checkout cifuzz repo.") - outPath = flag.String("out_path", "", "The directory to put any crashes/hangs/outputs found.") - projectID = flag.String("project_id", "", "ID of the Google Cloud project.") - skiaPath = flag.String("skia_path", "", "Path to skia repo root.") - taskID = flag.String("task_id", "", "task id this data was generated on") - taskName = flag.String("task_name", "", "Name of the task.") - workPath = flag.String("work_path", "", "The directory to use to store temporary files (e.g. fuzzers)") - - // Debugging flags. - local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)") - outputSteps = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") - ) - - // Setup. - ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local) - defer td.EndRun(ctx) - - // Absolute paths work more consistently than relative paths. - gitAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *gitExePath, "git_exe_path") - outAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *outPath, "out_path") - skiaAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *skiaPath, "skia_path") - workAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *workPath, "work_path") - - if !git_common.IsFromCIPD(gitAbsPath) { - fatalOrSleep(ctx, skerr.Fmt("Git %s must be from CIPD", gitAbsPath)) - } - - workDir := filepath.Join(workAbsPath, "cifuzz") - if err := os_steps.MkdirAll(ctx, workDir); err != nil { - fatalOrSleep(ctx, skerr.Wrap(err)) - } - - // Setup cifuzz repo and images - if err := setupCIFuzzRepoAndDocker(ctx, workDir, gitAbsPath); err != nil { - fatalOrSleep(ctx, skerr.Wrap(err)) - } - - // Prepare the skia checkout to be built with fuzzers. - if err := prepareSkiaCheckout(ctx, skiaAbsPath, workDir, gitAbsPath); err != nil { - td.Fatal(ctx, skerr.Wrap(err)) - } - - // build and run fuzzers. If it fails (errors), hold that until we cleanup and copy the output. - // That way, developers can have access to the crashes. - runErr := buildAndRunCIFuzz(ctx, workDir, skiaAbsPath, *fuzzDuration) - - if err := extractOutput(ctx, workDir, outAbsPath); err != nil { - fatalOrSleep(ctx, skerr.Wrap(err)) - } - - // Clean up compiled fuzzers, etc - if err := os_steps.RemoveAll(ctx, workDir); err != nil { - fatalOrSleep(ctx, skerr.Wrap(err)) - } - - if runErr != nil { - fatalOrSleep(ctx, skerr.Wrap(runErr)) - } -} - -func fatalOrSleep(ctx context.Context, err error) { - if *sleepOnFail { - sklog.Errorf("Sleeping after error: %s", err) - time.Sleep(30 * time.Minute) - } - td.Fatal(ctx, err) -} - -const ( - ossFuzzRepo = "https://github.com/google/oss-fuzz.git" - swiftShaderRepo = "https://swiftshader.googlesource.com/SwiftShader" - dockerExe = "docker" - cifuzzDockerImage = "gcr.io/oss-fuzz-base/cifuzz-base:latest" - buildFuzzersDockerImage = "local_build_fuzzers" - runFuzzersDockerImage = "local_run_fuzzers" -) - -func setupCIFuzzRepoAndDocker(ctx context.Context, workdir, gitAbsPath string) error { - ctx = td.StartStep(ctx, td.Props("setup cifuzz").Infra()) - defer td.EndStep(ctx) - - // Make these directories for cifuzz exist so docker does not create it w/ root permissions. - if err := os_steps.MkdirAll(ctx, filepath.Join(workdir, "out")); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - if _, err := exec.RunCwd(ctx, workdir, gitAbsPath, "clone", ossFuzzRepo, "--depth", "1"); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - if _, err := exec.RunCwd(ctx, workdir, dockerExe, "pull", cifuzzDockerImage); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - ossFuzzDir := filepath.Join(workdir, "oss-fuzz", "infra") - - if _, err := exec.RunCwd(ctx, ossFuzzDir, dockerExe, "build", "--tag", buildFuzzersDockerImage, - "--file", "build_fuzzers.Dockerfile", "."); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - if _, err := exec.RunCwd(ctx, ossFuzzDir, dockerExe, "build", "--tag", runFuzzersDockerImage, - "--file", "run_fuzzers.Dockerfile", "."); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - return nil -} - -func prepareSkiaCheckout(ctx context.Context, skiaAbsPath, workDir, gitAbsPath string) error { - ctx = td.StartStep(ctx, td.Props("prepare skia checkout for build").Infra()) - defer td.EndStep(ctx) - - swiftshaderDir := filepath.Join(skiaAbsPath, "third_party", "externals", "swiftshader") - - if _, err := exec.RunCwd(ctx, workDir, "rm", "-rf", swiftshaderDir); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - // We have to clone swiftshader *and* its deps (which are not DEPS, but git submodules) in order - // to build it with fuzzers. - if _, err := exec.RunCwd(ctx, skiaAbsPath, gitAbsPath, "clone", "--recursive", swiftShaderRepo, swiftshaderDir); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - return nil -} - -func buildAndRunCIFuzz(ctx context.Context, workDir, skiaAbsPath string, duration time.Duration) error { - ctx = td.StartStep(ctx, td.Props("build skia fuzzers and run them")) - defer td.EndStep(ctx) - - // See https://google.github.io/oss-fuzz/getting-started/continuous-integration/#optional-configuration - if _, err := exec.RunCwd(ctx, workDir, dockerExe, "run", - "--name", "build_fuzzers", "--rm", - "--env", "MANUAL_SRC_PATH="+skiaAbsPath, - "--env", "OSS_FUZZ_PROJECT_NAME=skia", - "--env", "GITHUB_WORKSPACE="+workDir, - "--env", "GITHUB_REPOSITORY=skia", // TODO(metzman) make this not required - "--env", "GITHUB_EVENT_NAME=push", // TODO(metzman) make this not required - "--env", "DRY_RUN=false", - "--env", "CI=true", - "--env", "CIFUZZ=true", - "--env", "SANITIZER=address", - "--env", "GITHUB_SHA=does_nothing", - "--volume", "/var/run/docker.sock:/var/run/docker.sock", - "--mount", fmt.Sprintf("type=bind,source=%s,destination=%s", skiaAbsPath, skiaAbsPath), - "--mount", fmt.Sprintf("type=bind,source=%s,destination=%s", workDir, workDir), - buildFuzzersDockerImage, - ); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - args := []string{"run", - "--name", "run_fuzzers", "--rm", - "--env", "OSS_FUZZ_PROJECT_NAME=skia", - "--env", "GITHUB_WORKSPACE=" + workDir, - "--env", "GITHUB_REPOSITORY=skia", // TODO(metzman) make this not required - "--env", "GITHUB_EVENT_NAME=push", // TODO(metzman) make this not required - "--env", "DRY_RUN=false", - "--env", "CI=true", - "--env", "CIFUZZ=true", - "--env", "FUZZ_TIME=" + fmt.Sprintf("%d", duration/time.Second), // This is split up between all affected fuzzers. - "--env", "SANITIZER=address", - "--env", "GITHUB_SHA=does_nothing", - "--volume", "/var/run/docker.sock:/var/run/docker.sock", - "--mount", fmt.Sprintf("type=bind,source=%s,destination=%s", workDir, workDir), - runFuzzersDockerImage, - } - - cmd := exec.Command{ - Name: dockerExe, - Args: args, - Dir: workDir, - Timeout: duration + 10*time.Minute, // Give a little padding in case fuzzing takes some extra time. - } - if _, err := exec.RunCommand(ctx, &cmd); err != nil { - if !exec.IsTimeout(err) { - return td.FailStep(ctx, skerr.Wrap(err)) - } else { - sklog.Warningf("Fuzzing timed out: %s", err) - } - } - return nil -} - -func extractOutput(ctx context.Context, workDir, outAbsPath string) error { - ctx = td.StartStep(ctx, td.Props("copy output directory").Infra()) - defer td.EndStep(ctx) - - cifuzzOutDir := filepath.Join(workDir, "out") - - // Fix up permissions of output directory (we need to delete extra folders here so we can - // clean up after we copy out the crash/hang files). - if _, err := exec.RunCwd(ctx, workDir, dockerExe, "run", - "--mount", fmt.Sprintf("type=bind,source=%s,destination=/OUT", cifuzzOutDir), - cifuzzDockerImage, - "/bin/bash", "-c", `rm -rf /OUT/*/ && chmod 0666 /OUT/*`, - ); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - // Make these directories for cifuzz exist so docker does not create it w/ root permissions. - if err := os_steps.MkdirAll(ctx, outAbsPath); err != nil { - return td.FailStep(ctx, skerr.Wrap(err)) - } - - files, err := os_steps.ReadDir(ctx, cifuzzOutDir) - if err != nil { - return td.FailStep(ctx, skerr.Wrapf(err, "getting output from %s", cifuzzOutDir)) - } - - for _, f := range files { - name := f.Name() - if strings.Contains(name, "crash-") || strings.Contains(name, "oom-") || strings.Contains(name, "timeout-") { - oldFile := filepath.Join(cifuzzOutDir, name) - newFile := filepath.Join(outAbsPath, name) - if err := os.Rename(oldFile, newFile); err != nil { - return td.FailStep(ctx, skerr.Wrapf(err, "copying %s to %s", oldFile, newFile)) - } - } - } - - return nil -} diff --git a/infra/bots/task_drivers/cifuzz/cifuzz_test.go b/infra/bots/task_drivers/cifuzz/cifuzz_test.go deleted file mode 100644 index b8f91f0193..0000000000 --- a/infra/bots/task_drivers/cifuzz/cifuzz_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package main - -import ( - "context" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "go.skia.org/infra/go/testutils/unittest" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.skia.org/infra/task_driver/go/td" -) - -func TestSetupCIFuzzRepoAndDocker_Success(t *testing.T) { - unittest.LinuxOnlyTest(t) - base := t.TempDir() - fakeWorkDir := filepath.Join(base, "work") - gb, err := exec.Command("which", "git").Output() - require.NoError(t, err) - gitExe := strings.TrimSpace(string(gb)) - res := td.RunTestSteps(t, false, func(ctx context.Context) error { - return setupCIFuzzRepoAndDocker(ctx, fakeWorkDir, gitExe) - }) - require.Empty(t, res.Errors) - require.Empty(t, res.Exceptions) - require.Len(t, res.Steps, 1) - step := res.Steps[0] - assert.Equal(t, td.StepResultSuccess, step.Result) - assert.Empty(t, step.Errors) - assert.Empty(t, step.Exceptions) - // Make sure the output directory is created - assert.DirExists(t, filepath.Join(fakeWorkDir, "out")) -} - -func TestExtractOutput_OnlyCopyFuzzOutputs(t *testing.T) { - base := t.TempDir() - fakeWorkDir := filepath.Join(base, "work") - fakeCIFuzzOut := filepath.Join(fakeWorkDir, "out") - fakeOutput := filepath.Join(base, "swarming_out") - - err := os.MkdirAll(fakeCIFuzzOut, 0777) - require.NoError(t, err) - - touch(t, filepath.Join(fakeCIFuzzOut, "crash-1234")) - touch(t, filepath.Join(fakeCIFuzzOut, "oom-1234")) - touch(t, filepath.Join(fakeCIFuzzOut, "timeout-1234")) - touch(t, filepath.Join(fakeCIFuzzOut, "a_fuzz_executable_that_should_not_be_copied")) - - _, err = os.Stat(fakeOutput) - require.Error(t, err) - - res := td.RunTestSteps(t, false, func(ctx context.Context) error { - return extractOutput(ctx, fakeWorkDir, fakeOutput) - }) - require.Empty(t, res.Errors) - require.Empty(t, res.Exceptions) - require.Len(t, res.Steps, 1) - step := res.Steps[0] - assert.Equal(t, td.StepResultSuccess, step.Result) - assert.Empty(t, step.Errors) - assert.Empty(t, step.Exceptions) - - copiedFiles, err := ioutil.ReadDir(fakeOutput) - require.NoError(t, err) - fileNames := make([]string, 0, len(copiedFiles)) - for _, f := range copiedFiles { - fileNames = append(fileNames, f.Name()) - } - assert.ElementsMatch(t, []string{"crash-1234", "oom-1234", "timeout-1234"}, fileNames) -} - -func touch(t *testing.T, file string) { - err := ioutil.WriteFile(file, []byte("Whatever"), 0666) - require.NoError(t, err) -} diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 57439be8cf..afb2a36976 100755 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -798,11 +798,6 @@ "FM-Win-MSVC-GCE-CPU-AVX2-x86_64-Debug-All" ] }, - "Fuzz-Debian10-Clang": { - "tasks": [ - "Fuzz-Debian10-Clang" - ] - }, "Housekeeper-Nightly-RecreateSKPs_DryRun": { "tasks": [ "Housekeeper-Nightly-RecreateSKPs_DryRun" @@ -18170,67 +18165,6 @@ "max_attempts": 1, "service_account": "skia-external-gm-uploader@skia-swarming-bots.iam.gserviceaccount.com" }, - "Fuzz-Debian10-Clang": { - "casSpec": "whole-repo", - "cipd_packages": [ - { - "name": "infra/3pp/tools/git/linux-amd64", - "path": "cipd_bin_packages", - "version": "version:2@2.32.0.chromium.6" - }, - { - "name": "infra/tools/git/${platform}", - "path": "cipd_bin_packages", - "version": "git_revision:cadd7dad6e9466c9988ec9eaed70210f2aa1ca86" - }, - { - "name": "infra/tools/luci-auth/${platform}", - "path": "cipd_bin_packages", - "version": "git_revision:cadd7dad6e9466c9988ec9eaed70210f2aa1ca86" - }, - { - "name": "infra/tools/luci/git-credential-luci/${platform}", - "path": "cipd_bin_packages", - "version": "git_revision:cadd7dad6e9466c9988ec9eaed70210f2aa1ca86" - } - ], - "command": [ - "./cifuzz", - "--project_id", - "skia-swarming-bots", - "--task_id", - "<(TASK_ID)", - "--task_name", - "Fuzz-Debian10-Clang", - "--git_exe_path", - "./cipd_bin_packages/git", - "--out_path", - "./cifuzz_out", - "--skia_path", - "./skia", - "--work_path", - "./cifuzz_work", - "--alsologtostderr" - ], - "dependencies": [ - "Housekeeper-PerCommit-BuildTaskDrivers_linux_amd64" - ], - "dimensions": [ - "docker_installed:true", - "cpu:x86-64-Haswell_GCE", - "gpu:none", - "machine_type:n1-standard-16", - "os:Debian-10.3", - "pool:Skia" - ], - "execution_timeout_ns": 3600000000000, - "io_timeout_ns": 3600000000000, - "max_attempts": 1, - "outputs": [ - "cifuzz_out" - ], - "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com" - }, "Housekeeper-Nightly-RecreateSKPs_DryRun": { "caches": [ {