// Copyright 2016 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. package main /* Generate the tasks.json file. */ import ( "encoding/json" "fmt" "os" "path" "sort" "strings" "time" "github.com/skia-dev/glog" "go.skia.org/infra/go/util" "go.skia.org/infra/task_scheduler/go/specs" ) const ( DEFAULT_OS = DEFAULT_OS_LINUX DEFAULT_OS_LINUX = "Ubuntu-14.04" // Pool for Skia bots. POOL_SKIA = "Skia" // Name prefix for upload jobs. PREFIX_UPLOAD = "Upload" ) var ( // "Constants" // Top-level list of all jobs to run at each commit. JOBS = []string{ "Build-Mac-Clang-Arm7-Debug-iOS", "Build-Mac-Clang-Arm7-Release-iOS", "Build-Mac-Clang-arm64-Debug-Android", "Build-Mac-Clang-arm64-Debug-GN_iOS", "Build-Mac-Clang-x86_64-Debug", "Build-Mac-Clang-x86_64-Debug-CommandBuffer", "Build-Mac-Clang-x86_64-Release", "Build-Mac-Clang-x86_64-Release-CommandBuffer", "Build-Ubuntu-Clang-arm-Debug-Android", "Build-Ubuntu-Clang-arm-Release-Android", "Build-Ubuntu-Clang-arm64-Debug-Android", "Build-Ubuntu-Clang-arm64-Debug-Android_FrameworkDefs", "Build-Ubuntu-Clang-arm64-Debug-Android_Vulkan", "Build-Ubuntu-Clang-arm64-Release-Android", "Build-Ubuntu-Clang-arm64-Release-Android_Vulkan", "Build-Ubuntu-Clang-mips64el-Debug-Android", "Build-Ubuntu-Clang-mips64el-Release-Android", "Build-Ubuntu-Clang-mipsel-Debug-Android", "Build-Ubuntu-Clang-mipsel-Release-Android", "Build-Ubuntu-Clang-x64-Debug-Android", "Build-Ubuntu-Clang-x64-Release-Android", "Build-Ubuntu-Clang-x86-Debug-Android", "Build-Ubuntu-Clang-x86-Debug-Android_Vulkan", "Build-Ubuntu-Clang-x86-Release-Android", "Build-Ubuntu-Clang-x86-Release-Android_Vulkan", "Build-Ubuntu-Clang-x86_64-Debug", "Build-Ubuntu-Clang-x86_64-Debug-ASAN", "Build-Ubuntu-Clang-x86_64-Debug-MSAN", "Build-Ubuntu-Clang-x86_64-Release", "Build-Ubuntu-Clang-x86_64-Release-TSAN", "Build-Ubuntu-GCC-x86-Debug", "Build-Ubuntu-GCC-x86-Release", "Build-Ubuntu-GCC-x86_64-Debug", "Build-Ubuntu-GCC-x86_64-Debug-NoGPU", "Build-Ubuntu-GCC-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE", "Build-Ubuntu-GCC-x86_64-Release", "Build-Ubuntu-GCC-x86_64-Release-ANGLE", "Build-Ubuntu-GCC-x86_64-Release-Fast", "Build-Ubuntu-GCC-x86_64-Release-Mesa", "Build-Ubuntu-GCC-x86_64-Release-NoGPU", "Build-Ubuntu-GCC-x86_64-Release-PDFium", "Build-Ubuntu-GCC-x86_64-Release-PDFium_SkiaPaths", "Build-Ubuntu-GCC-x86_64-Release-SKNX_NO_SIMD", "Build-Ubuntu-GCC-x86_64-Release-Shared", "Build-Ubuntu-GCC-x86_64-Release-Valgrind", "Build-Win-Clang-arm64-Release-Android", "Build-Win-MSVC-x86-Debug", "Build-Win-MSVC-x86-Debug-ANGLE", "Build-Win-MSVC-x86-Debug-Exceptions", "Build-Win-MSVC-x86-Debug-GDI", "Build-Win-MSVC-x86-Release", "Build-Win-MSVC-x86-Release-ANGLE", "Build-Win-MSVC-x86-Release-GDI", "Build-Win-MSVC-x86_64-Debug", "Build-Win-MSVC-x86_64-Debug-ANGLE", "Build-Win-MSVC-x86_64-Debug-GDI", "Build-Win-MSVC-x86_64-Debug-Vulkan", "Build-Win-MSVC-x86_64-Release", "Build-Win-MSVC-x86_64-Release-ANGLE", "Build-Win-MSVC-x86_64-Release-GDI", "Build-Win-MSVC-x86_64-Release-Vulkan", "Housekeeper-Nightly-RecreateSKPs_Canary", "Housekeeper-PerCommit", "Housekeeper-PerCommit-InfraTests", "Housekeeper-Weekly-RecreateSKPs", "Perf-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-Android", "Perf-Android-Clang-AndroidOne-CPU-MT6582-arm-Release-Android", "Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Debug-Android", "Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android", "Perf-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Debug-Android", "Perf-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Debug-Android_Vulkan", "Perf-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Release-Android", "Perf-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Release-Android_Vulkan", "Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android", "Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android_Vulkan", "Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Release-Android", "Perf-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Release-Android_Vulkan", "Perf-Android-Clang-Nexus10-GPU-MaliT604-arm-Debug-Android", "Perf-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android", "Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-Android", "Perf-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android", "Perf-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android", "Perf-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan", "Perf-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Release-Android", "Perf-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Release-Android_Vulkan", "Perf-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android", "Perf-Android-Clang-Nexus7-GPU-Tegra3-arm-Release-Android", "Perf-Android-Clang-NexusPlayer-CPU-Moorefield-x86-Debug-Android", "Perf-Android-Clang-NexusPlayer-CPU-Moorefield-x86-Release-Android", "Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android", "Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android_Vulkan", "Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android", "Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan", "Perf-Android-Clang-Pixel-GPU-Adreno530-arm64-Debug-Android", "Perf-Android-Clang-Pixel-GPU-Adreno530-arm64-Debug-Android_Vulkan", "Perf-Android-Clang-Pixel-GPU-Adreno530-arm64-Release-Android", "Perf-Android-Clang-Pixel-GPU-Adreno530-arm64-Release-Android_Vulkan", "Perf-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android", "Perf-Android-Clang-PixelC-CPU-TegraX1-arm64-Release-Android", "Perf-Android-Clang-PixelC-GPU-TegraX1-arm64-Release-Android_Skpbench", "Perf-Android-Clang-PixelC-GPU-TegraX1-arm64-Release-Android_Vulkan_Skpbench", "Perf-Mac-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Debug", "Perf-Mac-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release", "Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug", "Perf-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release", "Perf-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug", "Perf-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Release", "Perf-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Release-CommandBuffer", "Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug", "Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN", "Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN", "Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release", "Perf-Ubuntu-Clang-Golo-GPU-GT610-x86_64-Debug-ASAN", "Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug", "Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug", "Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE", "Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release", "Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_1k_SKPs", "Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs", "Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind", "Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Debug", "Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release", "Perf-Win10-MSVC-Golo-GPU-GT610-x86_64-Release", "Perf-Win10-MSVC-NUC-GPU-IntelIris6100-x86_64-Debug-ANGLE", "Perf-Win10-MSVC-NUC-GPU-IntelIris6100-x86_64-Release-ANGLE", "Perf-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug", "Perf-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan", "Perf-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Release", "Perf-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Release-Vulkan", "Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug", "Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE", "Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release", "Perf-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE", "Perf-Win10-MSVC-ShuttleC-GPU-iHD530-x86_64-Debug", "Perf-Win10-MSVC-ShuttleC-GPU-iHD530-x86_64-Release", "Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86-Debug", "Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Debug", "Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Debug-GDI", "Perf-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release", "Perf-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Debug", "Perf-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release", "Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Debug", "Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release", "Perf-iOS-Clang-iPadMini4-GPU-GX6450-Arm7-Debug", "Perf-iOS-Clang-iPadMini4-GPU-GX6450-Arm7-Release", "Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Debug-Android", "Test-Android-Clang-AndroidOne-CPU-MT6582-arm-Release-Android", "Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Debug-Android", "Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-Android", "Test-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Debug-Android", "Test-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Debug-Android_Vulkan", "Test-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Release-Android", "Test-Android-Clang-GalaxyS7-GPU-Adreno530-arm64-Release-Android_Vulkan", "Test-Android-Clang-GalaxyTab3-GPU-Vivante-arm-Debug-Android", "Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android", "Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Debug-Android_Vulkan", "Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Release-Android", "Test-Android-Clang-NVIDIA_Shield-GPU-TegraX1-arm64-Release-Android_Vulkan", "Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Debug-Android", "Test-Android-Clang-Nexus10-GPU-MaliT604-arm-Release-Android", "Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Debug-Android", "Test-Android-Clang-Nexus5-GPU-Adreno330-arm-Release-Android", "Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android", "Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Debug-Android_Vulkan", "Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Release-Android", "Test-Android-Clang-Nexus6p-GPU-Adreno430-arm64-Release-Android_Vulkan", "Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Debug-Android", "Test-Android-Clang-Nexus7-GPU-Tegra3-arm-Release-Android", "Test-Android-Clang-NexusPlayer-CPU-Moorefield-x86-Debug-Android", "Test-Android-Clang-NexusPlayer-CPU-Moorefield-x86-Release-Android", "Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android", "Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android_Vulkan", "Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android", "Test-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Release-Android_Vulkan", "Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Debug-Android", "Test-Android-Clang-PixelC-CPU-TegraX1-arm64-Release-Android", "Test-Android-Clang-PixelXL-GPU-Adreno530-arm64-Debug-Android", "Test-Android-Clang-PixelXL-GPU-Adreno530-arm64-Debug-Android_Vulkan", "Test-Android-Clang-PixelXL-GPU-Adreno530-arm64-Release-Android", "Test-Android-Clang-PixelXL-GPU-Adreno530-arm64-Release-Android_Vulkan", "Test-Mac-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Debug", "Test-Mac-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release", "Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Debug", "Test-Mac-Clang-MacMini6.2-CPU-AVX-x86_64-Release", "Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug", "Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Debug-CommandBuffer", "Test-Mac-Clang-MacMini6.2-GPU-HD4000-x86_64-Release", "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug", "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN", "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-MSAN", "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release", "Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN", "Test-Ubuntu-Clang-Golo-GPU-GT610-x86_64-Debug-ASAN", "Test-Ubuntu-Clang-Golo-GPU-GT610-x86_64-Release-TSAN", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_100k_SKPs", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Fast", "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD", "Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind", "Test-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Debug", "Test-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release", "Test-Win10-MSVC-Golo-GPU-GT610-x86_64-Release", "Test-Win10-MSVC-NUC-GPU-IntelIris6100-x86_64-Debug-ANGLE", "Test-Win10-MSVC-NUC-GPU-IntelIris6100-x86_64-Release-ANGLE", "Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug", "Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-Vulkan", "Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Release", "Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug", "Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Debug-ANGLE", "Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release", "Test-Win10-MSVC-ShuttleC-GPU-GTX960-x86_64-Release-ANGLE", "Test-Win10-MSVC-ShuttleC-GPU-iHD530-x86_64-Debug", "Test-Win10-MSVC-ShuttleC-GPU-iHD530-x86_64-Release", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86-Debug", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86-Release", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Debug", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Debug-GDI", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release", "Test-Win2k8-MSVC-GCE-CPU-AVX2-x86_64-Release-GDI", "Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Debug", "Test-Win8-MSVC-ShuttleA-GPU-HD7770-x86_64-Release", "Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Debug", "Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release", "Test-iOS-Clang-iPadMini4-GPU-GX6450-Arm7-Debug", "Test-iOS-Clang-iPadMini4-GPU-GX6450-Arm7-Release", } // LINUX_GCE_DIMENSIONS are the Swarming dimensions for Linux GCE // instances. LINUX_GCE_DIMENSIONS = []string{ "cpu:x86-64-avx2", "gpu:none", fmt.Sprintf("os:%s", DEFAULT_OS_LINUX), fmt.Sprintf("pool:%s", POOL_SKIA), } // Defines the structure of job names. jobNameSchema *JobNameSchema ) // deriveCompileTaskName returns the name of a compile task based on the given // job name. func deriveCompileTaskName(jobName string, parts map[string]string) string { if parts["role"] == "Housekeeper" { return "Build-Ubuntu-GCC-x86_64-Release-Shared" } else if parts["role"] == "Test" || parts["role"] == "Perf" { task_os := parts["os"] ec := parts["extra_config"] ec = strings.TrimSuffix(ec, "_Skpbench") if task_os == "Android" { if ec == "Vulkan" { ec = "Android_Vulkan" } task_os = "Ubuntu" } else if task_os == "iOS" { ec = task_os task_os = "Mac" } else if strings.Contains(task_os, "Win") { task_os = "Win" } jobNameMap := map[string]string{ "role": "Build", "os": task_os, "compiler": parts["compiler"], "target_arch": parts["arch"], "configuration": parts["configuration"], } if ec != "" { jobNameMap["extra_config"] = ec } name, err := jobNameSchema.MakeJobName(jobNameMap) if err != nil { glog.Fatal(err) } return name } else { return jobName } } // swarmDimensions generates swarming bot dimensions for the given task. func swarmDimensions(parts map[string]string) []string { d := map[string]string{ "pool": POOL_SKIA, } if os, ok := parts["os"]; ok { d["os"] = map[string]string{ "Android": "Android", "Mac": "Mac-10.11", "Ubuntu": DEFAULT_OS_LINUX, "Win": "Windows-2008ServerR2-SP1", "Win10": "Windows-10-10586", "Win2k8": "Windows-2008ServerR2-SP1", "Win8": "Windows-8.1-SP0", "iOS": "iOS-9.3.1", }[os] } else { d["os"] = DEFAULT_OS } if parts["role"] == "Test" || parts["role"] == "Perf" { if strings.Contains(parts["os"], "Android") { // For Android, the device type is a better dimension // than CPU or GPU. deviceInfo := map[string][]string{ "AndroidOne": {"sprout", "MOB30Q"}, "GalaxyS7": {"heroqlteatt", "MMB29M"}, "GalaxyTab3": {"goyawifi", "JDQ39"}, "NVIDIA_Shield": {"foster", "MRA58K"}, "Nexus10": {"manta", "LMY49J"}, "Nexus5": {"hammerhead", "MOB31E"}, "Nexus6": {"shamu", "M"}, "Nexus6p": {"angler", "NMF26C"}, "Nexus7": {"grouper", "LMY47V"}, "Nexus7v2": {"flo", "M"}, "NexusPlayer": {"fugu", "NRD90R"}, "Pixel": {"sailfish", "NMF25"}, "PixelC": {"dragon", "NMF26C"}, "PixelXL": {"marlin", "NMF25"}, }[parts["model"]] d["device_type"] = deviceInfo[0] d["device_os"] = deviceInfo[1] } else if strings.Contains(parts["os"], "iOS") { d["device"] = map[string]string{ "iPadMini4": "iPad5,1", }[parts["model"]] } else if parts["cpu_or_gpu"] == "CPU" { d["gpu"] = "none" d["cpu"] = map[string]string{ "AVX": "x86-64", "AVX2": "x86-64-avx2", "SSE4": "x86-64", }[parts["cpu_or_gpu_value"]] if strings.Contains(parts["os"], "Win") && parts["cpu_or_gpu_value"] == "AVX2" { // AVX2 is not correctly detected on Windows. Fall back on other // dimensions to ensure that we correctly target machines which we know // have AVX2 support. d["cpu"] = "x86-64" d["os"] = "Windows-2008ServerR2-SP1" } } else { d["gpu"] = map[string]string{ "GeForce320M": "10de:08a4", "GT610": "10de:104a", "GTX550Ti": "10de:1244", "GTX660": "10de:11c0", "GTX960": "10de:1401", "HD4000": "8086:0a2e", "HD4600": "8086:0412", "HD7770": "1002:683d", "iHD530": "8086:1912", "IntelIris6100": "8086:162b", }[parts["cpu_or_gpu_value"]] } } else { d["gpu"] = "none" } rv := make([]string, 0, len(d)) for k, v := range d { rv = append(rv, fmt.Sprintf("%s:%s", k, v)) } sort.Strings(rv) return rv } // compile generates a compile task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string { // Collect the necessary CIPD packages. pkgs := []*specs.CipdPackage{} // Android bots require a toolchain. if strings.Contains(name, "Android") { if strings.Contains(name, "Mac") { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("android_ndk_darwin")) } else if strings.Contains(name, "Win") { pkg := b.MustGetCipdPackageFromAsset("android_ndk_windows") pkg.Path = "n" pkgs = append(pkgs, pkg) } else { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("android_ndk_linux")) } } else if strings.Contains(name, "Ubuntu") && strings.Contains(name, "Clang") { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) } else if strings.Contains(name, "Win") { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("win_toolchain")) if strings.Contains(name, "Vulkan") { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("win_vulkan_sdk")) } } // Add the task. b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: pkgs, Dimensions: swarmDimensions(parts), ExtraArgs: []string{ "--workdir", "../../..", "swarm_compile", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "compile_skia.isolate", Priority: 0.8, }) // All compile tasks are runnable as their own Job. Assert that the Job // is listed in JOBS. if !util.In(name, JOBS) { glog.Fatalf("Job %q is missing from the JOBS list!", name) } return name } // recreateSKPs generates a RecreateSKPs task. Returns the name of the last // task in the generated chain of tasks, which the Job should add as a // dependency. func recreateSKPs(b *specs.TasksCfgBuilder, name string) string { b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: []*specs.CipdPackage{}, Dimensions: LINUX_GCE_DIMENSIONS, ExecutionTimeout: 4 * time.Hour, ExtraArgs: []string{ "--workdir", "../../..", "swarm_RecreateSKPs", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, IoTimeout: 40 * time.Minute, Isolate: "compile_skia.isolate", Priority: 0.8, }) return name } // ctSKPs generates a CT SKPs task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func ctSKPs(b *specs.TasksCfgBuilder, name string) string { b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: []*specs.CipdPackage{}, Dimensions: []string{"pool:SkiaCT"}, ExecutionTimeout: 24 * time.Hour, ExtraArgs: []string{ "--workdir", "../../..", "swarm_ct_skps", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, IoTimeout: time.Hour, Isolate: "ct_skps_skia.isolate", Priority: 0.8, }) return name } // housekeeper generates a Housekeeper task. Returns the name of the last task // in the generated chain of tasks, which the Job should add as a dependency. func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: []*specs.CipdPackage{b.MustGetCipdPackageFromAsset("go")}, Dependencies: []string{compileTaskName}, Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "swarm_housekeeper", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "housekeeper_skia.isolate", Priority: 0.8, }) return name } // infra generates an infra_tests task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func infra(b *specs.TasksCfgBuilder, name string) string { b.MustAddTask(name, &specs.TaskSpec{ CipdPackages: []*specs.CipdPackage{}, Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "swarm_infra", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "infra_skia.isolate", Priority: 0.8, }) return name } // doUpload indicates whether the given Job should upload its results. func doUpload(name string) bool { skipUploadBots := []string{ "ASAN", "Coverage", "MSAN", "TSAN", "UBSAN", "Valgrind", } for _, s := range skipUploadBots { if strings.Contains(name, s) { return false } } return true } // test generates a Test task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { s := &specs.TaskSpec{ CipdPackages: pkgs, Dependencies: []string{compileTaskName}, Dimensions: swarmDimensions(parts), ExecutionTimeout: 4 * time.Hour, Expiration: 20 * time.Hour, ExtraArgs: []string{ "--workdir", "../../..", "swarm_test", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, IoTimeout: 40 * time.Minute, Isolate: "test_skia.isolate", Priority: 0.8, } if strings.Contains(parts["extra_config"], "Valgrind") { s.ExecutionTimeout = 9 * time.Hour s.Expiration = 48 * time.Hour s.IoTimeout = time.Hour } else if strings.Contains(parts["extra_config"], "MSAN") { s.ExecutionTimeout = 9 * time.Hour } b.MustAddTask(name, s) // Upload results if necessary. if doUpload(name) { uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) b.MustAddTask(uploadName, &specs.TaskSpec{ Dependencies: []string{name}, Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_dm_results", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "upload_dm_results.isolate", Priority: 0.8, }) return uploadName } return name } // perf generates a Perf task. Returns the name of the last task in the // generated chain of tasks, which the Job should add as a dependency. func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compileTaskName string, pkgs []*specs.CipdPackage) string { recipe := "swarm_perf" isolate := "perf_skia.isolate" if strings.Contains(parts["extra_config"], "Skpbench") { recipe = "swarm_skpbench" isolate = "skpbench_skia.isolate" } s := &specs.TaskSpec{ CipdPackages: pkgs, Dependencies: []string{compileTaskName}, Dimensions: swarmDimensions(parts), ExecutionTimeout: 4 * time.Hour, Expiration: 20 * time.Hour, ExtraArgs: []string{ "--workdir", "../../..", recipe, fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, IoTimeout: 40 * time.Minute, Isolate: isolate, Priority: 0.8, } if strings.Contains(parts["extra_config"], "Valgrind") { s.ExecutionTimeout = 9 * time.Hour s.Expiration = 48 * time.Hour s.IoTimeout = time.Hour } else if strings.Contains(parts["extra_config"], "MSAN") { s.ExecutionTimeout = 9 * time.Hour } b.MustAddTask(name, s) // Upload results if necessary. if strings.Contains(name, "Release") && doUpload(name) { uploadName := fmt.Sprintf("%s%s%s", PREFIX_UPLOAD, jobNameSchema.Sep, name) b.MustAddTask(uploadName, &specs.TaskSpec{ Dependencies: []string{name}, Dimensions: LINUX_GCE_DIMENSIONS, ExtraArgs: []string{ "--workdir", "../../..", "upload_nano_results", fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), fmt.Sprintf("buildername=%s", name), "mastername=fake-master", "buildnumber=2", "slavename=fake-buildslave", "nobuildbot=True", fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), }, Isolate: "upload_nano_results.isolate", Priority: 0.8, }) return uploadName } return name } // process generates tasks and jobs for the given job name. func process(b *specs.TasksCfgBuilder, name string) { deps := []string{} parts, err := jobNameSchema.ParseJobName(name) if err != nil { glog.Fatal(err) } // RecreateSKPs. if strings.Contains(name, "RecreateSKPs") { deps = append(deps, recreateSKPs(b, name)) } // CT bots. if strings.Contains(name, "-CT_") { deps = append(deps, ctSKPs(b, name)) } // Infra tests. if name == "Housekeeper-PerCommit-InfraTests" { deps = append(deps, infra(b, name)) } // Compile bots. if parts["role"] == "Build" { deps = append(deps, compile(b, name, parts)) } // Most remaining bots need a compile task. compileTaskName := deriveCompileTaskName(name, parts) compileTaskParts, err := jobNameSchema.ParseJobName(compileTaskName) if err != nil { glog.Fatal(err) } // These bots do not need a compile task. if parts["role"] != "Build" && name != "Housekeeper-PerCommit-InfraTests" && !strings.Contains(name, "RecreateSKPs") && !strings.Contains(name, "-CT_") { compile(b, compileTaskName, compileTaskParts) } // Housekeeper. if name == "Housekeeper-PerCommit" { deps = append(deps, housekeeper(b, name, compileTaskName)) } // Common assets needed by the remaining bots. pkgs := []*specs.CipdPackage{ b.MustGetCipdPackageFromAsset("skimage"), b.MustGetCipdPackageFromAsset("skp"), b.MustGetCipdPackageFromAsset("svg"), } if strings.Contains(name, "Ubuntu") && strings.Contains(name, "SAN") { pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("clang_linux")) } // Skpbench only needs skps if strings.Contains(name, "Skpbench") { pkgs = []*specs.CipdPackage{ b.MustGetCipdPackageFromAsset("skp"), } } // Test bots. if parts["role"] == "Test" && !strings.Contains(name, "-CT_") { deps = append(deps, test(b, name, parts, compileTaskName, pkgs)) } // Perf bots. if parts["role"] == "Perf" && !strings.Contains(name, "-CT_") { deps = append(deps, perf(b, name, parts, compileTaskName, pkgs)) } // Add the Job spec. j := &specs.JobSpec{ Priority: 0.8, TaskSpecs: deps, } if name == "Housekeeper-Nightly-RecreateSKPs_Canary" { j.Trigger = "nightly" } if name == "Housekeeper-Weekly-RecreateSKPs" { j.Trigger = "weekly" } if name == "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs" { j.Trigger = "weekly" } b.MustAddJob(name, j) } // Regenerate the tasks.json file. func main() { b := specs.MustNewTasksCfgBuilder() // Create the JobNameSchema. schema, err := NewJobNameSchema(path.Join(b.CheckoutRoot(), "infra", "bots", "recipe_modules", "builder_name_schema", "builder_name_schema.json")) if err != nil { glog.Fatal(err) } jobNameSchema = schema // Create Tasks and Jobs. for _, name := range JOBS { process(b, name) } b.MustFinish() } // TODO(borenet): The below really belongs in its own file, probably next to the // builder_name_schema.json file. // JobNameSchema is a struct used for (de)constructing Job names in a // predictable format. type JobNameSchema struct { Schema map[string][]string `json:"builder_name_schema"` Sep string `json:"builder_name_sep"` } // NewJobNameSchema returns a JobNameSchema instance based on the given JSON // file. func NewJobNameSchema(jsonFile string) (*JobNameSchema, error) { var rv JobNameSchema f, err := os.Open(jsonFile) if err != nil { return nil, err } defer util.Close(f) if err := json.NewDecoder(f).Decode(&rv); err != nil { return nil, err } return &rv, nil } // ParseJobName splits the given Job name into its component parts, according // to the schema. func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error) { split := strings.Split(n, s.Sep) if len(split) < 2 { return nil, fmt.Errorf("Invalid job name: %q", n) } role := split[0] split = split[1:] keys, ok := s.Schema[role] if !ok { return nil, fmt.Errorf("Invalid job name; %q is not a valid role.", role) } extraConfig := "" if len(split) == len(keys)+1 { extraConfig = split[len(split)-1] split = split[:len(split)-1] } if len(split) != len(keys) { return nil, fmt.Errorf("Invalid job name; %q has incorrect number of parts.", n) } rv := make(map[string]string, len(keys)+2) rv["role"] = role if extraConfig != "" { rv["extra_config"] = extraConfig } for i, k := range keys { rv[k] = split[i] } return rv, nil } // MakeJobName assembles the given parts of a Job name, according to the schema. func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error) { role, ok := parts["role"] if !ok { return "", fmt.Errorf("Invalid job parts; jobs must have a role.") } keys, ok := s.Schema[role] if !ok { return "", fmt.Errorf("Invalid job parts; unknown role %q", role) } rvParts := make([]string, 0, len(parts)) rvParts = append(rvParts, role) for _, k := range keys { v, ok := parts[k] if !ok { return "", fmt.Errorf("Invalid job parts; missing %q", k) } rvParts = append(rvParts, v) } if _, ok := parts["extra_config"]; ok { rvParts = append(rvParts, parts["extra_config"]) } return strings.Join(rvParts, s.Sep), nil }