From 30a4e3da4bf341d5968b8cdf5bc2260e7f0d4b04 Mon Sep 17 00:00:00 2001 From: Ben Wagner Date: Tue, 7 Aug 2018 16:58:15 -0400 Subject: [PATCH] Add Win OpenCL Build job. Bug: skia:8081 Change-Id: Ibcb82389668bd9ea7395e498696417d5701761c0 Reviewed-on: https://skia-review.googlesource.com/140782 Commit-Queue: Ben Wagner Reviewed-by: Mike Klein --- BUILD.gn | 6 +- DEPS | 1 + infra/bots/gen_tasks.go | 5 + infra/bots/jobs.json | 1 + infra/bots/recipe_modules/build/default.py | 11 +- .../Build-Win-Clang-x86_64-Debug-OpenCL.json | 102 ++++++++++++ .../recipe_modules/build/examples/full.py | 1 + infra/bots/tasks.json | 146 ++++++++++++++++++ 8 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json diff --git a/BUILD.gn b/BUILD.gn index 535eed2f0e..3872bc4f89 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -2242,7 +2242,11 @@ if (skia_enable_tools) { "tools/hello-opencl.cpp", ] include_dirs = [ "src/compute/common" ] - libs = [ "OpenCL" ] + if (is_linux) { + libs = [ "OpenCL" ] + } else if (is_win) { + libs = [ "OpenCL.lib" ] + } } } } diff --git a/DEPS b/DEPS index aca4f36b08..b00f68d586 100644 --- a/DEPS +++ b/DEPS @@ -23,6 +23,7 @@ deps = { "third_party/externals/libwebp" : "https://chromium.googlesource.com/webm/libwebp.git@v0.6.1", "third_party/externals/lua" : "https://skia.googlesource.com/external/github.com/lua/lua.git@v5-3-4", "third_party/externals/microhttpd" : "https://android.googlesource.com/platform/external/libmicrohttpd@748945ec6f1c67b7efc934ab0808e1d32f2fb98d", + "third_party/externals/opencl-lib" : "https://skia.googlesource.com/external/github.com/GPUOpen-Tools/common-lib-amd-APPSDK-3.0@4e6d30e406d2e5a65e1d65e404fe6df5f772a32b", "third_party/externals/opengl-registry" : "https://skia.googlesource.com/external/github.com/KhronosGroup/OpenGL-Registry@14b80ebeab022b2c78f84a573f01028c96075553", "third_party/externals/piex" : "https://android.googlesource.com/platform/external/piex.git@bb217acdca1cc0c16b704669dd6f91a1b509c406", "third_party/externals/sdl" : "https://skia.googlesource.com/third_party/sdl@5d7cfcca344034aff9327f77fc181ae3754e7a90", diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 072d2e37a5..c19cf1f9cc 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -840,6 +840,11 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str if strings.Contains(name, "Vulkan") { task.Dependencies = append(task.Dependencies, isolateCIPDAsset(b, ISOLATE_WIN_VULKAN_SDK_NAME)) } + if strings.Contains(name, "OpenCL") { + task.CipdPackages = append(task.CipdPackages, + b.MustGetCipdPackageFromAsset("opencl_headers"), + ) + } } else if strings.Contains(name, "Mac") { task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_XCODE...) task.Caches = append(task.Caches, &specs.Cache{ diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index 9f1c50e2e6..221881b209 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -92,6 +92,7 @@ "Build-Win-Clang-x86-Release", "Build-Win-Clang-x86_64-Debug", "Build-Win-Clang-x86_64-Debug-ANGLE", + "Build-Win-Clang-x86_64-Debug-OpenCL", "Build-Win-Clang-x86_64-Debug-UBSAN", "Build-Win-Clang-x86_64-Debug-Vulkan", "Build-Win-Clang-x86_64-Release", diff --git a/infra/bots/recipe_modules/build/default.py b/infra/bots/recipe_modules/build/default.py index a7429736c5..fe989d6225 100644 --- a/infra/bots/recipe_modules/build/default.py +++ b/infra/bots/recipe_modules/build/default.py @@ -232,11 +232,18 @@ def compile_fn(api, checkout_root, out_dir): args['skia_use_metal'] = 'true' if 'OpenCL' in extra_tokens: args['skia_use_opencl'] = 'true' - extra_cflags.append( - '-isystem%s' % api.vars.slave_dir.join('opencl_headers')) if api.vars.is_linux: + extra_cflags.append( + '-isystem%s' % api.vars.slave_dir.join('opencl_headers')) extra_ldflags.append( '-L%s' % api.vars.slave_dir.join('opencl_ocl_icd_linux')) + elif 'Win' in os: + extra_cflags.append( + '-imsvc%s' % api.vars.slave_dir.join('opencl_headers')) + extra_ldflags.append( + '/LIBPATH:%s' % + skia_dir.join('third_party', 'externals', 'opencl-lib', '3-0', 'lib', + 'x86_64')) if 'iOS' in extra_tokens: # Bots use Chromium signing cert. args['skia_ios_identity'] = '".*GS9WA.*"' diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json new file mode 100644 index 0000000000..7061e7c61b --- /dev/null +++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json @@ -0,0 +1,102 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/cache/work/skia/infra/bots/assets/clang_win/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get clang_win VERSION" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/cache/work/skia/bin/fetch-gn" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[START_DIR]/cache/work/skia/bin/gn", + "gen", + "[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86_64-Debug-OpenCL/Debug_x64", + "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-imsvc[START_DIR]/opencl_headers\", \"-DDUMMY_clang_win_version=42\"] extra_ldflags=[\"/LIBPATH:[START_DIR]/cache/work/skia/third_party/externals/opencl-lib/3-0/lib/x86_64\"] skia_use_opencl=true target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/VC\"" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-k", + "0", + "-C", + "[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86_64-Debug-OpenCL/Debug_x64" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "ninja" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86_64-Debug-OpenCL/Debug_x64", + "[START_DIR]/[SWARM_OUT_DIR]/out/Debug_x64" + ], + "infra_step": true, + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/build/examples/full.py b/infra/bots/recipe_modules/build/examples/full.py index 8b59fe6d5f..9f7c501318 100644 --- a/infra/bots/recipe_modules/build/examples/full.py +++ b/infra/bots/recipe_modules/build/examples/full.py @@ -58,6 +58,7 @@ TEST_BUILDERS = [ 'Build-Mac-Clang-x86_64-Release-MoltenVK_Vulkan', 'Build-Win-Clang-arm64-Release-Android', 'Build-Win-Clang-x86-Debug-Exceptions', + 'Build-Win-Clang-x86_64-Debug-OpenCL', 'Build-Win-Clang-x86_64-Release-Vulkan', 'Housekeeper-PerCommit-CheckGeneratedFiles', ] diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 166a7979bb..be96ce9345 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -469,6 +469,11 @@ "Build-Win-Clang-x86_64-Debug-ANGLE" ] }, + "Build-Win-Clang-x86_64-Debug-OpenCL": { + "tasks": [ + "Build-Win-Clang-x86_64-Debug-OpenCL" + ] + }, "Build-Win-Clang-x86_64-Debug-UBSAN": { "tasks": [ "Build-Win-Clang-x86_64-Debug-UBSAN" @@ -14956,6 +14961,147 @@ ], "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com" }, + "Build-Win-Clang-x86_64-Debug-OpenCL": { + "caches": [ + { + "name": "vpython", + "path": "cache/vpython" + }, + { + "name": "git", + "path": "cache/git" + }, + { + "name": "git_cache", + "path": "cache/git_cache" + }, + { + "name": "work", + "path": "cache/work" + } + ], + "cipd_packages": [ + { + "name": "infra/tools/luci/kitchen/${platform}", + "path": ".", + "version": "git_revision:546aae39f1fb9dce9add528e2011afa574535ecd" + }, + { + "name": "infra/tools/luci-auth/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:e1abc57be62d198b5c2f487bfb2fa2d2eb0e867c" + }, + { + "name": "infra/tools/luci/vpython/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:b6cdec8586c9f8d3d728b1bc0bd4331330ba66fc" + }, + { + "name": "infra/python/cpython/${platform}", + "path": "cipd_bin_packages", + "version": "version:2.7.14.chromium14" + }, + { + "name": "infra/git/${platform}", + "path": "cipd_bin_packages", + "version": "version:2.17.1.chromium15" + }, + { + "name": "infra/tools/git/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:0ae21738597e5601ba90372315145fec18582fc4" + }, + { + "name": "infra/tools/luci/git-credential-luci/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:e1abc57be62d198b5c2f487bfb2fa2d2eb0e867c" + }, + { + "name": "skia/bots/clang_win", + "path": "clang_win", + "version": "version:7" + }, + { + "name": "skia/bots/opencl_headers", + "path": "opencl_headers", + "version": "version:0" + } + ], + "command": [ + "./kitchen${EXECUTABLE_SUFFIX}", + "cook", + "-checkout-dir", + "recipe_bundle", + "-mode", + "swarming", + "-luci-system-account", + "system", + "-cache-dir", + "cache", + "-temp-dir", + "tmp", + "-known-gerrit-host", + "android.googlesource.com", + "-known-gerrit-host", + "boringssl.googlesource.com", + "-known-gerrit-host", + "chromium.googlesource.com", + "-known-gerrit-host", + "dart.googlesource.com", + "-known-gerrit-host", + "fuchsia.googlesource.com", + "-known-gerrit-host", + "go.googlesource.com", + "-known-gerrit-host", + "llvm.googlesource.com", + "-known-gerrit-host", + "skia.googlesource.com", + "-known-gerrit-host", + "webrtc.googlesource.com", + "-output-result-json", + "${ISOLATED_OUTDIR}/build_result_filename", + "-workdir", + ".", + "-recipe", + "compile", + "-properties", + "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Build-Win-Clang-x86_64-Debug-OpenCL\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"build\"}", + "-logdog-annotation-url", + "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + ], + "dependencies": [ + "Housekeeper-PerCommit-BundleRecipes", + "Housekeeper-PerCommit-IsolateWinToolchain" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "gpu:none", + "image:windows-server-2016-dc-v20180710", + "machine_type:n1-highcpu-64", + "os:Windows-2016Server-14393", + "pool:Skia" + ], + "env_prefixes": { + "PATH": [ + "cipd_bin_packages", + "cipd_bin_packages/bin" + ], + "VPYTHON_VIRTUALENV_ROOT": [ + "${cache_dir}/vpython" + ] + }, + "execution_timeout_ns": 3600000000000, + "extra_tags": { + "log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + }, + "io_timeout_ns": 3600000000000, + "isolate": "swarm_recipe.isolate", + "max_attempts": 1, + "outputs": [ + "build" + ], + "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com" + }, "Build-Win-Clang-x86_64-Debug-UBSAN": { "caches": [ {