Add SwiftShader Build jobs.

Bug: skia:7671
No-Try: true
Change-Id: I34015cceca538c84d87a77b6688fa1490c620d04
Reviewed-on: https://skia-review.googlesource.com/121425
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Ben Wagner <benjaminwagner@google.com>
This commit is contained in:
Ben Wagner 2018-04-19 17:53:03 -04:00 committed by Skia Commit-Bot
parent 919c6a14ca
commit bbdee1befa
6 changed files with 451 additions and 0 deletions

View File

@ -710,6 +710,9 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
}
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("mips64el_toolchain_linux"))
}
if strings.Contains(name, "SwiftShader") {
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("cmake_linux"))
}
} else if strings.Contains(name, "Win") {
task.Dependencies = append(task.Dependencies, isolateCIPDAsset(b, ISOLATE_WIN_TOOLCHAIN_NAME))
if strings.Contains(name, "Clang") {

View File

@ -32,6 +32,7 @@
"Build-Debian9-Clang-x86_64-Debug-MSAN",
"Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE",
"Build-Debian9-Clang-x86_64-Debug-SafeStack",
"Build-Debian9-Clang-x86_64-Debug-SwiftShader",
"Build-Debian9-Clang-x86_64-Debug-Vulkan",
"Build-Debian9-Clang-x86_64-Debug-Vulkan_Coverage",
"Build-Debian9-Clang-x86_64-Release",
@ -46,6 +47,7 @@
"Build-Debian9-Clang-x86_64-Release-SK_CPU_LIMIT_SSE2",
"Build-Debian9-Clang-x86_64-Release-SK_CPU_LIMIT_SSE41",
"Build-Debian9-Clang-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER",
"Build-Debian9-Clang-x86_64-Release-SwiftShader",
"Build-Debian9-Clang-x86_64-Release-TSAN",
"Build-Debian9-Clang-x86_64-Release-Vulkan",
"Build-Debian9-EMCC-wasm-Release",

View File

@ -0,0 +1,159 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[CUSTOM_/_B_WORK]/skia/infra/bots/assets/clang_linux/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get clang_linux VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out"
],
"infra_step": true,
"name": "makedirs swiftshader_out"
},
{
"cmd": [
"cmake",
"[CUSTOM_/_B_WORK]/skia/third_party/externals/swiftshader",
"-GNinja"
],
"cwd": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out",
"env": {
"BUILDTYPE": "Release",
"CC": "[START_DIR]/clang_linux/bin/clang",
"CHROME_HEADLESS": "1",
"CXX": "[START_DIR]/clang_linux/bin/clang++",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/cmake_linux/bin",
"SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader"
},
"name": "swiftshader cmake"
},
{
"cmd": [
"ninja",
"-C",
"[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out",
"libEGL.so",
"libGLESv2.so"
],
"cwd": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out",
"env": {
"BUILDTYPE": "Release",
"CC": "[START_DIR]/clang_linux/bin/clang",
"CHROME_HEADLESS": "1",
"CXX": "[START_DIR]/clang_linux/bin/clang++",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/cmake_linux/bin",
"SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader"
},
"name": "swiftshader ninja"
},
{
"cmd": [
"python",
"-u",
"[CUSTOM_/_B_WORK]/skia/bin/fetch-gn"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[CUSTOM_/_B_WORK]/skia/bin/gn",
"gen",
"[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/Release",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\", \"-DGR_EGL_TRY_GLES3_THEN_GLES2\", \"-I[CUSTOM_/_B_WORK]/skia/third_party/externals/swiftshader/include\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\", \"-L[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out\"] is_debug=false skia_use_egl=true target_cpu=\"x86_64\""
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/Release"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[CUSTOM_/_B_WORK]/skia/out/Build-Debian9-Clang-x86_64-Release-SwiftShader/swiftshader_out",
"[START_DIR]/[SWARM_OUT_DIR]/out/swiftshader_out"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

View File

@ -68,6 +68,7 @@ TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-Fast',
'Build-Debian9-Clang-x86_64-Release-Mini',
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-SwiftShader',
'Build-Debian9-Clang-x86_64-Release-Vulkan',
'Build-Debian9-EMCC-wasm-Release',
'Build-Debian9-GCC-loongson3a-Release',

View File

@ -22,6 +22,29 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
'--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration),
'--no-sync', '--no-hooks', '--make-output-dir'])
def compile_swiftshader(self, swiftshader_root, cc, cxx, out):
"""Build SwiftShader with CMake.
Building SwiftShader works differently from any other Skia third_party lib.
See discussion in skia:7671 for more detail.
Args:
swiftshader_root: root of the SwiftShader checkout.
cc, cxx: compiler binaries to use
out: target directory for libEGL.so and libGLESv2.so
"""
cmake_bin = str(self.m.vars.slave_dir.join('cmake_linux', 'bin'))
env = {
'CC': cc,
'CXX': cxx,
'PATH': '%%(PATH)s:%s' % cmake_bin
}
self.m.file.ensure_directory('makedirs swiftshader_out', out)
with self.m.context(cwd=out, env=env):
self._run('swiftshader cmake', ['cmake', swiftshader_root, '-GNinja'])
self._run('swiftshader ninja',
['ninja', '-C', out, 'libEGL.so', 'libGLESv2.so'])
def compile(self, unused_target):
"""Build Skia with GN."""
compiler = self.m.vars.builder_cfg.get('compiler', '')
@ -121,6 +144,21 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
args['is_debug'] = 'false'
if 'ANGLE' in extra_tokens:
args['skia_use_angle'] = 'true'
if 'SwiftShader' in extra_tokens:
swiftshader_root = self.m.vars.skia_dir.join('third_party', 'externals',
'swiftshader')
swiftshader_out = self.m.vars.skia_out.join('swiftshader_out')
self.compile_swiftshader(swiftshader_root, cc, cxx, swiftshader_out)
args['skia_use_egl'] = 'true'
extra_cflags.extend([
'-DGR_EGL_TRY_GLES3_THEN_GLES2',
# TODO(dogben): Use headers from Khronos rather than SwiftShader's
# copy.
'-I%s' % swiftshader_root.join('include'),
])
extra_ldflags.extend([
'-L%s' % swiftshader_out,
])
if 'CommandBuffer' in extra_tokens:
self.m.run.run_once(self.build_command_buffer)
if 'MSAN' in extra_tokens:
@ -235,6 +273,12 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args])
self._run('ninja', [ninja, '-k', '0', '-C', self.out_dir])
def copy_extra_build_products(self, swarming_out_dir):
if 'SwiftShader' in self.m.vars.extra_tokens:
self.m.run.copy_build_products(
self.m.vars.skia_out.join('swiftshader_out'),
swarming_out_dir.join('out', 'swiftshader_out'))
def step(self, name, cmd):
app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0])
cmd = [app] + cmd[1:]

View File

@ -186,6 +186,12 @@
"Build-Debian9-Clang-x86_64-Debug-SafeStack"
]
},
"Build-Debian9-Clang-x86_64-Debug-SwiftShader": {
"priority": 0.8,
"tasks": [
"Build-Debian9-Clang-x86_64-Debug-SwiftShader"
]
},
"Build-Debian9-Clang-x86_64-Debug-Vulkan": {
"priority": 0.8,
"tasks": [
@ -276,6 +282,12 @@
"Build-Debian9-Clang-x86_64-Release-SK_FORCE_RASTER_PIPELINE_BLITTER"
]
},
"Build-Debian9-Clang-x86_64-Release-SwiftShader": {
"priority": 0.8,
"tasks": [
"Build-Debian9-Clang-x86_64-Release-SwiftShader"
]
},
"Build-Debian9-Clang-x86_64-Release-TSAN": {
"priority": 0.8,
"tasks": [
@ -6776,6 +6788,121 @@
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Debug-SwiftShader": {
"cipd_packages": [
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c"
},
{
"name": "infra/tools/authutil/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82"
},
{
"name": "infra/git/${platform}",
"path": "cipd_bin_packages",
"version": "version:2.15.0.chromium12"
},
{
"name": "infra/tools/git/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:fa7a52f4741f5e04bba0dfccc9b8456dc572c60b"
},
{
"name": "infra/tools/luci/git-credential-luci/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82"
},
{
"name": "skia/bots/clang_linux",
"path": "clang_linux",
"version": "version:11"
},
{
"name": "skia/bots/cmake_linux",
"path": "cmake_linux",
"version": "version:0"
}
],
"command": [
"./kitchen${EXECUTABLE_SUFFIX}",
"cook",
"-checkout-dir",
"recipe_bundle",
"-mode",
"swarming",
"-luci-system-account",
"system",
"-cache-dir",
"cache",
"-temp-dir",
"tmp",
"-known-gerrit-host",
"android.googlesource.com",
"-known-gerrit-host",
"boringssl.googlesource.com",
"-known-gerrit-host",
"chromium.googlesource.com",
"-known-gerrit-host",
"dart.googlesource.com",
"-known-gerrit-host",
"fuchsia.googlesource.com",
"-known-gerrit-host",
"go.googlesource.com",
"-known-gerrit-host",
"llvm.googlesource.com",
"-known-gerrit-host",
"pdfium.googlesource.com",
"-known-gerrit-host",
"skia.googlesource.com",
"-known-gerrit-host",
"webrtc.googlesource.com",
"-output-result-json",
"${ISOLATED_OUTDIR}/build_result_filename",
"-workdir",
".",
"-recipe",
"compile",
"-properties",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Build-Debian9-Clang-x86_64-Debug-SwiftShader\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"build\"}",
"-logdog-annotation-url",
"logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.4",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"${cache_dir}/vpython"
]
},
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
},
"isolate": "swarm_recipe.isolate",
"outputs": [
"build"
],
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Debug-Vulkan": {
"cipd_packages": [
{
@ -8461,6 +8588,121 @@
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Release-SwiftShader": {
"cipd_packages": [
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c"
},
{
"name": "infra/tools/authutil/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82"
},
{
"name": "infra/git/${platform}",
"path": "cipd_bin_packages",
"version": "version:2.15.0.chromium12"
},
{
"name": "infra/tools/git/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:fa7a52f4741f5e04bba0dfccc9b8456dc572c60b"
},
{
"name": "infra/tools/luci/git-credential-luci/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82"
},
{
"name": "skia/bots/clang_linux",
"path": "clang_linux",
"version": "version:11"
},
{
"name": "skia/bots/cmake_linux",
"path": "cmake_linux",
"version": "version:0"
}
],
"command": [
"./kitchen${EXECUTABLE_SUFFIX}",
"cook",
"-checkout-dir",
"recipe_bundle",
"-mode",
"swarming",
"-luci-system-account",
"system",
"-cache-dir",
"cache",
"-temp-dir",
"tmp",
"-known-gerrit-host",
"android.googlesource.com",
"-known-gerrit-host",
"boringssl.googlesource.com",
"-known-gerrit-host",
"chromium.googlesource.com",
"-known-gerrit-host",
"dart.googlesource.com",
"-known-gerrit-host",
"fuchsia.googlesource.com",
"-known-gerrit-host",
"go.googlesource.com",
"-known-gerrit-host",
"llvm.googlesource.com",
"-known-gerrit-host",
"pdfium.googlesource.com",
"-known-gerrit-host",
"skia.googlesource.com",
"-known-gerrit-host",
"webrtc.googlesource.com",
"-output-result-json",
"${ISOLATED_OUTDIR}/build_result_filename",
"-workdir",
".",
"-recipe",
"compile",
"-properties",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Build-Debian9-Clang-x86_64-Release-SwiftShader\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"build\"}",
"-logdog-annotation-url",
"logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.4",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"${cache_dir}/vpython"
]
},
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
},
"isolate": "swarm_recipe.isolate",
"outputs": [
"build"
],
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Release-TSAN": {
"cipd_packages": [
{