Add OpenCL build.

Bug: skia:8081
Change-Id: I8b2a88cc25970398511aa078d456ca8a1182792b
Reviewed-on: https://skia-review.googlesource.com/136594
Commit-Queue: Mike Klein <mtklein@google.com>
Auto-Submit: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Mike Klein <mtklein@google.com>
This commit is contained in:
Ben Wagner 2018-06-28 20:41:04 -04:00 committed by Skia Commit-Bot
parent eb8f8106f3
commit 55a7d22beb
19 changed files with 561 additions and 2 deletions

2
.gitignore vendored
View File

@ -39,7 +39,7 @@ tools/skp/page_sets/data/*.json
tools/skp/page_sets/data/*.wpr
xcodebuild
build
/build
buildtools
tools/clang
third_party/llvm-build

View File

@ -0,0 +1 @@
0

View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Common vars used by scripts in this directory."""
import os
import sys
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
sys.path.insert(0, INFRA_BOTS_DIR)
from assets import assets
ASSET_NAME = os.path.basename(FILE_DIR)
def run(cmd):
"""Run a command, eg. "upload" or "download". """
assets.main([cmd, ASSET_NAME] + sys.argv[1:])

View File

@ -0,0 +1,48 @@
#!/usr/bin/env python
#
# Copyright 2018 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset."""
import argparse
import common
import os
import shutil
import subprocess
import utils
# The OpenCL C headers are available at
# https://github.com/KhronosGroup/OpenCL-Headers, but the C++ header cl.hpp
# would need to be generated from https://github.com/KhronosGroup/OpenCL-CLHPP.
# Instead, we just grab the pre-built headers from the Debian packages.
PKGS = [
'opencl-c-headers',
'opencl-clhpp-headers',
]
def create_asset(target_dir):
"""Create the asset."""
with utils.tmp_dir():
# Download required Debian packages.
subprocess.check_call(['apt-get', 'download'] + PKGS)
# Extract to CWD.
for f in os.listdir('.'):
subprocess.check_call(['dpkg-deb', '--extract', f, '.'])
# Copy usr/include/CL to target_dir.
shutil.move(os.path.join(os.getcwd(), 'usr', 'include', 'CL'), target_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--target_dir', '-t', required=True)
args = parser.parse_args()
create_asset(args.target_dir)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,42 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset and upload it."""
import argparse
import common
import os
import subprocess
import sys
import utils
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gsutil')
args = parser.parse_args()
with utils.tmp_dir():
cwd = os.getcwd()
create_script = os.path.join(common.FILE_DIR, 'create.py')
upload_script = os.path.join(common.FILE_DIR, 'upload.py')
try:
subprocess.check_call(['python', create_script, '-t', cwd])
cmd = ['python', upload_script, '-t', cwd]
if args.gsutil:
cmd.extend(['--gsutil', args.gsutil])
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
# Trap exceptions to avoid printing two stacktraces.
sys.exit(1)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download the current version of the asset."""
import common
if __name__ == '__main__':
common.run('download')

View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Upload a new version of the asset."""
import common
if __name__ == '__main__':
common.run('upload')

View File

@ -0,0 +1 @@
0

View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Common vars used by scripts in this directory."""
import os
import sys
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
sys.path.insert(0, INFRA_BOTS_DIR)
from assets import assets
ASSET_NAME = os.path.basename(FILE_DIR)
def run(cmd):
"""Run a command, eg. "upload" or "download". """
assets.main([cmd, ASSET_NAME] + sys.argv[1:])

View File

@ -0,0 +1,47 @@
#!/usr/bin/env python
#
# Copyright 2018 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset."""
import argparse
import common
import os
import shutil
import subprocess
import utils
# Use libOpenCL.so from the ocl-icd-opencl-dev Debian package.
PKGS = [
'ocl-icd-opencl-dev',
'ocl-icd-libopencl1',
]
def create_asset(target_dir):
"""Create the asset."""
with utils.tmp_dir():
# Download required Debian packages.
subprocess.check_call(['apt-get', 'download'] + PKGS)
# Extract to CWD.
for f in os.listdir('.'):
subprocess.check_call(['dpkg-deb', '--extract', f, '.'])
# Copy usr/lib/x86_64-linux-gnu/* to target_dir.
lib_dir = os.path.join(os.getcwd(), 'usr', 'lib', 'x86_64-linux-gnu')
for f in os.listdir(lib_dir):
shutil.move(os.path.join(lib_dir, f), target_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--target_dir', '-t', required=True)
args = parser.parse_args()
create_asset(args.target_dir)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,42 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset and upload it."""
import argparse
import common
import os
import subprocess
import sys
import utils
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gsutil')
args = parser.parse_args()
with utils.tmp_dir():
cwd = os.getcwd()
create_script = os.path.join(common.FILE_DIR, 'create.py')
upload_script = os.path.join(common.FILE_DIR, 'upload.py')
try:
subprocess.check_call(['python', create_script, '-t', cwd])
cmd = ['python', upload_script, '-t', cwd]
if args.gsutil:
cmd.extend(['--gsutil', args.gsutil])
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
# Trap exceptions to avoid printing two stacktraces.
sys.exit(1)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download the current version of the asset."""
import common
if __name__ == '__main__':
common.run('download')

View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Upload a new version of the asset."""
import common
if __name__ == '__main__':
common.run('upload')

View File

@ -803,6 +803,12 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
if strings.Contains(name, "SwiftShader") {
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("cmake_linux"))
}
if strings.Contains(name, "OpenCL") {
task.CipdPackages = append(task.CipdPackages,
b.MustGetCipdPackageFromAsset("opencl_headers"),
b.MustGetCipdPackageFromAsset("opencl_ocl_icd_linux"),
)
}
} else if strings.Contains(name, "Win") {
task.Dependencies = append(task.Dependencies, isolateCIPDAsset(b, ISOLATE_WIN_TOOLCHAIN_NAME))
if strings.Contains(name, "Clang") {

View File

@ -30,6 +30,7 @@
"Build-Debian9-Clang-x86_64-Debug-Chromebook_GLES",
"Build-Debian9-Clang-x86_64-Debug-Coverage",
"Build-Debian9-Clang-x86_64-Debug-MSAN",
"Build-Debian9-Clang-x86_64-Debug-OpenCL",
"Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE",
"Build-Debian9-Clang-x86_64-Debug-SafeStack",
"Build-Debian9-Clang-x86_64-Debug-Static",

View File

@ -204,6 +204,13 @@ def compile_fn(api, checkout_root, out_dir):
args['skia_moltenvk_path'] = '"%s"' % moltenvk
if 'Metal' in extra_tokens:
args['skia_use_metal'] = 'true'
if 'OpenCL' in extra_tokens:
args['skia_use_opencl'] = 'true'
extra_cflags.append(
'-isystem%s' % api.vars.slave_dir.join('opencl_headers'))
if api.vars.is_linux:
extra_ldflags.append(
'-L%s' % api.vars.slave_dir.join('opencl_ocl_icd_linux'))
if 'iOS' in extra_tokens:
# Bots use Chromium signing cert.
args['skia_ios_identity'] = '".*GS9WA.*"'

View File

@ -0,0 +1,102 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/work/skia/infra/bots/assets/clang_linux/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get clang_linux VERSION"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/cache/work/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Debug-OpenCL/Debug",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\", \"-O1\", \"-isystem[START_DIR]/opencl_headers\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\", \"-L[START_DIR]/opencl_ocl_icd_linux\"] skia_use_opencl=true target_cpu=\"x86_64\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Debug-OpenCL/Debug"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Debug-OpenCL/Debug",
"[START_DIR]/[SWARM_OUT_DIR]/out/Debug"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

View File

@ -32,6 +32,7 @@ TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Debug-Chromebook_GLES',
'Build-Debian9-Clang-x86_64-Debug-Coverage',
'Build-Debian9-Clang-x86_64-Debug-MSAN',
'Build-Debian9-Clang-x86_64-Debug-OpenCL',
'Build-Debian9-Clang-x86_64-Debug-SK_CPU_LIMIT_SSE41',
'Build-Debian9-Clang-x86_64-Debug-SafeStack',
'Build-Debian9-Clang-x86_64-Release-ASAN',
@ -76,4 +77,3 @@ def GenTests(api):
api.properties(**defaultProps(buildername))
)
yield test

View File

@ -187,6 +187,12 @@
"Build-Debian9-Clang-x86_64-Debug-MSAN"
]
},
"Build-Debian9-Clang-x86_64-Debug-OpenCL": {
"priority": 0.8,
"tasks": [
"Build-Debian9-Clang-x86_64-Debug-OpenCL"
]
},
"Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE": {
"priority": 0.8,
"tasks": [
@ -7482,6 +7488,146 @@
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Debug-OpenCL": {
"caches": [
{
"name": "vpython",
"path": "cache/vpython"
},
{
"name": "git",
"path": "cache/git"
},
{
"name": "git_cache",
"path": "cache/git_cache"
},
{
"name": "work",
"path": "cache/work"
}
],
"cipd_packages": [
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:546aae39f1fb9dce9add528e2011afa574535ecd"
},
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:e1abc57be62d198b5c2f487bfb2fa2d2eb0e867c"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:ad60019cb66a75b59991d43b95a43f68e3fff81b"
},
{
"name": "infra/git/${platform}",
"path": "cipd_bin_packages",
"version": "version:2.17.1.chromium15"
},
{
"name": "infra/tools/git/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:0ae21738597e5601ba90372315145fec18582fc4"
},
{
"name": "infra/tools/luci/git-credential-luci/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:e1abc57be62d198b5c2f487bfb2fa2d2eb0e867c"
},
{
"name": "skia/bots/clang_linux",
"path": "clang_linux",
"version": "version:11"
},
{
"name": "skia/bots/opencl_headers",
"path": "opencl_headers",
"version": "version:0"
},
{
"name": "skia/bots/opencl_ocl_icd_linux",
"path": "opencl_ocl_icd_linux",
"version": "version:0"
}
],
"command": [
"./kitchen${EXECUTABLE_SUFFIX}",
"cook",
"-checkout-dir",
"recipe_bundle",
"-mode",
"swarming",
"-luci-system-account",
"system",
"-cache-dir",
"cache",
"-temp-dir",
"tmp",
"-known-gerrit-host",
"android.googlesource.com",
"-known-gerrit-host",
"boringssl.googlesource.com",
"-known-gerrit-host",
"chromium.googlesource.com",
"-known-gerrit-host",
"dart.googlesource.com",
"-known-gerrit-host",
"fuchsia.googlesource.com",
"-known-gerrit-host",
"go.googlesource.com",
"-known-gerrit-host",
"llvm.googlesource.com",
"-known-gerrit-host",
"skia.googlesource.com",
"-known-gerrit-host",
"webrtc.googlesource.com",
"-output-result-json",
"${ISOLATED_OUTDIR}/build_result_filename",
"-workdir",
".",
"-recipe",
"compile",
"-properties",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Build-Debian9-Clang-x86_64-Debug-OpenCL\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"build\"}",
"-logdog-annotation-url",
"logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-highcpu-64",
"os:Debian-9.4",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"${cache_dir}/vpython"
]
},
"execution_timeout_ns": 3600000000000,
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations"
},
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"max_attempts": 1,
"outputs": [
"build"
],
"priority": 0.8,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE": {
"caches": [
{