Add Goma support in gn_flavor.
Using full paths in skia.h causes "file not found" with Goma. All other Builds seem fine without the path, so I changed find_headers.py to use the basename. Change-Id: Ib520e91a92ebffe36a736eb53f643d359f5bb2ce Reviewed-on: https://skia-review.googlesource.com/79360 Reviewed-by: Greg Daniel <egdaniel@google.com> Reviewed-by: Eric Boren <borenet@google.com> Commit-Queue: Ben Wagner <benjaminwagner@google.com>
This commit is contained in:
Ben Wagner
Skia Commit-Bot
parent
53e5e7d4ce
commit
3f39bf8fe5
@ -35,7 +35,7 @@ with open(skia_h, "w") as f:
|
||||
f.write('#ifndef skia_h_DEFINED\n')
|
||||
f.write('#define skia_h_DEFINED\n')
|
||||
for h in headers:
|
||||
f.write('#include "' + h + '"\n')
|
||||
f.write('#include "' + os.path.basename(h) + '"\n')
|
||||
f.write('#endif//skia_h_DEFINED\n')
|
||||
|
||||
with open(skia_h + '.deps', "w") as f:
|
||||
|
||||
@ -65,6 +65,7 @@
|
||||
"Build-Win-Clang-arm64-Release-Android",
|
||||
"Build-Win-Clang-x86_64-Debug",
|
||||
"Build-Win-Clang-x86_64-Debug-ANGLE",
|
||||
"Build-Win-Clang-x86_64-Debug-Goma",
|
||||
"Build-Win-Clang-x86_64-Debug-Vulkan",
|
||||
"Build-Win-Clang-x86_64-Release",
|
||||
"Build-Win-Clang-x86_64-Release-ANGLE",
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
DEPS = [
|
||||
'builder_name_schema',
|
||||
'depot_tools/bot_update',
|
||||
'depot_tools/cipd',
|
||||
'env',
|
||||
'recipe_engine/context',
|
||||
'recipe_engine/file',
|
||||
|
||||
@ -0,0 +1,196 @@
|
||||
[
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
|
||||
"--json-output",
|
||||
"/path/to/tmp/json",
|
||||
"ensure-directory",
|
||||
"--mode",
|
||||
"0777",
|
||||
"[START_DIR]/tmp"
|
||||
],
|
||||
"infra_step": true,
|
||||
"name": "makedirs tmp_dir"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"\nimport os\nimport sys\nimport urllib2\n\nTOKEN_URL = (\n 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nwith open(sys.argv[1], 'w') as f:\n f.write(contents)\n",
|
||||
"[START_DIR]/tmp/jwt_service_account_goma-client.json"
|
||||
],
|
||||
"infra_step": true,
|
||||
"name": "download jwt_service_account_goma-client.json",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import os@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@TOKEN_URL = (@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
|
||||
"@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
|
||||
"@@@STEP_LOG_END@python.inline@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"cipd",
|
||||
"ensure",
|
||||
"-root",
|
||||
"[CACHE]/goma",
|
||||
"-ensure-file",
|
||||
"infra_internal/goma/client/linux-amd64 release",
|
||||
"-json-output",
|
||||
"/path/to/tmp/json",
|
||||
"-service-account-json",
|
||||
"[START_DIR]/tmp/jwt_service_account_goma-client.json"
|
||||
],
|
||||
"name": "ensure_installed",
|
||||
"~followup_annotations": [
|
||||
"@@@STEP_LOG_LINE@json.output@{@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ {@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ \"package\": \"infra_internal/goma/client/linux-amd64\"@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ }@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@ ]@@@",
|
||||
"@@@STEP_LOG_LINE@json.output@}@@@",
|
||||
"@@@STEP_LOG_END@json.output@@@"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"goma_ctl.py",
|
||||
"ensure_start"
|
||||
],
|
||||
"cwd": "[CACHE]/goma",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"GOMA_FALLBACK": "0",
|
||||
"GOMA_HERMETIC": "error",
|
||||
"GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
|
||||
"GOMA_USE_LOCAL": "0",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"infra_step": true,
|
||||
"name": "start goma"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"[CUSTOM_C:\\_B_WORK]/skia/bin/fetch-gn"
|
||||
],
|
||||
"cwd": "[CUSTOM_C:\\_B_WORK]/skia",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"infra_step": true,
|
||||
"name": "fetch-gn"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"[CUSTOM_C:\\_B_WORK]/skia/bin/gn.exe",
|
||||
"gen",
|
||||
"[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64",
|
||||
"--args=cc=\"clang\" cc_wrapper=\"[CACHE]/goma/gomacc\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" is_debug=false target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/VC\""
|
||||
],
|
||||
"cwd": "[CUSTOM_C:\\_B_WORK]/skia",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"GOMA_FALLBACK": "0",
|
||||
"GOMA_HERMETIC": "error",
|
||||
"GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
|
||||
"GOMA_USE_LOCAL": "0",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"name": "gn gen"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"ninja.exe",
|
||||
"-k",
|
||||
"0",
|
||||
"-C",
|
||||
"[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64",
|
||||
"-j",
|
||||
"100"
|
||||
],
|
||||
"cwd": "[CUSTOM_C:\\_B_WORK]/skia",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"GOMA_FALLBACK": "0",
|
||||
"GOMA_HERMETIC": "error",
|
||||
"GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
|
||||
"GOMA_USE_LOCAL": "0",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"name": "ninja"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"goma_ctl.py",
|
||||
"stat"
|
||||
],
|
||||
"cwd": "[CACHE]/goma",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"GOMA_FALLBACK": "0",
|
||||
"GOMA_HERMETIC": "error",
|
||||
"GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
|
||||
"GOMA_USE_LOCAL": "0",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"infra_step": true,
|
||||
"name": "print goma stats"
|
||||
},
|
||||
{
|
||||
"cmd": [
|
||||
"python",
|
||||
"-u",
|
||||
"goma_ctl.py",
|
||||
"stop"
|
||||
],
|
||||
"cwd": "[CACHE]/goma",
|
||||
"env": {
|
||||
"BUILDTYPE": "Release_x64",
|
||||
"CHROME_HEADLESS": "1",
|
||||
"GOMA_FALLBACK": "0",
|
||||
"GOMA_HERMETIC": "error",
|
||||
"GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
|
||||
"GOMA_USE_LOCAL": "0",
|
||||
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
|
||||
"SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
|
||||
},
|
||||
"infra_step": true,
|
||||
"name": "stop goma"
|
||||
},
|
||||
{
|
||||
"name": "$result",
|
||||
"recipe_result": null,
|
||||
"status_code": 0
|
||||
}
|
||||
]
|
||||
@ -77,6 +77,7 @@ TEST_BUILDERS = [
|
||||
'Build-Win-Clang-arm64-Release-Android',
|
||||
'Build-Win-Clang-x86_64-Debug-GDI',
|
||||
'Build-Win-Clang-x86_64-Release',
|
||||
'Build-Win-Clang-x86_64-Release-Goma',
|
||||
'Build-Win-Clang-x86_64-Release-Vulkan',
|
||||
'Build-Win-MSVC-x86-Debug-Exceptions',
|
||||
'Housekeeper-PerCommit-CheckGeneratedFiles',
|
||||
|
||||
@ -22,6 +22,36 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
|
||||
'--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration),
|
||||
'--no-sync', '--make-output-dir'])
|
||||
|
||||
def _get_goma_json(self):
|
||||
json_key = 'jwt_service_account_goma-client'
|
||||
json_filename = json_key + '.json'
|
||||
|
||||
# Ensure that the tmp_dir exists.
|
||||
self.m.run.run_once(self.m.file.ensure_directory,
|
||||
'makedirs tmp_dir',
|
||||
self.m.vars.tmp_dir)
|
||||
|
||||
json_file = self.m.vars.tmp_dir.join(json_filename)
|
||||
self.m.python.inline(
|
||||
'download ' + json_filename,
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import urllib2
|
||||
|
||||
TOKEN_URL = (
|
||||
'http://metadata/computeMetadata/v1/project/attributes/%s')
|
||||
|
||||
req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
|
||||
contents = urllib2.urlopen(req).read()
|
||||
|
||||
with open(sys.argv[1], 'w') as f:
|
||||
f.write(contents)
|
||||
""" % json_key,
|
||||
args=[json_file],
|
||||
infra_step=True)
|
||||
return json_file
|
||||
|
||||
def compile(self, unused_target):
|
||||
"""Build Skia with GN."""
|
||||
compiler = self.m.vars.builder_cfg.get('compiler', '')
|
||||
@ -30,6 +60,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
|
||||
os = self.m.vars.builder_cfg.get('os', '')
|
||||
target_arch = self.m.vars.builder_cfg.get('target_arch', '')
|
||||
|
||||
goma_dir = None
|
||||
clang_linux = str(self.m.vars.slave_dir.join('clang_linux'))
|
||||
emscripten_sdk = str(self.m.vars.slave_dir.join('emscripten_sdk'))
|
||||
linux_vulkan_sdk = str(self.m.vars.slave_dir.join('linux_vulkan_sdk'))
|
||||
@ -81,6 +112,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
|
||||
extra_ldflags.append('-L' + clang_linux + '/msan')
|
||||
|
||||
args = {}
|
||||
ninja_args = ['-k', '0', '-C', self.out_dir]
|
||||
env = {}
|
||||
|
||||
if configuration != 'Debug':
|
||||
args['is_debug'] = 'false'
|
||||
@ -130,6 +163,21 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
|
||||
'skia_use_icu': 'false',
|
||||
'skia_enable_gpu': 'false',
|
||||
})
|
||||
if 'Goma' in extra_config:
|
||||
json_file = self._get_goma_json()
|
||||
self.m.cipd.set_service_account_credentials(json_file)
|
||||
goma_package = ('infra_internal/goma/client/%s' %
|
||||
self.m.cipd.platform_suffix())
|
||||
goma_dir = self.m.path['cache'].join('goma')
|
||||
self.m.cipd.ensure(goma_dir, {goma_package: 'release'})
|
||||
env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = json_file
|
||||
env['GOMA_HERMETIC'] = 'error'
|
||||
env['GOMA_USE_LOCAL'] = '0'
|
||||
env['GOMA_FALLBACK'] = '0'
|
||||
with self.m.context(cwd=goma_dir, env=env):
|
||||
self._py('start goma', 'goma_ctl.py', args=['ensure_start'])
|
||||
args['cc_wrapper'] = '"%s"' % goma_dir.join('gomacc')
|
||||
ninja_args.extend(['-j', '100'])
|
||||
|
||||
sanitize = ''
|
||||
if 'SAN' in extra_config:
|
||||
@ -159,21 +207,30 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils):
|
||||
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
|
||||
gn = self.m.vars.skia_dir.join('bin', gn)
|
||||
|
||||
with self.m.context(cwd=self.m.vars.skia_dir):
|
||||
self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn'))
|
||||
env = {}
|
||||
if 'CheckGeneratedFiles' in extra_config:
|
||||
env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin')
|
||||
self._py(
|
||||
'fetch-clang-format',
|
||||
self.m.vars.skia_dir.join('bin', 'fetch-clang-format'))
|
||||
if target_arch == 'wasm':
|
||||
fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin'
|
||||
env['PATH'] = '%s:%%(PATH)s' % fastcomp
|
||||
try:
|
||||
with self.m.context(cwd=self.m.vars.skia_dir):
|
||||
self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn'))
|
||||
if 'CheckGeneratedFiles' in extra_config:
|
||||
env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin')
|
||||
self._py(
|
||||
'fetch-clang-format',
|
||||
self.m.vars.skia_dir.join('bin', 'fetch-clang-format'))
|
||||
if target_arch == 'wasm':
|
||||
fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin'
|
||||
env['PATH'] = '%s:%%(PATH)s' % fastcomp
|
||||
|
||||
with self.m.env(env):
|
||||
self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args])
|
||||
self._run('ninja', [ninja, '-k', '0', '-C', self.out_dir])
|
||||
with self.m.env(env):
|
||||
self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args])
|
||||
self._run('ninja', [ninja] + ninja_args)
|
||||
finally:
|
||||
if goma_dir:
|
||||
with self.m.context(cwd=goma_dir, env=env):
|
||||
self.m.run(self.m.python, 'print goma stats',
|
||||
script='goma_ctl.py', args=['stat'], infra_step=True,
|
||||
abort_on_failure=False, fail_build_on_failure=False)
|
||||
self.m.run(self.m.python, 'stop goma',
|
||||
script='goma_ctl.py', args=['stop'], infra_step=True,
|
||||
abort_on_failure=False, fail_build_on_failure=False)
|
||||
|
||||
def copy_extra_build_products(self, swarming_out_dir):
|
||||
configuration = self.m.vars.builder_cfg.get('configuration', '')
|
||||
|
||||
@ -401,6 +401,12 @@
|
||||
"Build-Win-Clang-x86_64-Debug-ANGLE"
|
||||
]
|
||||
},
|
||||
"Build-Win-Clang-x86_64-Debug-Goma": {
|
||||
"priority": 0.8,
|
||||
"tasks": [
|
||||
"Build-Win-Clang-x86_64-Debug-Goma"
|
||||
]
|
||||
},
|
||||
"Build-Win-Clang-x86_64-Debug-Vulkan": {
|
||||
"priority": 0.8,
|
||||
"tasks": [
|
||||
@ -4785,6 +4791,39 @@
|
||||
"isolate": "compile_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Build-Win-Clang-x86_64-Debug-Goma": {
|
||||
"cipd_packages": [
|
||||
{
|
||||
"name": "skia/bots/clang_win",
|
||||
"path": "clang_win",
|
||||
"version": "version:2"
|
||||
}
|
||||
],
|
||||
"dependencies": [
|
||||
"Housekeeper-PerCommit-IsolateWinToolchain"
|
||||
],
|
||||
"dimensions": [
|
||||
"cpu:x86-64-Haswell_GCE",
|
||||
"gpu:none",
|
||||
"os:Windows-2016Server-14393",
|
||||
"pool:Skia"
|
||||
],
|
||||
"extra_args": [
|
||||
"--workdir",
|
||||
"../../..",
|
||||
"compile",
|
||||
"repository=<(REPO)",
|
||||
"buildername=Build-Win-Clang-x86_64-Debug-Goma",
|
||||
"swarm_out_dir=${ISOLATED_OUTDIR}",
|
||||
"revision=<(REVISION)",
|
||||
"patch_repo=<(PATCH_REPO)",
|
||||
"patch_storage=<(PATCH_STORAGE)",
|
||||
"patch_issue=<(ISSUE)",
|
||||
"patch_set=<(PATCHSET)"
|
||||
],
|
||||
"isolate": "compile_skia.isolate",
|
||||
"priority": 0.8
|
||||
},
|
||||
"Build-Win-Clang-x86_64-Debug-Vulkan": {
|
||||
"cipd_packages": [
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user