AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add bot which builds without DEPS

Browse Source 
Bug: skia:7646
Change-Id: Iddf55940230c99fea9d838a1b03a8c3bb476262e
Reviewed-on: https://skia-review.googlesource.com/109360
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Mike Klein <mtklein@google.com>
This commit is contained in:
Eric Boren 2018-02-22 10:03:56 -05:00 committed by Skia Commit-Bot
parent 85fdbe2b5c
commit 86a1146852
15 changed files with 480 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infra/bots/jobs.json
View File

@ -37,6 +37,7 @@
"Build-Debian9-Clang-x86_64-Release-Chromebook_GLES",
"Build-Debian9-Clang-x86_64-Release-Fast",
"Build-Debian9-Clang-x86_64-Release-Mini",
"Build-Debian9-Clang-x86_64-Release-NoDEPS",
"Build-Debian9-Clang-x86_64-Release-ParentRevision",
"Build-Debian9-Clang-x86_64-Release-SKNX_NO_SIMD",
"Build-Debian9-Clang-x86_64-Release-SK_CPU_LIMIT_SSE2",

33
infra/bots/recipe_modules/core/api.py
View File

@ -17,13 +17,16 @@ from recipe_engine import config_types
class SkiaApi(recipe_api.RecipeApi):
def setup(self):
def setup(self, bot_update=True):
"""Prepare the bot to run."""
# Setup dependencies.
self.m.vars.setup()
# Check out the Skia code.
self.checkout_steps()
if bot_update:
self.checkout_bot_update()
else:
self.checkout_git()
if not self.m.path.exists(self.m.vars.tmp_dir):
self.m.run.run_once(self.m.file.ensure_directory,
@ -32,8 +35,23 @@ class SkiaApi(recipe_api.RecipeApi):
self.m.flavor.setup()
def checkout_steps(self):
"""Run the steps to obtain a checkout of Skia."""
def patch_ref(self, issue, patchset):
"""Build a ref for the given issue and patchset."""
return 'refs/changes/%s/%s/%s' % (issue[-2:], issue, patchset)
def checkout_git(self):
"""Run the steps to perform a pure-git checkout without DEPS."""
self.m.git.checkout(
self.m.properties['repository'], dir_path=self.m.vars.skia_dir,
ref=self.m.properties['revision'], submodules=False)
if self.m.vars.is_trybot:
ref = self.patch_ref(str(self.m.vars.issue), str(self.m.vars.patchset))
self.m.git('fetch', 'origin', ref)
self.m.git('checkout', 'FETCH_HEAD')
self.m.git('rebase', self.m.properties['revision'])
def checkout_bot_update(self):
"""Run the steps to obtain a checkout using bot_update."""
cfg_kwargs = {}
is_parent_revision = 'ParentRevision' in self.m.vars.extra_tokens
if not self.m.vars.persistent_checkout:
@ -140,11 +158,8 @@ class SkiaApi(recipe_api.RecipeApi):
# Hack the patch ref if necessary.
if self.m.bot_update._issue and self.m.bot_update._patchset:
self.m.bot_update._gerrit_ref = 'refs/changes/%s/%d/%d' % (
str(self.m.bot_update._issue)[-2:],
self.m.bot_update._issue,
self.m.bot_update._patchset,
)
self.m.bot_update._gerrit_ref = self.patch_ref(
str(self.m.bot_update._issue), str(self.m.bot_update._patchset))
self.m.bot_update._repository = patch_repo
if not self.m.vars.is_trybot and is_parent_revision:

118
infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json Normal file
View File

@ -0,0 +1,118 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
"--path",
"[START_DIR]/skia",
"--url",
"https://skia.googlesource.com/skia.git"
],
"name": "git setup"
},
{
"cmd": [
"git",
"retry",
"fetch",
"origin",
"abc123",
"--progress"
],
"cwd": "[START_DIR]/skia",
"env": {
"PATH": "RECIPE_PACKAGE_REPO[depot_tools]:<PATH>"
},
"infra_step": true,
"name": "git fetch"
},
{
"cmd": [
"git",
"checkout",
"-f",
"FETCH_HEAD"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git checkout"
},
{
"cmd": [
"git",
"rev-parse",
"HEAD"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "read revision",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@"
]
},
{
"cmd": [
"git",
"clean",
"-f",
"-d",
"-x"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git clean"
},
{
"cmd": [
"git",
"fetch",
"origin",
"refs/changes/00/500/1"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git fetch (2)"
},
{
"cmd": [
"git",
"checkout",
"FETCH_HEAD"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git checkout (2)"
},
{
"cmd": [
"git",
"rebase",
"abc123"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git rebase"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

20
infra/bots/recipe_modules/core/examples/full.py
View File

@ -11,7 +11,10 @@ DEPS = [
def RunSteps(api):
api.core.setup()
bot_update = True
if 'NoDEPS' in api.properties['buildername']:
bot_update = False
api.core.setup(bot_update=bot_update)
def GenTests(api):
@ -130,6 +133,21 @@ def GenTests(api):
api.path.exists(api.path['start_dir'].join('skp_output'))
)
builder = 'Build-Debian9-Clang-x86_64-Release-NoDEPS'
yield (
api.test(builder) +
api.properties(buildername=builder,
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]',
patch_issue=500,
patch_repo='https://skia.googlesource.com/skia.git',
patch_set=1,
patch_storage='gerrit') +
api.path.exists(api.path['start_dir'].join('skp_output'))
)
buildername = 'Build-Debian9-GCC-x86_64-Release'
yield (
api.test('cross_repo_trybot') +

56
infra/bots/recipe_modules/flavor/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json Normal file
View File

@ -0,0 +1,56 @@
[
{
"cmd": [
"python",
"-u",
"[START_DIR]/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/skia/bin/gn",
"gen",
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false is_official_build=true skia_enable_effects=false skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\""
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"name": "ninja"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

1
infra/bots/recipe_modules/flavor/examples/full.py
View File

@ -65,6 +65,7 @@ TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Debug-SK_USE_DISCARDABLE_SCALEDIMAGECACHE',
'Build-Debian9-Clang-x86_64-Release-Fast',
'Build-Debian9-Clang-x86_64-Release-Mini',
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-Vulkan',
'Build-Debian9-EMCC-wasm-Release',
'Build-Debian9-GCC-x86_64-Debug-EmbededResouces',

16
infra/bots/recipe_modules/flavor/gn_flavor.py
View File

@ -143,6 +143,22 @@ with open(sys.argv[1], 'w') as f:
'skia_use_libwebp': 'false',
'skia_use_zlib': 'false',
})
if 'NoDEPS' in extra_tokens:
args.update({
'is_official_build': 'true',
'skia_enable_fontmgr_empty': 'true',
'skia_enable_gpu': 'true',
'skia_enable_effects': 'false',
'skia_enable_pdf': 'false',
'skia_use_expat': 'false',
'skia_use_freetype': 'false',
'skia_use_libjpeg_turbo': 'false',
'skia_use_libpng': 'false',
'skia_use_libwebp': 'false',
'skia_use_vulkan': 'false',
'skia_use_zlib': 'false',
})
if 'NoGPU' in extra_tokens:
args['skia_enable_gpu'] = 'false'
if 'EmbededResouces' in extra_tokens:

2
infra/bots/recipe_modules/vars/api.py
View File

@ -39,7 +39,7 @@ class SkiaVarsApi(recipe_api.RecipeApi):
self.persistent_checkout = False
# Compile bots keep a persistent checkout.
if self.is_compile_bot:
if self.is_compile_bot and 'NoDEPS' not in self.builder_name:
self.persistent_checkout = True
if 'Housekeeper' in self.builder_name:
self.persistent_checkout = True

35
infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json Normal file
View File

@ -0,0 +1,35 @@
[
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n"
],
"name": "get swarming bot id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n"
],
"name": "get swarming task id",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

1
infra/bots/recipe_modules/vars/examples/full.py
View File

@ -22,6 +22,7 @@ def RunSteps(api):
TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-ParentRevision',
'Build-Debian9-Clang-x86_64-Release-SKNX_NO_SIMD',
'Build-Debian9-GCC-x86_64-Release-Flutter_Android',

2
infra/bots/recipes/bookmaker.py
View File

@ -43,7 +43,7 @@ def go_get_fiddlecli(api):
def RunSteps(api):
api.vars.setup()
api.core.checkout_steps()
api.core.checkout_bot_update()
api.infra.go_version()
go_get_fiddlecli(api)

165
infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json Normal file
View File

@ -0,0 +1,165 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
"--path",
"[START_DIR]/skia",
"--url",
"https://skia.googlesource.com/skia.git"
],
"name": "git setup"
},
{
"cmd": [
"git",
"retry",
"fetch",
"origin",
"abc123",
"--progress"
],
"cwd": "[START_DIR]/skia",
"env": {
"PATH": "RECIPE_PACKAGE_REPO[depot_tools]:<PATH>"
},
"infra_step": true,
"name": "git fetch"
},
{
"cmd": [
"git",
"checkout",
"-f",
"FETCH_HEAD"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git checkout"
},
{
"cmd": [
"git",
"rev-parse",
"HEAD"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "read revision",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@"
]
},
{
"cmd": [
"git",
"clean",
"-f",
"-d",
"-x"
],
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git clean"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CC": "/usr/bin/clang",
"CHROME_HEADLESS": "1",
"CXX": "/usr/bin/clang++",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/skia/bin/gn",
"gen",
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false is_official_build=true skia_enable_effects=false skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\""
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CC": "/usr/bin/clang",
"CHROME_HEADLESS": "1",
"CXX": "/usr/bin/clang++",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release",
"CC": "/usr/bin/clang",
"CHROME_HEADLESS": "1",
"CXX": "/usr/bin/clang++",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
"[CUSTOM_[SWARM_OUT_DIR]]/out/Release"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

6
infra/bots/recipes/compile.py
View File

@ -43,7 +43,10 @@ def get_extra_env_vars(vars_api):
def RunSteps(api):
api.core.setup()
bot_update=True
if 'NoDEPS' in api.properties['buildername']:
bot_update = False
api.core.setup(bot_update=bot_update)
env = get_extra_env_vars(api.vars)
build_targets = build_targets_from_builder_dict(api.vars.builder_cfg)
@ -88,6 +91,7 @@ TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-Chromebook_GLES',
'Build-Debian9-Clang-x86_64-Release-Fast',
'Build-Debian9-Clang-x86_64-Release-Mini',
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-Vulkan',
'Build-Debian9-Clang-x86_64-Release-Vulkan_Coverage',
'Build-Debian9-EMCC-wasm-Release',

2
infra/bots/recipes/infra.py
View File

@ -20,7 +20,7 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
api.core.checkout_steps()
api.core.checkout_bot_update()
api.infra.update_go_deps()
# Run the infra tests.

36
infra/bots/tasks.json
View File

@ -228,6 +228,12 @@
"Build-Debian9-Clang-x86_64-Release-Mini"
]
},
"Build-Debian9-Clang-x86_64-Release-NoDEPS": {
"priority": 0.8,
"tasks": [
"Build-Debian9-Clang-x86_64-Release-NoDEPS"
]
},
"Build-Debian9-Clang-x86_64-Release-ParentRevision": {
"priority": 0.8,
"tasks": [
@ -4385,6 +4391,36 @@
"isolate": "compile_skia.isolate",
"priority": 0.8
},
"Build-Debian9-Clang-x86_64-Release-NoDEPS": {
"cipd_packages": [
{
"name": "skia/bots/clang_linux",
"path": "clang_linux",
"version": "version:10"
}
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"extra_args": [
"--workdir",
"../../..",
"compile",
"repository=<(REPO)",
"buildername=Build-Debian9-Clang-x86_64-Release-NoDEPS",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)",
"patch_repo=<(PATCH_REPO)",
"patch_storage=<(PATCH_STORAGE)",
"patch_issue=<(ISSUE)",
"patch_set=<(PATCHSET)"
],
"isolate": "compile_skia.isolate",
"priority": 0.8
},
"Build-Debian9-Clang-x86_64-Release-ParentRevision": {
"cipd_packages": [
{