AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Bundle recipes for use on RPi

Browse Source 
This saves ~1:30 of overhead when the isolate cache is warm.

Bug: skia:5813
Change-Id: I5053d66e865ca6500b82567f83be8042568e22cf
Reviewed-on: https://skia-review.googlesource.com/11063
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
This commit is contained in:
Eric Boren 2017-04-04 09:06:16 -04:00 committed by Skia Commit-Bot
parent ab015efc48
commit 8b3f9e64ff
13 changed files with 530 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
infra/bots/assets.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'files': [
'assets/',
],
},
}

14
infra/bots/bundle_recipes.isolate Normal file
View File

@ -0,0 +1,14 @@
{
'includes': [
'infrabots.isolate',
],
'variables': {
'command': [
'python', 'recipes.py', 'run', '--timestamps',
],
'files': [
'../../../.gclient',
'../config/recipes.cfg',
],
},
}

44
infra/bots/gen_tasks.go
View File

@ -26,6 +26,8 @@ import (
)
const (
BUNDLE_RECIPES_NAME = "Housekeeper-PerCommit-BundleRecipes"
DEFAULT_OS = DEFAULT_OS_LINUX
DEFAULT_OS_LINUX = "Ubuntu-14.04"
@ -214,6 +216,31 @@ func swarmDimensions(parts map[string]string) []string {
return rv
}
// bundleRecipes generates the task to bundle and isolate the recipes.
func bundleRecipes(b *specs.TasksCfgBuilder) string {
b.MustAddTask(BUNDLE_RECIPES_NAME, &specs.TaskSpec{
CipdPackages: []*specs.CipdPackage{},
Dimensions: linuxGceDimensions(),
ExtraArgs: []string{
"--workdir", "../../..", "bundle_recipes",
fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO),
fmt.Sprintf("buildername=%s", BUNDLE_RECIPES_NAME),
"mastername=fake-master",
"buildnumber=2",
"slavename=fake-buildslave",
"nobuildbot=True",
fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR),
fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION),
fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE),
fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE),
fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET),
},
Isolate: "bundle_recipes.isolate",
Priority: 0.95,
})
return BUNDLE_RECIPES_NAME
}
// compile generates a compile task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) string {
@ -441,6 +468,10 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
MaxAttempts: 1,
Priority: 0.8,
}
if parts["os"] == "Android" {
s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME)
s.Isolate = "test_skia_bundled.isolate"
}
if strings.Contains(parts["extra_config"], "Valgrind") {
s.ExecutionTimeout = 9 * time.Hour
s.Expiration = 48 * time.Hour
@ -487,6 +518,11 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
if strings.Contains(parts["extra_config"], "Skpbench") {
recipe = "swarm_skpbench"
isolate = "skpbench_skia.isolate"
if parts["os"] == "Android" {
isolate = "skpbench_skia_bundled.isolate"
}
} else if parts["os"] == "Android" {
isolate = "perf_skia_bundled.isolate"
}
s := &specs.TaskSpec{
CipdPackages: pkgs,
@ -513,6 +549,9 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
MaxAttempts: 1,
Priority: 0.8,
}
if parts["os"] == "Android" {
s.Dependencies = append(s.Dependencies, BUNDLE_RECIPES_NAME)
}
if strings.Contains(parts["extra_config"], "Valgrind") {
s.ExecutionTimeout = 9 * time.Hour
s.Expiration = 48 * time.Hour
@ -555,6 +594,11 @@ func perf(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil
func process(b *specs.TasksCfgBuilder, name string) {
deps := []string{}
// Bundle Recipes.
if name == BUNDLE_RECIPES_NAME {
deps = append(deps, bundleRecipes(b))
}
parts, err := jobNameSchema.ParseJobName(name)
if err != nil {
glog.Fatal(err)

1
infra/bots/jobs.json
View File

@ -70,6 +70,7 @@
"Build-Win-MSVC-x86_64-Release-Vulkan",
"Housekeeper-Nightly-RecreateSKPs_Canary",
"Housekeeper-PerCommit",
"Housekeeper-PerCommit-BundleRecipes",
"Housekeeper-PerCommit-InfraTests",
"Housekeeper-Weekly-RecreateSKPs",
"Perf-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Debug-Android",

14
infra/bots/perf_skia_bundled.isolate Normal file
View File

@ -0,0 +1,14 @@
{
'includes': [
'android_bin.isolate',
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe_bundled.isolate',
],
'variables': {
'files': [
'../../../.gclient',
],
},
}

1
infra/bots/recipe_modules/vars/api.py
View File

@ -57,6 +57,7 @@ class SkiaVarsApi(recipe_api.RecipeApi):
'-CT_' in self.builder_name or
'Presubmit' in self.builder_name or
'InfraTests' in self.builder_name or
'BundleRecipes' in self.builder_name or
self.builder_name == "Housekeeper-PerCommit")
if self.persistent_checkout:
if 'Win' in self.builder_name:

110
infra/bots/recipes/bundle_recipes.expected/BundleRecipes.json Normal file
View File

@ -0,0 +1,110 @@
[
{
"cmd": [
"python",
"-u",
"\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n",
"[CUSTOM_/_B_WORK]",
"511"
],
"infra_step": true,
"name": "makedirs checkout_path",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import sys, os@@@",
"@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@",
"@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec",
"cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"skia\": \"got_revision\"}",
"--git-cache-dir",
"[CUSTOM_/_B_CACHE]",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123",
"--output_manifest"
],
"cwd": "[CUSTOM_/_B_WORK]",
"env": {
"GIT_HTTP_LOW_SPEED_LIMIT": "1000",
"GIT_HTTP_LOW_SPEED_TIME": "300",
"PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]"
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"[CUSTOM_/_B_WORK]/skia/infra/bots/recipes.py",
"bundle"
],
"env": {
"BUILDTYPE": "Release",
"CHROME_HEADLESS": "1",
"PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/out"
},
"infra_step": true,
"name": "Bundle Recipes"
},
{
"cmd": [
"python",
"-u",
"\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n",
"[START_DIR]/bundle",
"[CUSTOM_[SWARM_OUT_DIR]]/recipe_bundle",
"0"
],
"name": "Copy Recipe Bundle"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

45
infra/bots/recipes/bundle_recipes.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Recipe module for Skia Swarming compile.
DEPS = [
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/shutil',
'recipe_engine/step',
'core',
'run',
'vars',
]
def RunSteps(api):
api.core.setup()
recipes_py = api.vars.infrabots_dir.join('recipes.py')
api.run(api.step, 'Bundle Recipes', infra_step=True,
cmd=['python', recipes_py, 'bundle'])
src_dir = api.path['start_dir'].join('bundle')
dst_dir = api.vars.swarming_out_dir.join('recipe_bundle')
api.shutil.copytree('Copy Recipe Bundle', src_dir, dst_dir)
api.run.check_failure()
def GenTests(api):
yield (
api.test('BundleRecipes') +
api.properties(buildername='Housekeeper-PerCommit-BundleRecipes',
mastername='fake-master',
slavename='fake-slave',
buildnumber=5,
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.path.exists(
api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
)
)

2
infra/bots/skpbench_skia.isolate
View File

@ -8,4 +8,4 @@
'../../tools/skpbench/',
],
},
}
}

12
infra/bots/skpbench_skia_bundled.isolate Normal file
View File

@ -0,0 +1,12 @@
{
'includes': [
'assets.isolate',
'swarm_recipe_bundled.isolate',
],
'variables': {
'files': [
'../../../.gclient',
'../../tools/skpbench/',
],
},
}

7
infra/bots/swarm_recipe_bundled.isolate Normal file
View File

@ -0,0 +1,7 @@
{
'variables': {
'command': [
'../../../recipe_bundle/recipes', 'run', '--timestamps',
],
},
}

412
infra/bots/tasks.json
View File

File diff suppressed because it is too large Load Diff

14
infra/bots/test_skia_bundled.isolate Normal file
View File

@ -0,0 +1,14 @@
{
'includes': [
'android_bin.isolate',
'assets.isolate',
'ios_bin.isolate',
'resources.isolate',
'swarm_recipe_bundled.isolate',
],
'variables': {
'files': [
'../../../.gclient',
],
},
}