AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[infra] Make infra tests idempotent

Browse Source 
Change-Id: I1a75b211753ea5e097234dd4104d0906cfd04673
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/223916
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
This commit is contained in:
Eric Boren 2019-06-27 15:25:03 -04:00 committed by Skia Commit-Bot
parent edea01deac
commit 9e416c15a4
5 changed files with 77 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
infra/bots/gen_tasks.go
View File

@ -1057,9 +1057,13 @@ func infra(b *specs.TasksCfgBuilder, name string) string {
fmt.Sprintf("pool:%s", CONFIG.Pool),
}
}
task := kitchenTask(name, "infra", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, dims, EXTRA_PROPS, OUTPUT_NONE)
extraProps := map[string]string{
"repository": specs.PLACEHOLDER_REPO,
}
task := kitchenTask(name, "infra", "infra_tests.isolate", SERVICE_ACCOUNT_COMPILE, dims, extraProps, OUTPUT_NONE)
task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GSUTIL...)
usesGit(task, name)
task.Idempotent = true
usesGit(task, name) // We don't run bot_update, but Go needs a git repo.
usesGo(b, task, name)
b.MustAddTask(name, task)
return name

11
infra/bots/infra_tests.isolate Normal file
View File

@ -0,0 +1,11 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../go.mod',
'../../go.sum',
],
},
}

137
infra/bots/recipes/infra.expected/infra_tests.json
View File

@ -1,105 +1,60 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
"git",
"init"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_REPO[depot_tools]"
]
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache",
"GOPATH": "[START_DIR]/cache/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
"name": "git init"
},
{
"cmd": [
"git",
"add",
"."
],
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache",
"GOPATH": "[START_DIR]/cache/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
},
"name": "git add ."
},
{
"cmd": [
"git",
"commit",
"-a",
"-m",
"initial commit"
],
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache",
"GOPATH": "[START_DIR]/cache/gopath",
"GOROOT": "[START_DIR]/go/go",
"PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
},
"name": "git commit"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/cache/work/skia/infra/bots/infra_tests.py"
"[START_DIR]/skia/infra/bots/infra_tests.py"
],
"cwd": "[START_DIR]/cache/work/skia",
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache",

14
infra/bots/recipes/infra.py
View File

@ -7,9 +7,9 @@
DEPS = [
'checkout',
'infra',
'recipe_engine/context',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
'vars',
@ -18,14 +18,12 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
checkout_root = api.checkout.default_checkout_root
api.checkout.bot_update(checkout_root=checkout_root)
# Run the infra tests.
repo_name = api.properties['repository'].split('/')[-1]
if repo_name.endswith('.git'):
repo_name = repo_name[:-len('.git')]
repo_root = checkout_root.join(repo_name)
repo_root = api.path['start_dir'].join(repo_name)
infra_tests = repo_root.join('infra', 'bots', 'infra_tests.py')
# Merge the default environment with the Go environment.
@ -40,7 +38,12 @@ def RunSteps(api):
else:
env[k] = v
with api.context(cwd=checkout_root.join(repo_name), env=env):
with api.context(cwd=repo_root, env=env):
# Some tests assume that they're being run inside a git repo.
api.step('git init', cmd=['git', 'init'])
api.step('git add .', cmd=['git', 'add', '.'])
api.step('git commit', cmd=['git', 'commit', '-a', '-m', 'initial commit'])
# Unfortunately, the recipe tests are flaky due to file removal on Windows.
# Run multiple attempts.
last_exc = None
@ -58,7 +61,6 @@ def GenTests(api):
api.test('infra_tests') +
api.properties(buildername='Housekeeper-PerCommit-InfraTests_Win',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]')
)

10
infra/bots/tasks.json
View File

@ -17485,7 +17485,7 @@
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"infra",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia"
],
"dependencies": [
@ -17511,8 +17511,9 @@
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
"idempotent": true,
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"isolate": "infra_tests.isolate",
"max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
@ -17595,7 +17596,7 @@
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"infra",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia"
],
"dependencies": [
@ -17621,8 +17622,9 @@
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
"idempotent": true,
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"isolate": "infra_tests.isolate",
"max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},