AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[infra] Make infra tests idempotent

Browse Source 
Change-Id: I1a75b211753ea5e097234dd4104d0906cfd04673
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/223916
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Eric Boren <borenet@google.com>
This commit is contained in:
Eric Boren 2019-06-27 15:25:03 -04:00 committed by Skia Commit-Bot
parent edea01deac
commit 9e416c15a4
5 changed files with 77 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
infra/bots/gen_tasks.go
View File

@ -1057,9 +1057,13 @@ func infra(b *specs.TasksCfgBuilder, name string) string {
fmt.Sprintf("pool:%s", CONFIG.Pool), fmt.Sprintf("pool:%s", CONFIG.Pool),
} }
} }
task := kitchenTask(name, "infra", "swarm_recipe.isolate", SERVICE_ACCOUNT_COMPILE, dims, EXTRA_PROPS, OUTPUT_NONE) extraProps := map[string]string{
"repository": specs.PLACEHOLDER_REPO,
}
task := kitchenTask(name, "infra", "infra_tests.isolate", SERVICE_ACCOUNT_COMPILE, dims, extraProps, OUTPUT_NONE)
task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GSUTIL...) task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_GSUTIL...)
usesGit(task, name) task.Idempotent = true
usesGit(task, name) // We don't run bot_update, but Go needs a git repo.
usesGo(b, task, name) usesGo(b, task, name)
b.MustAddTask(name, task) b.MustAddTask(name, task)
return name return name

11
infra/bots/infra_tests.isolate Normal file
View File

@ -0,0 +1,11 @@
{
'includes': [
'swarm_recipe.isolate',
],
'variables': {
'files': [
'../../go.mod',
'../../go.sum',
],
},
}

137
infra/bots/recipes/infra.expected/infra_tests.json
View File

@ -1,105 +1,60 @@
[ [
{ {
"cmd": [ "cmd": [
"python", "git",
"-u", "init"
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
], ],
"infra_step": true, "cwd": "[START_DIR]/skia",
"name": "makedirs checkout_path" "env": {
}, "CHROME_HEADLESS": "1",
{ "GOCACHE": "[START_DIR]/cache/go_cache",
"cmd": [ "GOPATH": "[START_DIR]/cache/gopath",
"python", "GOROOT": "[START_DIR]/go/go",
"-u", "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_REPO[depot_tools]"
]
}, },
"infra_step": true, "name": "git init"
"name": "bot_update", },
"~followup_annotations": [ {
"@@@STEP_TEXT@Some step text@@@", "cmd": [
"@@@STEP_LOG_LINE@json.output@{@@@", "git",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", "add",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", "."
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", ],
"@@@STEP_LOG_LINE@json.output@ }, @@@", "cwd": "[START_DIR]/skia",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", "env": {
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", "CHROME_HEADLESS": "1",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", "GOCACHE": "[START_DIR]/cache/go_cache",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", "GOPATH": "[START_DIR]/cache/gopath",
"@@@STEP_LOG_LINE@json.output@ }@@@", "GOROOT": "[START_DIR]/go/go",
"@@@STEP_LOG_LINE@json.output@ }, @@@", "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", },
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", "name": "git add ."
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", },
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", {
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", "cmd": [
"@@@STEP_LOG_LINE@json.output@ }, @@@", "git",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", "commit",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", "-a",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", "-m",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", "initial commit"
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", ],
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@", "cwd": "[START_DIR]/skia",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", "env": {
"@@@STEP_LOG_LINE@json.output@ }@@@", "CHROME_HEADLESS": "1",
"@@@STEP_LOG_LINE@json.output@ }@@@", "GOCACHE": "[START_DIR]/cache/go_cache",
"@@@STEP_LOG_LINE@json.output@ }, @@@", "GOPATH": "[START_DIR]/cache/gopath",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", "GOROOT": "[START_DIR]/go/go",
"@@@STEP_LOG_LINE@json.output@ }, @@@", "PATH": "[START_DIR]/go/go/bin:[START_DIR]/cache/gopath/bin:<PATH>:RECIPE_REPO[depot_tools]"
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", },
"@@@STEP_LOG_LINE@json.output@}@@@", "name": "git commit"
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
}, },
{ {
"cmd": [ "cmd": [
"python", "python",
"-u", "-u",
"[START_DIR]/cache/work/skia/infra/bots/infra_tests.py" "[START_DIR]/skia/infra/bots/infra_tests.py"
], ],
"cwd": "[START_DIR]/cache/work/skia", "cwd": "[START_DIR]/skia",
"env": { "env": {
"CHROME_HEADLESS": "1", "CHROME_HEADLESS": "1",
"GOCACHE": "[START_DIR]/cache/go_cache", "GOCACHE": "[START_DIR]/cache/go_cache",

14
infra/bots/recipes/infra.py
View File

@ -7,9 +7,9 @@
DEPS = [ DEPS = [
'checkout',
'infra', 'infra',
'recipe_engine/context', 'recipe_engine/context',
'recipe_engine/path',
'recipe_engine/properties', 'recipe_engine/properties',
'recipe_engine/step', 'recipe_engine/step',
'vars', 'vars',
@ -18,14 +18,12 @@ DEPS = [
def RunSteps(api): def RunSteps(api):
api.vars.setup() api.vars.setup()
checkout_root = api.checkout.default_checkout_root
api.checkout.bot_update(checkout_root=checkout_root)
# Run the infra tests. # Run the infra tests.
repo_name = api.properties['repository'].split('/')[-1] repo_name = api.properties['repository'].split('/')[-1]
if repo_name.endswith('.git'): if repo_name.endswith('.git'):
repo_name = repo_name[:-len('.git')] repo_name = repo_name[:-len('.git')]
repo_root = checkout_root.join(repo_name) repo_root = api.path['start_dir'].join(repo_name)
infra_tests = repo_root.join('infra', 'bots', 'infra_tests.py') infra_tests = repo_root.join('infra', 'bots', 'infra_tests.py')
# Merge the default environment with the Go environment. # Merge the default environment with the Go environment.
@ -40,7 +38,12 @@ def RunSteps(api):
else: else:
env[k] = v env[k] = v
with api.context(cwd=checkout_root.join(repo_name), env=env): with api.context(cwd=repo_root, env=env):
# Some tests assume that they're being run inside a git repo.
api.step('git init', cmd=['git', 'init'])
api.step('git add .', cmd=['git', 'add', '.'])
api.step('git commit', cmd=['git', 'commit', '-a', '-m', 'initial commit'])
# Unfortunately, the recipe tests are flaky due to file removal on Windows. # Unfortunately, the recipe tests are flaky due to file removal on Windows.
# Run multiple attempts. # Run multiple attempts.
last_exc = None last_exc = None
@ -58,7 +61,6 @@ def GenTests(api):
api.test('infra_tests') + api.test('infra_tests') +
api.properties(buildername='Housekeeper-PerCommit-InfraTests_Win', api.properties(buildername='Housekeeper-PerCommit-InfraTests_Win',
repository='https://skia.googlesource.com/skia.git', repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen', path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') swarm_out_dir='[SWARM_OUT_DIR]')
) )

10
infra/bots/tasks.json
View File

@ -17485,7 +17485,7 @@
"skia/infra/bots/run_recipe.py", "skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}", "${ISOLATED_OUTDIR}",
"infra", "infra",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}", "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Linux\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia" "skia"
], ],
"dependencies": [ "dependencies": [
@ -17511,8 +17511,9 @@
"extra_tags": { "extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations" "log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
}, },
"idempotent": true,
"io_timeout_ns": 3600000000000, "io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate", "isolate": "infra_tests.isolate",
"max_attempts": 2, "max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com" "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
}, },
@ -17595,7 +17596,7 @@
"skia/infra/bots/run_recipe.py", "skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}", "${ISOLATED_OUTDIR}",
"infra", "infra",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"patch_issue\":\"<(ISSUE)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}", "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Housekeeper-PerCommit-InfraTests_Win\",\"repository\":\"<(REPO)\",\"swarm_out_dir\":\"output_ignored\"}",
"skia" "skia"
], ],
"dependencies": [ "dependencies": [
@ -17621,8 +17622,9 @@
"extra_tags": { "extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations" "log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
}, },
"idempotent": true,
"io_timeout_ns": 3600000000000, "io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate", "isolate": "infra_tests.isolate",
"max_attempts": 2, "max_attempts": 2,
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com" "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
}, },