AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Revert "[infra] Make most builds idempotent"

Browse Source 
This reverts commit 57aa178dea.

Reason for revert: Causes Build-Debian9-Clang-x86-devrel-Android_SKQP to fail

Original change's description:
> [infra] Make most builds idempotent
> 
> Change-Id: I084645a46feda183747e9d7fc3b40b1aa56c5f78
> Reviewed-on: https://skia-review.googlesource.com/c/skia/+/211103
> Commit-Queue: Eric Boren <borenet@google.com>
> Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>

TBR=borenet@google.com,benjaminwagner@google.com

Change-Id: I886483a01c53da2cb7816651a02b35b780ffcf67
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/228441
Reviewed-by: Ben Wagner aka dogben <benjaminwagner@google.com>
Commit-Queue: Ben Wagner aka dogben <benjaminwagner@google.com>
This commit is contained in:
Ben Wagner aka dogben 2019-07-18 21:30:04 +00:00 committed by Skia Commit-Bot
parent 4456a0d35c
commit de71a74fc4
14 changed files with 4016 additions and 883 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
PRESUBMIT.py
View File

@ -192,28 +192,6 @@ def _CheckGNFormatted(input_api, output_api):
return results
def _CheckCompileIsolate(input_api, output_api):
"""Ensure that gen_compile_isolate.py does not change compile.isolate."""
# Only run the check if files were added or removed.
results = []
script = os.path.join('infra', 'bots', 'gen_compile_isolate.py')
isolate = os.path.join('infra', 'bots', 'compile.isolated')
for f in input_api.AffectedFiles():
if f.Action() in ('A', 'D', 'R'):
break
if f.LocalPath() in (script, isolate):
break
else:
return results
cmd = ['python', script, 'test']
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
results.append(output_api.PresubmitError(e.output))
return results
class _WarningsAsErrors():
def __init__(self, output_api):
self.output_api = output_api
@ -250,7 +228,6 @@ def _CommonChecks(input_api, output_api):
results.extend(_CopyrightChecks(input_api, output_api,
source_file_filter=sources))
results.extend(_ToolFlags(input_api, output_api))
results.extend(_CheckCompileIsolate(input_api, output_api))
return results

47
infra/bots/compile.isolate
View File

@ -1,47 +0,0 @@
{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
'../../../.gclient',
'../../.clang-format',
'../../.clang-tidy',
'../../.gn',
'../../BUILD.gn',
'../../bench',
'../../bin/fetch-clang-format',
'../../bin/fetch-gn',
'../../buildtools',
'../../dm',
'../../docs/examples',
'../../example',
'../../experimental',
'../../fuzz',
'../../gm',
'../../gn',
'../../include',
'../../modules',
'../../platform_tools/android/apps/arcore/src/main/cpp',
'../../platform_tools/android/apps/skottie/src/main/cpp',
'../../platform_tools/android/launcher/skia_launcher.cpp',
'../../platform_tools/android/vulkan/Skia_Vulkan_Android.h',
'../../platform_tools/libraries/include/arcore_c_api.h',
'../../resources',
'../../samplecode',
'../../src',
'../../tests',
'../../third_party',
'../../tools',
'../canvaskit',
'../pathkit',
'assets/android_ndk_darwin/VERSION',
'assets/android_ndk_linux/VERSION',
'assets/android_ndk_windows/VERSION',
'assets/cast_toolchain/VERSION',
'assets/clang_linux/VERSION',
'assets/clang_win/VERSION',
'assets/mips64el_toolchain_linux/VERSION',
],
},
}

233
infra/bots/gen_compile_isolate.py
View File

@ -1,233 +0,0 @@
#!/usr/bin/env python
#
# Copyright 2019 Google LLC
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import difflib
import os
import re
import subprocess
import sys
# Any files in Git which match these patterns will be included, either directly
# or indirectly via a parent dir.
PATH_PATTERNS = [
r'.*\.c$',
r'.*\.cc$',
r'.*\.cpp$',
r'.*\.gn$',
r'.*\.gni$',
r'.*\.h$',
]
# These paths are always added to the inclusion list. Note that they may not
# appear in the isolate if they are included indirectly via a parent dir.
EXPLICIT_PATHS = [
'../.gclient',
'.clang-format',
'.clang-tidy',
'bin/fetch-clang-format',
'bin/fetch-gn',
'buildtools',
'infra/bots/assets/android_ndk_darwin/VERSION',
'infra/bots/assets/android_ndk_linux/VERSION',
'infra/bots/assets/android_ndk_windows/VERSION',
'infra/bots/assets/cast_toolchain/VERSION',
'infra/bots/assets/clang_linux/VERSION',
'infra/bots/assets/clang_win/VERSION',
'infra/bots/assets/mips64el_toolchain_linux/VERSION',
'infra/canvaskit',
'infra/pathkit',
'resources',
'third_party/externals',
]
# If a parent path contains more than this many immediate child paths (ie. files
# and dirs which are directly inside it as opposed to indirect descendants), we
# will include the parent in the isolate file instead of the children. This
# results in a simpler isolate file which should need to be changed less often.
COMBINE_PATHS_THRESHOLD = 3
# Template for the isolate file content.
ISOLATE_TMPL = '''{
'includes': [
'run_recipe.isolate',
],
'variables': {
'files': [
%s
],
},
}
'''
# Absolute path to the infra/bots dir.
INFRABOTS_DIR = os.path.realpath(os.path.dirname(os.path.abspath(__file__)))
# Absolute path to the compile.isolate file.
ISOLATE_FILE = os.path.join(INFRABOTS_DIR, 'compile.isolate')
def all_paths():
"""Return all paths which are checked in to git."""
repo_root = os.path.abspath(os.path.join(INFRABOTS_DIR, os.pardir, os.pardir))
output = subprocess.check_output(['git', 'ls-files'], cwd=repo_root).rstrip()
return output.splitlines()
def get_relevant_paths():
"""Return all checked-in paths in PATH_PATTERNS or EXPLICIT_PATHS."""
paths = []
for f in all_paths():
for regexp in PATH_PATTERNS:
if re.match(regexp, f):
paths.append(f)
break
paths.extend(EXPLICIT_PATHS)
return paths
class Tree(object):
"""Tree helps with deduplicating and collapsing paths."""
class Node(object):
"""Node represents an individual node in a Tree."""
def __init__(self, name):
self._children = {}
self._name = name
self._is_leaf = False
@property
def is_root(self):
"""Return True iff this is the root node."""
return self._name is None
def add(self, entry):
"""Add the given entry (given as a list of strings) to the Node."""
# Remove the first element if we're not the root node.
if not self.is_root:
if entry[0] != self._name:
raise ValueError('Cannot add a non-matching entry to a Node!')
entry = entry[1:]
# If the entry is now empty, this node is a leaf.
if not entry:
self._is_leaf = True
return
# Add a child node.
if not self._is_leaf:
child = self._children.get(entry[0])
if not child:
child = Tree.Node(entry[0])
self._children[entry[0]] = child
child.add(entry)
# If we have more than COMBINE_PATHS_THRESHOLD immediate children,
# combine them into this node.
immediate_children = 0
for child in self._children.itervalues():
if child._is_leaf:
immediate_children += 1
if not self.is_root and immediate_children >= COMBINE_PATHS_THRESHOLD:
self._is_leaf = True
self._children = {}
def entries(self):
"""Return the entries represented by this node and its children.
Will not return children in the following cases:
- This Node is a leaf, ie. it represents an entry which was explicitly
inserted into the Tree, as opposed to only part of a path to other
entries.
- This Node has immediate children exceeding COMBINE_PATHS_THRESHOLD and
thus has been upgraded to a leaf node.
"""
if self._is_leaf:
return [self._name]
rv = []
for child in self._children.itervalues():
for entry in child.entries():
if not self.is_root:
entry = self._name + '/' + entry
rv.append(entry)
return rv
def __init__(self):
self._root = Tree.Node(None)
def add(self, entry):
"""Add the given entry to the tree."""
split = entry.split('/')
if split[-1] == '':
split = split[:-1]
self._root.add(split)
def entries(self):
"""Return the list of entries in the tree.
Entries will be de-duplicated as follows:
- Any entry which is a sub-path of another entry will not be returned.
- Any entry which was not explicitly inserted but has children exceeding
the COMBINE_PATHS_THRESHOLD will be returned while its children will not
be returned.
"""
return self._root.entries()
def relpath(repo_path):
"""Return a relative path to the given path within the repo.
The path is relative to the infra/bots dir, where the compile.isolate file
lives.
"""
repo_path = '../../' + repo_path
repo_path = repo_path.replace('../../infra/', '../')
repo_path = repo_path.replace('../bots/', '')
return repo_path
def get_isolate_content(paths):
"""Construct the new content of the isolate file based on the given paths."""
lines = [' \'%s\',' % relpath(p) for p in paths]
lines.sort()
return ISOLATE_TMPL % '\n'.join(lines)
def main():
"""Regenerate the compile.isolate file, or verify that it hasn't changed."""
testing = False
if len(sys.argv) == 2 and sys.argv[1] == 'test':
testing = True
elif len(sys.argv) != 1:
print >> sys.stderr, 'Usage: %s [test]' % sys.argv[0]
sys.exit(1)
tree = Tree()
for p in get_relevant_paths():
tree.add(p)
content = get_isolate_content(tree.entries())
if testing:
with open(ISOLATE_FILE, 'rb') as f:
expect_content = f.read()
if content != expect_content:
print >> sys.stderr, 'Found diff in %s:' % ISOLATE_FILE
a = expect_content.splitlines()
b = content.splitlines()
diff = difflib.context_diff(a, b, lineterm='')
for line in diff:
sys.stderr.write(line + '\n')
print >> sys.stderr, 'You may need to run:\n\n\tpython %s' % sys.argv[0]
sys.exit(1)
else:
with open(ISOLATE_FILE, 'wb') as f:
f.write(content)
if __name__ == '__main__':
main()

1
infra/bots/gen_tasks.go
View File

@ -12,7 +12,6 @@ import (
"go.skia.org/skia/infra/bots/gen_tasks_logic"
)
// Regenerate the tasks.json file.
func main() {
gen_tasks_logic.GenTasks(nil)
}

22
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -997,26 +997,8 @@ func attempts(name string) int {
// compile generates a compile task. Returns the name of the last task in the
// generated chain of tasks, which the Job should add as a dependency.
func (b *builder) compile(name string, parts map[string]string) string {
recipe := "compile"
isolate := "compile.isolate"
var props map[string]string
needSync := false
if strings.Contains(name, "NoDEPS") ||
strings.Contains(name, "CMake") ||
strings.Contains(name, "CommandBuffer") ||
strings.Contains(name, "Flutter") ||
strings.Contains(name, "ParentRevision") {
recipe = "sync_and_compile"
isolate = "swarm_recipe.isolate"
props = EXTRA_PROPS
needSync = true
}
task := b.kitchenTask(name, recipe, isolate, b.cfg.ServiceAccountCompile, b.swarmDimensions(parts), props, OUTPUT_BUILD)
if needSync {
b.usesGit(task, name)
} else {
task.Idempotent = true
}
task := b.kitchenTask(name, "compile", "swarm_recipe.isolate", b.cfg.ServiceAccountCompile, b.swarmDimensions(parts), EXTRA_PROPS, OUTPUT_BUILD)
b.usesGit(task, name)
usesDocker(task, name)
// Android bots require a toolchain.

113
infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json
View File

@ -1,4 +1,113 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]/cache/work/.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]/cache/work/.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]/cache/git",
"--cleanup-dir",
"[CLEANUP]/bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123"
],
"cwd": "[START_DIR]/cache/work",
"env_prefixes": {
"PATH": [
"RECIPE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"cmd": [
"python",
@ -22,7 +131,7 @@
"--workdir",
"/SRC/skia/infra/skqp",
"--volume",
"[START_DIR]:/SRC",
"[START_DIR]/cache/work:/SRC",
"--volume",
"[START_DIR]/cache/docker/skqp:/OUT",
"gcr.io/skia-public/android-skqp:r20_v1",
@ -122,7 +231,7 @@
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/skia/infra/cts/whitelist_devices.json",
"[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json",
"[START_DIR]/[SWARM_OUT_DIR]"
],
"infra_step": true,

69
infra/bots/recipes/sync_and_compile.expected/Build-Win10-Clang-x86_64-Release-NoDEPS.json → infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json
View File

@ -3,9 +3,9 @@
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::git]\\resources\\git_setup.py",
"RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
"--path",
"[START_DIR]\\skia",
"[START_DIR]/skia",
"--url",
"https://skia.googlesource.com/skia.git"
],
@ -20,9 +20,9 @@
"abc123",
"--progress"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"env": {
"PATH": "RECIPE_REPO[depot_tools];<PATH>"
"PATH": "RECIPE_REPO[depot_tools]:<PATH>"
},
"infra_step": true,
"name": "git fetch"
@ -34,7 +34,7 @@
"-f",
"FETCH_HEAD"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git checkout"
},
@ -44,7 +44,7 @@
"rev-parse",
"HEAD"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "read revision",
"~followup_annotations": [
@ -59,7 +59,7 @@
"-d",
"-x"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"infra_step": true,
"name": "git clean"
},
@ -67,13 +67,13 @@
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]\\tmp"
"[START_DIR]/tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
@ -82,41 +82,41 @@
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
"[START_DIR]/skia/infra/bots/assets/clang_linux/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get clang_win VERSION"
"name": "Get clang_linux VERSION"
},
{
"cmd": [
"python",
"-u",
"[START_DIR]\\skia\\bin\\fetch-gn"
"[START_DIR]/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"
"PATH": "<PATH>:RECIPE_REPO[depot_tools]"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]\\skia\\bin\\gn",
"[START_DIR]/skia/bin/gn",
"gen",
"[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64",
"--args=cc=\"clang\" clang_win=\"[START_DIR]\\clang_win\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_clang_win_version=42\"] is_debug=false is_official_build=true skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_harfbuzz=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\" werror=true win_sdk=\"[START_DIR]\\win_toolchain/win_sdk\" win_vc=\"[START_DIR]\\win_toolchain/VC\""
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
"--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false is_official_build=true skia_enable_fontmgr_empty=true skia_enable_gpu=true skia_enable_pdf=false skia_use_expat=false skia_use_freetype=false skia_use_harfbuzz=false skia_use_libjpeg_turbo=false skia_use_libpng=false skia_use_libwebp=false skia_use_vulkan=false skia_use_zlib=false target_cpu=\"x86_64\" werror=true"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"
"PATH": "<PATH>:RECIPE_REPO[depot_tools]"
},
"name": "gn gen"
},
@ -124,12 +124,12 @@
"cmd": [
"ninja",
"-C",
"[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64"
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"
"PATH": "<PATH>:RECIPE_REPO[depot_tools]"
},
"name": "ninja"
},
@ -138,8 +138,8 @@
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'skottie_tool', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]\\skia\\out\\Build-Win10-Clang-x86_64-Release-NoDEPS\\Release_x64",
"[START_DIR]\\[SWARM_OUT_DIR]"
"[START_DIR]/skia/out/Build-Debian9-Clang-x86_64-Release-NoDEPS/Release",
"[START_DIR]/[SWARM_OUT_DIR]"
],
"infra_step": true,
"name": "copy build products",
@ -171,25 +171,6 @@
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n"
],
"infra_step": true,
"name": "cleanup",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import psutil@@@",
"@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@",
"@@@STEP_LOG_LINE@python.inline@ try:@@@",
"@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@",
"@@@STEP_LOG_LINE@python.inline@ p.kill()@@@",
"@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@",
"@@@STEP_LOG_LINE@python.inline@ pass@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result"
}

0
infra/bots/recipes/sync_and_compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json → infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json
View File

0
infra/bots/recipes/sync_and_compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json → infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json
View File

0
infra/bots/recipes/sync_and_compile.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json → infra/bots/recipes/compile.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json
View File

121
infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json
View File

@ -1,4 +1,113 @@
[
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]\\cache\\work"
],
"infra_step": true,
"name": "makedirs checkout_path"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
"--json-output",
"/path/to/tmp/json",
"remove",
"[START_DIR]\\cache\\work\\.gclient_entries"
],
"infra_step": true,
"name": "remove [START_DIR]\\cache\\work\\.gclient_entries"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[depot_tools::bot_update]\\resources\\bot_update.py",
"--spec-path",
"cache_dir = '[START_DIR]\\\\cache\\\\git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]",
"--patch_root",
"skia",
"--revision_mapping_file",
"{\"got_revision\": \"skia\"}",
"--git-cache-dir",
"[START_DIR]\\cache\\git",
"--cleanup-dir",
"[CLEANUP]\\bot_update",
"--output_json",
"/path/to/tmp/json",
"--revision",
"skia@abc123"
],
"cwd": "[START_DIR]\\cache\\work",
"env_prefixes": {
"PATH": [
"RECIPE_REPO[depot_tools]"
]
},
"infra_step": true,
"name": "bot_update",
"~followup_annotations": [
"@@@STEP_TEXT@Some step text@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
"@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
"@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
"@@@STEP_LOG_LINE@json.output@}@@@",
"@@@STEP_LOG_END@json.output@@@",
"@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@",
"@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]\\resources\\fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]\\tmp"
],
"infra_step": true,
"name": "makedirs tmp_dir"
},
{
"cmd": [
"python",
@ -7,7 +116,7 @@
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
"[START_DIR]\\cache\\work\\skia\\infra\\bots\\assets\\clang_win\\VERSION",
"/path/to/tmp/"
],
"infra_step": true,
@ -17,9 +126,9 @@
"cmd": [
"python",
"-u",
"[START_DIR]\\skia\\bin\\fetch-gn"
"[START_DIR]\\cache\\work\\skia\\bin\\fetch-gn"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]\\cache\\work\\skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"
@ -29,12 +138,12 @@
},
{
"cmd": [
"[START_DIR]\\skia\\bin\\gn",
"[START_DIR]\\cache\\work\\skia\\bin\\gn",
"gen",
"[START_DIR]\\cache\\work\\skia\\out\\Build-Win-Clang-x86-Debug\\Debug",
"--args=cc=\"clang\" clang_win=\"[START_DIR]\\clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" werror=true win_sdk=\"[START_DIR]\\win_toolchain/win_sdk\" win_vc=\"[START_DIR]\\win_toolchain/VC\""
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]\\cache\\work\\skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"
@ -47,7 +156,7 @@
"-C",
"[START_DIR]\\cache\\work\\skia\\out\\Build-Win-Clang-x86-Debug\\Debug"
],
"cwd": "[START_DIR]\\skia",
"cwd": "[START_DIR]\\cache\\work\\skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>;RECIPE_REPO[depot_tools]"

48
infra/bots/recipes/compile.py
View File

@ -25,15 +25,53 @@ DEPS = [
def RunSteps(api):
api.vars.setup()
checkout_root = api.path['start_dir']
out_dir = api.vars.cache_dir.join(
'work', 'skia', 'out', api.vars.builder_name, api.vars.configuration)
# Check out code.
bot_update = True
checkout_root = api.checkout.default_checkout_root
checkout_chromium = False
checkout_flutter = False
extra_gclient_env = {}
flutter_android = False
parent_rev = False
if 'NoDEPS' in api.properties['buildername']:
bot_update = False
checkout_root = api.path['start_dir']
if 'CommandBuffer' in api.vars.builder_name:
checkout_chromium = True
if 'Flutter' in api.vars.builder_name:
checkout_root = checkout_root.join('flutter')
checkout_flutter = True
if 'Android' in api.vars.builder_name:
flutter_android = True
if 'ParentRevision' in api.vars.builder_name:
parent_rev = True
if bot_update:
api.checkout.bot_update(
checkout_root=checkout_root,
checkout_chromium=checkout_chromium,
checkout_flutter=checkout_flutter,
extra_gclient_env=extra_gclient_env,
flutter_android=flutter_android,
parent_rev=parent_rev)
else:
api.checkout.git(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
out_dir = checkout_root.join(
'skia', 'out', api.vars.builder_name, api.vars.configuration)
if 'Flutter' in api.vars.builder_name:
out_dir = checkout_root.join('src', 'out', 'android_release')
try:
api.build(checkout_root=checkout_root, out_dir=out_dir)
# TODO(borenet): Move this out of the try/finally.
dst = api.vars.swarming_out_dir
if 'ParentRevision' in api.vars.builder_name:
dst = api.vars.swarming_out_dir.join('ParentRevision')
api.build.copy_build_products(out_dir=out_dir, dst=dst)
if 'SKQP' in api.vars.extra_tokens:
wlist = checkout_root.join(
@ -58,6 +96,10 @@ for p in psutil.process_iter():
TEST_BUILDERS = [
'Build-Debian9-Clang-universal-devrel-Android_SKQP',
'Build-Debian9-Clang-x86_64-Release-NoDEPS',
'Build-Debian9-Clang-x86_64-Release-ParentRevision',
'Build-Debian9-GCC-x86_64-Release-Flutter_Android',
'Build-Mac-Clang-x86_64-Debug-CommandBuffer',
'Build-Win-Clang-x86-Debug',
]

114
infra/bots/recipes/sync_and_compile.py
View File

@ -1,114 +0,0 @@
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Recipe module for Skia Swarming compile.
DEPS = [
'build',
'checkout',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/python',
'recipe_engine/step',
'run',
'vars',
]
def RunSteps(api):
api.vars.setup()
# Check out code.
bot_update = True
checkout_root = api.checkout.default_checkout_root
checkout_chromium = False
checkout_flutter = False
flutter_android = False
parent_rev = False
if 'NoDEPS' in api.properties['buildername']:
bot_update = False
checkout_root = api.path['start_dir']
if 'CommandBuffer' in api.vars.builder_name:
checkout_chromium = True
if 'Flutter' in api.vars.builder_name:
checkout_root = checkout_root.join('flutter')
checkout_flutter = True
if 'Android' in api.vars.builder_name:
flutter_android = True
if 'ParentRevision' in api.vars.builder_name:
parent_rev = True
if bot_update:
api.checkout.bot_update(
checkout_root=checkout_root,
checkout_chromium=checkout_chromium,
checkout_flutter=checkout_flutter,
flutter_android=flutter_android,
parent_rev=parent_rev)
else:
api.checkout.git(checkout_root=checkout_root)
api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir)
out_dir = checkout_root.join(
'skia', 'out', api.vars.builder_name, api.vars.configuration)
if 'Flutter' in api.vars.builder_name:
out_dir = checkout_root.join('src', 'out', 'android_release')
try:
api.build(checkout_root=checkout_root, out_dir=out_dir)
# TODO(borenet): Move this out of the try/finally.
dst = api.vars.swarming_out_dir
if 'ParentRevision' in api.vars.builder_name:
dst = api.vars.swarming_out_dir.join('ParentRevision')
api.build.copy_build_products(out_dir=out_dir, dst=dst)
finally:
if 'Win' in api.vars.builder_cfg.get('os', ''):
api.python.inline(
name='cleanup',
program='''import psutil
for p in psutil.process_iter():
try:
if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):
p.kill()
except psutil._error.AccessDenied:
pass
''',
infra_step=True)
api.run.check_failure()
TEST_BUILDERS = [
'Build-Debian9-Clang-x86_64-Release-ParentRevision',
'Build-Debian9-GCC-x86_64-Release-Flutter_Android',
'Build-Mac-Clang-x86_64-Debug-CommandBuffer',
'Build-Win10-Clang-x86_64-Release-NoDEPS',
]
def GenTests(api):
for builder in TEST_BUILDERS:
test = (
api.test(builder) +
api.properties(buildername=builder,
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.path.exists(
api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt')
)
)
if 'Win' in builder:
test += api.platform('win', 64)
yield test

4108
infra/bots/tasks.json
View File

File diff suppressed because it is too large Load Diff