AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
e81f7b393a
skia2/infra/bots/recipes/bookmaker.py
Eric Boren 72f6668eb7 [recipes] Move a lot of logic out of vars module 
In general, vars should only contain variables which are the same for
all tasks.  Variables specific to compilation belong in the build
module (or compile recipe), and those specific to running tests belong
in the flavor module, or the individual recipe which uses them.

Bug: skia:6473
Change-Id: Ifd55a57118c5801e6f4934a6b5de9d1567415b9a
Reviewed-on: https://skia-review.googlesource.com/128545
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
2018-05-18 11:58:14 +00:00

323 lines
13 KiB
Python
Raw Blame History

# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Recipe which:
# 1) Extracts all fiddles out of markdown files.
# 2) Forces fiddle.skia.org to compile all those fiddles and get output in JSON.
# 3) Scans the output and reports any compiletime/runtime errors.
# 4) Updates markdown in site/user/api/ using the new hashes (if any) from
# fiddle.skia.org.
import json
DEPS = [
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
'core',
'infra',
'run',
'vars',
]
def go_get_fiddlecli(api):
env = api.context.env
env.update(api.infra.go_env)
with api.context(env=env):
api.run.with_retry(
api.step,
'go get fiddlecli',
5, # Update attempts.
cmd=[api.infra.go_exe, 'get', '-u', '-t',
'go.skia.org/infra/fiddle/go/fiddlecli'])
def RunSteps(api):
api.vars.setup()
checkout_root = api.core.default_checkout_root
api.core.checkout_bot_update(checkout_root=checkout_root)
api.infra.go_version()
go_get_fiddlecli(api)
skia_dir = checkout_root.join('skia')
with api.context(cwd=skia_dir, env=api.infra.go_env):
bookmaker_binary = api.vars.skia_out.join('bookmaker')
buildername = api.vars.builder_name
if 'PerCommit' in buildername:
# Check to see if docs matches include/core.
cmd = [bookmaker_binary,
'-a', 'docs/status.json', # File containing status of docs.
'-x', # Check bmh against includes.
]
try:
api.run(api.step, 'Validate docs match include/core/*.h', cmd=cmd)
except api.step.StepFailure as e:
# Display what needs to be fixed.
e.reason += (
'\n\nView the output of the "Validate docs match include/core/*.h" '
'step to see how to get this bot green.'
'\n\nhttps://skia.org/user/api/usingBookmaker details how to build '
'and run the bookmaker utility locally if needed.')
raise e
elif 'Nightly' in buildername:
fiddlecli_binary = api.path.join(api.infra.gopath, 'bin', 'fiddlecli')
fiddlecli_input = api.path.join(api.path['start_dir'], 'fiddle.json')
fiddlecli_output = api.path.join(api.path['start_dir'], 'fiddleout.json')
# Step 1: Extract all fiddles out of markdown files.
cmd = [bookmaker_binary,
'-a', 'docs/status.json', # File containing status of docs.
'-e', fiddlecli_input, # Fiddle cli input.
]
api.run(api.step, 'Extract all fiddles out of md files', cmd=cmd)
# Step 2: Forces fiddle.skia.org to compile all fiddles extracted out of
# markdown files and get output in JSON.
cmd = [fiddlecli_binary,
'--input', fiddlecli_input,
'--output', fiddlecli_output,
'--logtostderr',
'--force',
]
api.run(api.step, 'Force fiddle to compile all examples', cmd=cmd)
# Step 3: Scan the output of fiddlecli for any compiletime/runtime errors.
# Fail the recipe is there are any errors and summarize results at
# the end.
if api.path.exists(fiddlecli_output):
test_data = api.properties.get('fiddleout_test_data', '{}')
content = api.file.read_text('Read fiddleout.json',
fiddlecli_output, test_data=test_data)
out = json.loads(content)
# Output fiddleout.json for easy debugging.
api.run(api.step, 'Output fiddleout.json',
cmd=['cat', fiddlecli_output])
failing_fiddles_to_errors = {}
for fiddle_name in out:
props = out[fiddle_name]
if props['compile_errors'] or props['runtime_error']:
# Construct the error.
error = props['runtime_error']
if props['compile_errors']:
for e in props['compile_errors']:
error += '%s\n' % e['text']
failing_fiddles_to_errors[props['fiddleHash']] = error
if failing_fiddles_to_errors:
# create an eror message and fail the bot!
failure_msg = 'Failed fiddles with their errors:\n\n\n'
counter = 0
for fiddle_hash, error in failing_fiddles_to_errors.iteritems():
counter += 1
failure_msg += '%d. https://fiddle.skia.org/c/%s\n\n' % (
counter, fiddle_hash)
failure_msg += '%s\n\n' % error
raise api.step.StepFailure(failure_msg)
# Step 4: Update docs in site/user/api/ with the output of fiddlecli.
# If there are any new changes then upload and commit the changes.
cmd = ['python',
skia_dir.join('infra', 'bots', 'upload_md.py'),
'--bookmaker_binary', bookmaker_binary,
'--fiddlecli_output', fiddlecli_output]
with api.context(cwd=skia_dir, env=api.infra.go_env):
api.run(api.step, 'Generate and Upload Markdown files', cmd=cmd)
def GenTests(api):
fiddleout_no_errors_test_data = """
{"fiddle1": {"fiddleHash": "abc",
"compile_errors": [],
"runtime_error": ""}}
"""
fiddleout_with_errors_test_data = """
{"fiddle1": {"fiddleHash": "abc",
"compile_errors": [
{
"text": "ninja: Entering directory `out/Release'",
"line": 0,
"col": 0
},
{
"text": "[1/7] ACTION //:skia.h(//gn/toolchain:gcc_like)",
"line": 0,
"col": 0
},
{
"text": "[2/7] stamp obj/skia.h.stamp",
"line": 0,
"col": 0
},
{
"text": "[3/7] compile ../../tools/fiddle/draw.cpp",
"line": 0,
"col": 0
},
{
"text": "FAILED: obj/tools/fiddle/fiddle.draw.o ",
"line": 0,
"col": 0
},
{
"text": "c++ -MMD -MF obj/tools/fiddle/fiddle.draw.o.d -DNDEBUG -DSK_HAS_HEIF_LIBRARY -DSK_HAS_JPEG_LIBRARY -DSK_SUPPORT_PDF -DSK_PDF_USE_SFNTLY -DSK_HAS_PNG_LIBRARY -DSK_CODEC_DECODES_RAW -DSK_HAS_WEBP_LIBRARY -DSK_XML -DSK_GAMMA_APPLY_TO_A8 -DSK_ENABLE_DISCRETE_GPU -DGR_TEST_UTILS=1 -DSK_SAMPLES_FOR_X -DSK_SUPPORT_ATLAS_TEXT=1 -I../../tools/flags -I../../include/private -I../../src/c -I../../src/codec -I../../src/core -I../../src/effects -I../../src/fonts -I../../src/image -I../../src/images -I../../src/lazy -I../../src/opts -I../../src/pathops -I../../src/pdf -I../../src/ports -I../../src/sfnt -I../../src/shaders -I../../src/shaders/gradients -I../../src/sksl -I../../src/utils -I../../src/utils/win -I../../src/xml -I../../third_party/gif -I../../src/gpu -I../../tools/gpu -I../../include/android -I../../include/c -I../../include/codec -I../../include/config -I../../include/core -I../../include/effects -I../../include/encode -I../../include/gpu -I../../include/gpu/gl -I../../include/atlastext -I../../include/pathops -I../../include/ports -I../../include/svg -I../../include/utils -I../../include/utils/mac -I../../include/atlastext -Igen -fstrict-aliasing -fPIC -Werror -Wall -Wextra -Winit-self -Wpointer-arith -Wsign-compare -Wvla -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-unused-parameter -O3 -fdata-sections -ffunction-sections -g -std=c++11 -fno-exceptions -fno-rtti -Wnon-virtual-dtor -Wno-error -c ../../tools/fiddle/draw.cpp -o obj/tools/fiddle/fiddle.draw.o",
"line": 0,
"col": 0
},
{
"text": "../../tools/fiddle/draw.cpp: In function 'void draw(SkCanvas*)':",
"line": 0,
"col": 0
},
{
"text": "draw.cpp:5:12: error: aggregate 'SkMask mask' has incomplete type and cannot be defined",
"line": 5,
"col": 12
},
{
"text": " }",
"line": 0,
"col": 0
},
{
"text": " ^ ",
"line": 0,
"col": 0
},
{
"text": "draw.cpp:6:28: error: incomplete type 'SkMask' used in nested name specifier",
"line": 6,
"col": 28
},
{
"text": " ",
"line": 0,
"col": 0
},
{
"text": " ^ ",
"line": 0,
"col": 0
},
{
"text": "draw.cpp:14:28: error: incomplete type 'SkMask' used in nested name specifier",
"line": 14,
"col": 28
},
{
"text": " uint8_t bytes[] = { 0, 1, 2, 3, 4, 5, 6, 7 };",
"line": 0,
"col": 0
},
{
"text": " ^~~~~~~~~~",
"line": 0,
"col": 0
},
{
"text": "[4/7] compile ../../tools/fiddle/egl_context.cpp",
"line": 0,
"col": 0
},
{
"text": "[5/7] compile ../../tools/fiddle/fiddle_main.cpp",
"line": 0,
"col": 0
},
{
"text": "[6/7] link libskia.a",
"line": 0,
"col": 0
},
{
"text": "ninja: build stopped: subcommand failed.",
"line": 0,
"col": 0
},
{
"text": "",
"line": 0,
"col": 0
}
],
"runtime_error": ""}}
"""
yield (
api.test('percommit_bookmaker') +
api.properties(buildername='Housekeeper-PerCommit-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]')
)
yield (
api.test('percommit_failed_validation') +
api.properties(buildername='Housekeeper-PerCommit-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.step_data('Validate docs match include/core/*.h', retcode=1)
)
yield (
api.test('nightly_bookmaker') +
api.properties(buildername='Housekeeper-Nightly-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
fiddleout_test_data=fiddleout_no_errors_test_data,
swarm_out_dir='[SWARM_OUT_DIR]') +
api.path.exists(api.path['start_dir'].join('fiddleout.json'))
)
yield (
api.test('nightly_failed_fiddles') +
api.properties(buildername='Housekeeper-Nightly-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
fiddleout_test_data=fiddleout_with_errors_test_data,
swarm_out_dir='[SWARM_OUT_DIR]') +
api.path.exists(api.path['start_dir'].join('fiddleout.json'))
)
yield (
api.test('nightly_failed_extract_fiddles') +
api.properties(buildername='Housekeeper-Nightly-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.step_data('Extract all fiddles out of md files', retcode=1)
)
yield (
api.test('nightly_failed_fiddlecli') +
api.properties(buildername='Housekeeper-Nightly-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.step_data('Force fiddle to compile all examples', retcode=1)
)
yield (
api.test('nightly_failed_upload') +
api.properties(buildername='Housekeeper-Nightly-Bookmaker',
repository='https://skia.googlesource.com/skia.git',
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
api.step_data('Generate and Upload Markdown files', retcode=1)
)
Reference in New Issue View Git Blame Copy Permalink