AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Dockerize Flutter Android job.

Browse Source 
Use a Debian9 container because the build scripts for Flutter Android require jdk8, which isn't available on Debian10.
-Shell script executes the GN and Ninja commands
-flutter.py executes the script in Debian9 container
--the Flutter build is particular about directory structure and uses various relative paths, so the entire swarming directory is mounted.
-Followup CL will update remove the old Flutter Android job and update the BuildStats job.

Bug: skia:9438
Change-Id: I60c62425f21214b4ae6a810a787418a8975d204e
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/282616
Commit-Queue: Weston Tracey <westont@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
This commit is contained in:
Weston Tracey 2020-04-16 11:55:38 -04:00 committed by Skia Commit-Bot
parent efebaa2a11
commit 5fae9adb0f
6 changed files with 324 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infra/bots/jobs.json
View File

@ -1,4 +1,5 @@
[
"Build-Debian9-Clang-arm-Release-Flutter_Android_Docker",
"Build-Debian10-GCC-loongson3a-Debug-Docker",
"Build-Debian10-GCC-loongson3a-Release-Docker",
"Build-Debian10-GCC-mips64el-Debug-Docker",

168
infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker.json Normal file
View File

@ -0,0 +1,168 @@
[
{
"cmd": [],
"name": "Docker setup"
},
{
"cmd": [
"python",
"-u",
"import os\nprint '%d:%d' % (os.getuid(), os.getgid())\n"
],
"name": "Docker setup.Get uid and gid",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@print '%d:%d' % (os.getuid(), os.getgid())@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"vpython",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release"
],
"infra_step": true,
"name": "Docker setup.mkdirs out_dir",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"chmod",
"777",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release"
],
"infra_step": true,
"name": "Docker setup.chmod 777 [START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"chmod",
"755",
"[START_DIR]"
],
"infra_step": true,
"name": "Docker setup.chmod 755 [START_DIR]",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"chmod",
"0755",
"RECIPE_MODULE[skia::build]/resources/docker-flutter-compile.sh"
],
"infra_step": true,
"name": "Docker setup.chmod 0755 RECIPE_MODULE[skia::build]/resources/docker-flutter-compile.sh",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"docker",
"run",
"--shm-size=2gb",
"--rm",
"--user",
"13:17",
"--mount",
"type=bind,source=[START_DIR],target=[START_DIR]",
"--mount",
"type=bind,source=[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release,target=[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release",
"gcr.io/skia-public/debian9@sha256:f48edc56b621a44ee2f985555f9b09c2802da15d56bb4e43666faabbec80d569",
"RECIPE_MODULE[skia::build]/resources/docker-flutter-compile.sh",
"[START_DIR]",
"[START_DIR]/cache/work/src",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release",
"--runtime-mode=release",
"--android"
],
"env": {
"CHROME_HEADLESS": "1",
"DOCKER_CONFIG": "/home/chrome-bot/.docker",
"PATH": "<PATH>:RECIPE_REPO[depot_tools]"
},
"name": "Run build script in Docker"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products = ['*.so', 'lib/*.so']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release",
"[START_DIR]/[SWARM_OUT_DIR]/out/Release"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products = ['*.so', 'lib/*.so']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\n\nif not os.path.isdir(os.path.dirname(dst)):\n os.makedirs(os.path.dirname(dst))\n\nshutil.copyfile(src, dst)\n",
"[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-arm-Release-Flutter_Android_Docker/Release/lib.stripped/libflutter.so",
"[START_DIR]/[SWARM_OUT_DIR]/out/Release/libflutter_stripped.so"
],
"infra_step": true,
"name": "copy stripped library",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(os.path.dirname(dst)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst))@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@shutil.copyfile(src, dst)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"name": "$result"
}
]

1
infra/bots/recipe_modules/build/examples/full.py
View File

@ -26,6 +26,7 @@ def RunSteps(api):
TEST_BUILDERS = [
'Build-Debian9-Clang-arm-Release-Flutter_Android_Docker',
'Build-Debian10-GCC-loongson3a-Release-Docker',
'Build-Debian10-GCC-x86-Debug-Docker',
'Build-Debian10-GCC-x86_64-Debug-Docker',

31
infra/bots/recipe_modules/build/flutter.py
View File

@ -4,19 +4,36 @@
from . import util
IMAGES = {
# Used to build Flutter_Android in Debian9, since the underlying build
# scripts require jdk8.
'Debian9': (
'gcr.io/skia-public/debian9@sha256:'
'f48edc56b621a44ee2f985555f9b09c2802da15d56bb4e43666faabbec80d569'),
}
def compile_fn(api, checkout_root, out_dir):
flutter_dir = checkout_root.join('src')
configuration = api.vars.builder_cfg.get('configuration').lower()
os_name = api.vars.builder_cfg.get('os', '')
extra_tokens = api.vars.extra_tokens
builder_name = api.vars.builder_name
# Setup GN args.
gn_args = ['--runtime-mode=%s' % configuration,]
if 'Android' in extra_tokens:
gn_args.append('--android')
if os_name == 'Debian9' and 'Docker' in builder_name:
script = api.build.resource('docker-flutter-compile.sh')
image_hash = IMAGES[os_name]
api.docker.run('Run build script in Docker', image_hash,
api.path['start_dir'], out_dir, script,
[api.path['start_dir'], flutter_dir, out_dir] + gn_args,
match_directory_structure=True)
return
with api.context(cwd=flutter_dir):
# Setup GN args.
gn_args = [
'--runtime-mode=%s' % configuration,
]
if 'Android' in extra_tokens:
gn_args.append('--android')
# Delete out_dir so that we start from a clean slate. See skbug/6310.
api.run.rmtree(out_dir)

23
infra/bots/recipe_modules/build/resources/docker-flutter-compile.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
# Copyright 2020 Google LLC
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Call with <swarming dir> <top level of flutter checkout> <out_dir> [GN arguments...]
set -e
set -x
export PATH="$1/recipe_bundle/depot_tools:${PATH}"
# e.g. /mnt/pd0/s/w/ir/cache/work/flutter/src
source_dir=$2
# e.g. /mnt/pd0/s/w/ir/cache/work/flutter/src/out/android_release
out_dir=$3
shift 3
cd $source_dir
flutter/tools/gn "$@"
# This is why we have to mount the entire swarming directory, GN sets a command
# to do a version check via relative path (../../flutter/third_party/gn/gn)
ninja -v -C $out_dir -j100

107
infra/bots/tasks.json
View File

@ -394,6 +394,12 @@
"Build-Debian10-GCC-x86_64-Release-Shared_Docker"
]
},
"Build-Debian9-Clang-arm-Release-Flutter_Android_Docker": {
"tasks": [
"Build-Debian9-Clang-arm-Release-Flutter_Android_Docker"
],
"trigger": "master"
},
"Build-Debian9-Clang-x86_64-Debug-Chromebook_GLES_Docker": {
"tasks": [
"Build-Debian9-Clang-x86_64-Debug-Chromebook_GLES_Docker"
@ -9244,6 +9250,107 @@
],
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-arm-Release-Flutter_Android_Docker": {
"caches": [
{
"name": "docker",
"path": "cache/docker"
},
{
"name": "git",
"path": "cache/git"
},
{
"name": "git_cache",
"path": "cache/git_cache"
},
{
"name": "vpython",
"path": "cache/vpython"
},
{
"name": "work",
"path": "cache/work"
}
],
"cipd_packages": [
{
"name": "infra/3pp/tools/git/linux-amd64",
"path": "cipd_bin_packages",
"version": "version:2.26.1.chromium.6"
},
{
"name": "infra/tools/git/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:dc79de8d1d36f90511b5184e3ff704b71519c418"
},
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:dc79de8d1d36f90511b5184e3ff704b71519c418"
},
{
"name": "infra/tools/luci/git-credential-luci/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:dc79de8d1d36f90511b5184e3ff704b71519c418"
},
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:dc79de8d1d36f90511b5184e3ff704b71519c418"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:dc79de8d1d36f90511b5184e3ff704b71519c418"
},
{
"name": "skia/bots/android_ndk_linux",
"path": "android_ndk_linux",
"version": "version:15"
}
],
"command": [
"cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}",
"-u",
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"sync_and_compile",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Build-Debian9-Clang-arm-Release-Flutter_Android_Docker\",\"patch_issue\":\"<(ISSUE_INT)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET_INT)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"build\",\"task_id\":\"<(TASK_ID)\"}",
"skia"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-standard-16",
"os:Debian-10.3",
"pool:Skia",
"docker_installed:true"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"cache/vpython"
]
},
"execution_timeout_ns": 3600000000000,
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"max_attempts": 2,
"outputs": [
"build"
],
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-x86_64-Debug-Chromebook_GLES_Docker": {
"caches": [
{