AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add ProcDump support to gn_flavor.py.

Browse Source 
Adds procdump_win asset.

Enable ProcDump for some of the jobs failing in skia:7177 as a test
case. If it has no ill effect, we can proceed with enabling it for all
Win bots (and remove "ProcDump" tag).

Bug: skia:7626, skia:7177
Change-Id: I50c67ecfca86fe0c6d91d5f970f81485cc9cfd0a
Reviewed-on: https://skia-review.googlesource.com/113265
Commit-Queue: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
This commit is contained in:
Ben Wagner 2018-03-09 13:42:56 -05:00 committed by Skia Commit-Bot
parent 79ee1b0c4e
commit 299b882f20
13 changed files with 564 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
infra/bots/assets/procdump_win/VERSION Normal file
View File

@ -0,0 +1 @@
0

26
infra/bots/assets/procdump_win/common.py Executable file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Common vars used by scripts in this directory."""
import os
import sys
FILE_DIR = os.path.dirname(os.path.abspath(__file__))
INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
sys.path.insert(0, INFRA_BOTS_DIR)
from assets import assets
ASSET_NAME = os.path.basename(FILE_DIR)
def run(cmd):
"""Run a command, eg. "upload" or "download". """
assets.main([cmd, ASSET_NAME] + sys.argv[1:])

38
infra/bots/assets/procdump_win/create.py Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env python
#
# Copyright 2018 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset."""
import argparse
import common
import subprocess
import utils
# Download URL can be found on this page:
# https://docs.microsoft.com/en-us/sysinternals/downloads/procdump
PROCDUMP_URL = 'https://download.sysinternals.com/files/Procdump.zip'
def create_asset(target_dir):
"""Create the asset."""
with utils.tmp_dir():
subprocess.check_call(["curl", PROCDUMP_URL, "-o", "procdump.zip"])
subprocess.check_call(["unzip", "procdump.zip", "-d", target_dir])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--target_dir', '-t', required=True)
args = parser.parse_args()
create_asset(args.target_dir)
if __name__ == '__main__':
main()

42
infra/bots/assets/procdump_win/create_and_upload.py Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Create the asset and upload it."""
import argparse
import common
import os
import subprocess
import sys
import utils
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gsutil')
args = parser.parse_args()
with utils.tmp_dir():
cwd = os.getcwd()
create_script = os.path.join(common.FILE_DIR, 'create.py')
upload_script = os.path.join(common.FILE_DIR, 'upload.py')
try:
subprocess.check_call(['python', create_script, '-t', cwd])
cmd = ['python', upload_script, '-t', cwd]
if args.gsutil:
cmd.extend(['--gsutil', args.gsutil])
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
# Trap exceptions to avoid printing two stacktraces.
sys.exit(1)
if __name__ == '__main__':
main()

16
infra/bots/assets/procdump_win/download.py Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download the current version of the asset."""
import common
if __name__ == '__main__':
common.run('download')

16
infra/bots/assets/procdump_win/upload.py Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Upload a new version of the asset."""
import common
if __name__ == '__main__':
common.run('upload')

5
infra/bots/gen_tasks.go
View File

@ -130,7 +130,7 @@ func deriveCompileTaskName(jobName string, parts map[string]string) string {
ec := []string{}
if val := parts["extra_config"]; val != "" {
ec = strings.Split(val, "_")
ignore := []string{"Skpbench", "AbandonGpuContext", "PreAbandonGpuContext", "Valgrind", "ReleaseAndAbandonGpuContext", "CCPR", "FSAA", "FAAA", "FDAA", "NativeFonts", "GDI", "NoGPUThreads"}
ignore := []string{"Skpbench", "AbandonGpuContext", "PreAbandonGpuContext", "Valgrind", "ReleaseAndAbandonGpuContext", "CCPR", "FSAA", "FAAA", "FDAA", "NativeFonts", "GDI", "NoGPUThreads", "ProcDump"}
keep := make([]string, 0, len(ec))
for _, part := range ec {
if !util.In(part, ignore) {
@ -1287,6 +1287,9 @@ func process(b *specs.TasksCfgBuilder, name string) {
}
}
}
if strings.Contains(name, "ProcDump") {
pkgs = append(pkgs, b.MustGetCipdPackageFromAsset("procdump_win"))
}
// Test bots.

2
infra/bots/jobs.json
View File

@ -464,11 +464,13 @@
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-ANGLE",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-ANGLE_Goma",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_Goma",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ANGLE",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-ReleaseAndAbandonGpuContext",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan",
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump",
"Test-Win10-Clang-NUC5i7RYH-CPU-AVX2-x86_64-Debug-All-NativeFonts",
"Test-Win10-Clang-NUC5i7RYH-GPU-IntelIris6100-x86_64-Debug-All",
"Test-Win10-Clang-NUC5i7RYH-GPU-IntelIris6100-x86_64-Debug-All-ANGLE",

231
infra/bots/recipe_modules/flavor/examples/full.expected/Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump.json Normal file
View File

@ -0,0 +1,231 @@
[
{
"cmd": [
"python",
"-u",
"[START_DIR]/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release_x64",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/out"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/skia/bin/gn.exe",
"gen",
"[START_DIR]/out/Release_x64",
"--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" is_debug=false skia_enable_vulkan_debug_layers=false skia_vulkan_sdk=\"[START_DIR]/win_vulkan_sdk\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/VC\""
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release_x64",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/out"
},
"name": "gn gen"
},
{
"cmd": [
"ninja.exe",
"-k",
"0",
"-C",
"[START_DIR]/out/Release_x64"
],
"cwd": "[START_DIR]/skia",
"env": {
"BUILDTYPE": "Release_x64",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/out"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"rmtree",
"results_dir"
],
"infra_step": true,
"name": "rmtree results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"results_dir"
],
"infra_step": true,
"name": "makedirs results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"rmtree",
"device_results_dir"
],
"infra_step": true,
"name": "rmtree device_results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"device_results_dir"
],
"infra_step": true,
"name": "makedirs device_results_dir"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/skia/infra/bots/assets/skp/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get downloaded SKP VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SKP_VERSION"
],
"infra_step": true,
"name": "write SKP_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/skia/infra/bots/assets/skimage/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get downloaded skimage VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SK_IMAGE_VERSION"
],
"infra_step": true,
"name": "write SK_IMAGE_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/skia/infra/bots/assets/svg/VERSION",
"/path/to/tmp/"
],
"infra_step": true,
"name": "Get downloaded SVG VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"42",
"[START_DIR]/tmp/SVG_VERSION"
],
"infra_step": true,
"name": "write SVG_VERSION"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
"[CUSTOM_[SWARM_OUT_DIR]]/dumps"
],
"infra_step": true,
"name": "makedirs dumps"
},
{
"cmd": [
"[START_DIR]/procdump_win/procdump64.exe",
"-accepteula",
"-mp",
"-e",
"1",
"-x",
"[CUSTOM_[SWARM_OUT_DIR]]/dumps",
"[START_DIR]/out/Release_x64/dm",
"--some-flag"
],
"env": {
"BUILDTYPE": "Release_x64",
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
"SKIA_OUT": "[START_DIR]/out"
},
"name": "dm"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]

1
infra/bots/recipe_modules/flavor/examples/full.py
View File

@ -112,6 +112,7 @@ TEST_BUILDERS = [
'Test-Ubuntu16-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Debug-All-Vulkan',
('Test-Ubuntu17-GCC-Golo-GPU-QuadroP400-x86_64-Release-All'
'-Valgrind_AbandonGpuContext_SK_CPU_LIMIT_SSE41'),
'Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump',
'Test-Win10-MSVC-ShuttleA-GPU-GTX660-x86_64-Debug-All',
'Test-iOS-Clang-iPadPro-GPU-GT7800-arm64-Debug-All',
'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SafeStack',

13
infra/bots/recipe_modules/flavor/gn_flavor.py
View File

@ -314,6 +314,19 @@ with open(sys.argv[1], 'w') as f:
ld_library_path.append(clang_linux + '/lib')
elif self.m.vars.is_linux:
cmd = ['catchsegv'] + cmd
elif 'ProcDump' in extra_tokens:
self.m.file.ensure_directory('makedirs dumps', self.m.vars.dumps_dir)
procdump = str(self.m.vars.slave_dir.join('procdump_win',
'procdump64.exe'))
# Full docs for ProcDump here:
# https://docs.microsoft.com/en-us/sysinternals/downloads/procdump
# -accepteula automatically accepts the license agreement
# -mp saves a packed minidump to save space
# -e 1 tells procdump to dump once
# -x <dump dir> <exe> <args> launches exe and writes dumps to the
# specified dir
cmd = [procdump, '-accepteula', '-mp', '-e', '1',
'-x', self.m.vars.dumps_dir] + cmd
if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens:
env[ 'ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'

2
infra/bots/recipe_modules/vars/api.py
View File

@ -148,6 +148,8 @@ class SkiaVarsApi(recipe_api.RecipeApi):
self.swarming_out_dir, 'dm')
self.perf_data_dir = self.m.path.join(self.swarming_out_dir,
'perfdata', self.builder_name, 'data')
self.dumps_dir = self.m.path.join(
self.swarming_out_dir, 'dumps')
self._swarming_bot_id = None
self._swarming_task_id = None

172
infra/bots/tasks.json
View File

@ -2812,6 +2812,12 @@
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_Goma"
]
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump": {
"priority": 0.8,
"tasks": [
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump"
]
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All": {
"priority": 0.8,
"tasks": [
@ -2836,6 +2842,12 @@
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan"
]
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump": {
"priority": 0.8,
"tasks": [
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump"
]
},
"Test-Win10-Clang-NUC5i7RYH-CPU-AVX2-x86_64-Debug-All-NativeFonts": {
"priority": 0.8,
"tasks": [
@ -22090,6 +22102,59 @@
"max_attempts": 1,
"priority": 0.8
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump": {
"cipd_packages": [
{
"name": "skia/bots/skimage",
"path": "skimage",
"version": "version:34"
},
{
"name": "skia/bots/skp",
"path": "skp",
"version": "version:115"
},
{
"name": "skia/bots/svg",
"path": "svg",
"version": "version:9"
},
{
"name": "skia/bots/procdump_win",
"path": "procdump_win",
"version": "version:0"
}
],
"dependencies": [
"Build-Win-Clang-x86_64-Debug-Vulkan",
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"gpu:10de:1cb3-22.21.13.8205",
"os:Windows-10-15063",
"pool:Skia"
],
"execution_timeout_ns": 14400000000000,
"expiration_ns": 72000000000000,
"extra_args": [
"--workdir",
"../../..",
"test",
"repository=<(REPO)",
"buildbucket_build_id=<(BUILDBUCKET_BUILD_ID)",
"buildername=Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)",
"patch_repo=<(PATCH_REPO)",
"patch_storage=<(PATCH_STORAGE)",
"patch_issue=<(ISSUE)",
"patch_set=<(PATCHSET)"
],
"io_timeout_ns": 2400000000000,
"isolate": "test_skia_bundled_win.isolate",
"max_attempts": 1,
"priority": 0.8
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All": {
"cipd_packages": [
{
@ -22282,6 +22347,59 @@
"max_attempts": 1,
"priority": 0.8
},
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump": {
"cipd_packages": [
{
"name": "skia/bots/skimage",
"path": "skimage",
"version": "version:34"
},
{
"name": "skia/bots/skp",
"path": "skp",
"version": "version:115"
},
{
"name": "skia/bots/svg",
"path": "svg",
"version": "version:9"
},
{
"name": "skia/bots/procdump_win",
"path": "procdump_win",
"version": "version:0"
}
],
"dependencies": [
"Build-Win-Clang-x86_64-Release-Vulkan",
"Housekeeper-PerCommit-BundleRecipes"
],
"dimensions": [
"gpu:10de:1cb3-22.21.13.8205",
"os:Windows-10-15063",
"pool:Skia"
],
"execution_timeout_ns": 14400000000000,
"expiration_ns": 72000000000000,
"extra_args": [
"--workdir",
"../../..",
"test",
"repository=<(REPO)",
"buildbucket_build_id=<(BUILDBUCKET_BUILD_ID)",
"buildername=Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)",
"patch_repo=<(PATCH_REPO)",
"patch_storage=<(PATCH_STORAGE)",
"patch_issue=<(ISSUE)",
"patch_set=<(PATCHSET)"
],
"io_timeout_ns": 2400000000000,
"isolate": "test_skia_bundled_win.isolate",
"max_attempts": 1,
"priority": 0.8
},
"Test-Win10-Clang-NUC5i7RYH-CPU-AVX2-x86_64-Debug-All-NativeFonts": {
"cipd_packages": [
{
@ -32269,6 +32387,33 @@
"isolate": "upload_dm_results.isolate",
"priority": 0.8
},
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump": {
"dependencies": [
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"extra_args": [
"--workdir",
"../../..",
"upload_dm_results",
"repository=<(REPO)",
"buildername=Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Debug-All-Vulkan_ProcDump",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)",
"patch_repo=<(PATCH_REPO)",
"patch_storage=<(PATCH_STORAGE)",
"patch_issue=<(ISSUE)",
"patch_set=<(PATCHSET)",
"gs_bucket=skia-infra-gm"
],
"isolate": "upload_dm_results.isolate",
"priority": 0.8
},
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All": {
"dependencies": [
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All"
@ -32350,6 +32495,33 @@
"isolate": "upload_dm_results.isolate",
"priority": 0.8
},
"Upload-Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump": {
"dependencies": [
"Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"os:Debian-9.2",
"pool:Skia"
],
"extra_args": [
"--workdir",
"../../..",
"upload_dm_results",
"repository=<(REPO)",
"buildername=Test-Win10-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan_ProcDump",
"swarm_out_dir=${ISOLATED_OUTDIR}",
"revision=<(REVISION)",
"patch_repo=<(PATCH_REPO)",
"patch_storage=<(PATCH_STORAGE)",
"patch_issue=<(ISSUE)",
"patch_set=<(PATCHSET)",
"gs_bucket=skia-infra-gm"
],
"isolate": "upload_dm_results.isolate",
"priority": 0.8
},
"Upload-Test-Win10-Clang-NUC5i7RYH-CPU-AVX2-x86_64-Debug-All-NativeFonts": {
"dependencies": [
"Test-Win10-Clang-NUC5i7RYH-CPU-AVX2-x86_64-Debug-All-NativeFonts"