From fec8c611c81d04e6bf65c44dffef7d6d2a74afe4 Mon Sep 17 00:00:00 2001 From: Liviu Rau Date: Fri, 27 Jan 2023 14:16:14 +0100 Subject: [PATCH] Configure PGO builders - add new d8_pgo isolate to be used in profile tasks; - configure swarming task (test) for PGO instrumentation builder - create script for profile only execution (no build) to be used in swarming Led tested #1 with spike recipe #2. Example swarming task #3. #1 https://chromium-swarm.appspot.com/task?id=60089d22e809e410 #2 https://chromium-review.googlesource.com/c/chromium/tools/build/+/4162092/11 #3 https://chromium-swarm.appspot.com/task?id=60089fcbafb0a610 Bug: chromium:1382471 Change-Id: I782b7874c8dd77e821715395dd54dbc8990bf2c6 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4173584 Reviewed-by: Michael Achenbach Commit-Queue: Liviu Rau Cr-Commit-Position: refs/heads/main@{#85521} --- infra/mb/gn_isolate_map.pyl | 4 ++ infra/testing/builders.pyl | 16 ++++++ test/BUILD.gn | 14 +++++ tools/builtins-pgo/profile_only.py | 83 ++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100755 tools/builtins-pgo/profile_only.py diff --git a/infra/mb/gn_isolate_map.pyl b/infra/mb/gn_isolate_map.pyl index 5732d86218..934ff59b2f 100644 --- a/infra/mb/gn_isolate_map.pyl +++ b/infra/mb/gn_isolate_map.pyl @@ -31,6 +31,10 @@ "label": "//test:v8_d8_default", "type": "script", }, + "d8_pgo": { + "label": "//test:d8_pgo", + "type": "script", + }, "generate-bytecode-expectations": { "label": "//test/unittests:generate-bytecode-expectations", "type": "script", diff --git a/infra/testing/builders.pyl b/infra/testing/builders.pyl index d63fff9b4f..5887804e83 100644 --- a/infra/testing/builders.pyl +++ b/infra/testing/builders.pyl @@ -1289,6 +1289,14 @@ {'name': 'v8testing'}, ], }, + 'V8 Linux PGO instrumentation - builder' : { + 'swarming_dimensions': { + 'os': 'Ubuntu-18.04', + }, + 'tests': [ + {'name': 'pgo_instrumentation'} + ], + }, 'V8 Linux64': { 'swarming_dimensions': { 'cpu': 'x86-64-avx2', @@ -1653,6 +1661,14 @@ }, ], }, + 'V8 Linux64 PGO instrumentation - builder' : { + 'swarming_dimensions' : { + 'os': 'Ubuntu-18.04', + }, + 'tests': [ + {'name': 'pgo_instrumentation'} + ], + }, 'V8 Linux64 TSAN': { 'swarming_dimensions' : { 'os': 'Ubuntu-18.04', diff --git a/test/BUILD.gn b/test/BUILD.gn index 0f455b641a..a8e9d42573 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -72,6 +72,20 @@ group("v8_perf") { ] } +group("d8_pgo") { + testonly = true + + data_deps = [ + "..:d8", + "..:v8_python_base", + ] + + data = [ + "../tools/builtins-pgo/profile_only.py", + "../tools/builtins-pgo/get_hints.py", + ] +} + group("v8_bot_default") { testonly = true diff --git a/tools/builtins-pgo/profile_only.py b/tools/builtins-pgo/profile_only.py new file mode 100755 index 0000000000..7d938e983d --- /dev/null +++ b/tools/builtins-pgo/profile_only.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# Copyright 2023 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can +# be found in the LICENSE file. + +from pathlib import Path +import argparse +import subprocess +import sys + + +def main(): + args = parse_arguments() + run_benchmark(args.benchmark_path, args.d8_path, args.output_dir) + run_get_hints(args.output_dir, args.v8_target_cpu) + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description=('Generate builtin PGO profiles. ' + 'The script is designed to run in swarming context where ' + 'the isolate aready contains the instrumented binary.')) + parser.add_argument( + '--v8-target-cpu', + help='target cpu to build the profile for: x64 or arm64') + parser.add_argument( + '--benchmark_path', + default=Path('./JetStream2/cli.js'), + help='path to benchmark runner .js file, usually JetStream2\'s `cli.js`', + type=Path) + parser.add_argument( + '--d8-path', + default=Path('./out/build/d8'), + help=('path to the d8 executable, by default `./out/build/d8` in ' + 'swarming context. Use together with `--profile-only`'), + type=Path) + parser.add_argument('--output-dir', type=Path) + return parser.parse_args() + + +def run_benchmark(benchmark_path, d8_path, output_dir): + root_dir = tools_pgo_dir().parent.parent + benchmark_dir = (root_dir / benchmark_path).parent.absolute() + assert benchmark_dir.exists(), "Could not find benchmark path!" + + benchmark_file = benchmark_path.name + d8_path_abs = (root_dir / d8_path).absolute() + assert d8_path_abs.exists(), "Could not find d8 path!" + + log_path = benchmark_log_path(output_dir) + cmd = [d8_path_abs, f"--turbo-profiling-output={log_path}", benchmark_file] + run(cmd, cwd=benchmark_dir) + assert log_path.exists(), "Could not find benchmark logs path!" + + +def tools_pgo_dir(): + return Path(__file__).parent + + +def benchmark_log_path(output_dir): + return (output_dir / "v8.builtins.pgo").absolute() + + +def run_get_hints(output_dir, v8_target_cpu): + get_hints_path = (tools_pgo_dir() / "get_hints.py").absolute() + assert get_hints_path.exists(), "Could not find get_hints.py script path!" + + profile_path = (output_dir / f"{v8_target_cpu}.profile").absolute() + run([ + sys.executable, '-u', get_hints_path, + benchmark_log_path(output_dir), profile_path + ]) + assert profile_path.exists(), "Could not find profile path!" + + +def run(cmd, **kwargs): + print(f"# CMD: {cmd} {kwargs}") + subprocess.run(cmd, **kwargs, check=True) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main())