AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[gcmole] Run gcmole on bots with matching architecture

Browse Source 
This does:
- Move gcmole runs to bots with matching arch.
- Update mb_configs to enable gcmole on the bots that run it.
- Add a GN target that extracts some compiler flags from a
ninja file of one of V8's other targets.
- Use the extracted flags in the gcmole script and remove other
hard-coded arch-specific flags.

This is done for DEFINES and includes for now. Other compiler flags
are still a TODO.

Bug: v8:9287
Change-Id: Icba9ce59e0bfffd138d9207b1c2ad64d42bf6a91
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4055629
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Reviewed-by: Alexander Schulze <alexschulze@chromium.org>
Cr-Commit-Position: refs/heads/main@{#85341}
This commit is contained in:
Michael Achenbach 2023-01-17 11:00:41 +01:00 committed by V8 LUCI CQ
parent 6350ef168c
commit 2a20b228ef
8 changed files with 178 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BUILD.gn
View File

@ -4503,7 +4503,11 @@ group("v8_tracing") {
}
v8_source_set("v8_base_without_compiler") {
visibility = [ ":*" ] # Only targets in this file can depend on this.
# Only targets in this file and gcmole can depend on this.
visibility = [
":*",
"tools/gcmole/:*",
]
# Split static libraries on windows into two.
split_count = 2

30
infra/mb/mb_config.pyl
View File

@ -63,7 +63,7 @@
'V8 Linux - noi18n - debug builder': 'debug_x86_no_i18n',
'V8 Linux - verify csa - builder': 'release_x86_verify_csa',
# Linux64.
'V8 Linux64 - builder': 'release_x64',
'V8 Linux64 - builder': 'release_x64_gcmole',
'V8 Linux64 - builder (goma cache silo)': 'release_x64',
'V8 Linux64 - builder (reclient)': 'release_x64_reclient',
'V8 Linux64 - builder (reclient compare)': 'release_x64_reclient',
@ -187,7 +187,7 @@
'V8 Arm - builder': 'release_arm',
'V8 Arm - debug builder': 'debug_arm',
'V8 Android Arm - builder': 'release_android_arm',
'V8 Linux - arm - sim - builder': 'release_simulate_arm',
'V8 Linux - arm - sim - builder': 'release_simulate_arm_gcmole',
'V8 Linux - arm - sim - debug builder': 'debug_simulate_arm',
'V8 Linux - arm - sim - lite - builder': 'release_simulate_arm_lite',
'V8 Linux - arm - sim - lite - debug builder': 'debug_simulate_arm_lite',
@ -195,7 +195,7 @@
'V8 Android Arm64 - builder': 'release_android_arm64',
'V8 Android Arm64 - debug builder': 'debug_android_arm64',
'V8 Arm64 - builder': 'release_arm64_hard_float',
'V8 Linux - arm64 - sim - builder': 'release_simulate_arm64',
'V8 Linux - arm64 - sim - builder': 'release_simulate_arm64_gcmole',
'V8 Linux - arm64 - sim - debug builder': 'debug_simulate_arm64',
'V8 Linux - arm64 - sim - gc stress - builder': 'debug_simulate_arm64',
# Mips.
@ -247,7 +247,7 @@
'v8_linux64_nodcheck_compile_rel': 'release_x64',
'v8_linux64_perfetto_compile_dbg': 'debug_x64_perfetto',
'v8_linux64_no_pointer_compression_compile_rel': 'release_x64_no_pointer_compression',
'v8_linux64_compile_rel': 'release_x64_test_features_trybot',
'v8_linux64_compile_rel': 'release_x64_test_features_gcmole_trybot',
'v8_linux64_no_sandbox_compile_rel': 'release_x64_no_sandbox',
'v8_linux64_predictable_compile_rel': 'release_x64_predictable',
'v8_linux64_shared_compile_rel': 'release_x64_shared_verify_heap',
@ -290,13 +290,13 @@
'v8_mac64_compile_dbg': 'debug_x64',
'v8_mac64_noopt_compile_dbg': 'full_debug_x64',
'v8_mac64_asan_compile_rel': 'release_x64_asan_no_lsan',
'v8_linux_arm_compile_rel': 'release_simulate_arm_trybot',
'v8_linux_arm_compile_rel': 'release_simulate_arm_gcmole_trybot',
'v8_linux_arm_lite_compile_dbg': 'debug_simulate_arm_lite',
'v8_linux_arm_lite_compile_rel': 'release_simulate_arm_lite_trybot',
'v8_linux_arm_compile_dbg': 'debug_simulate_arm',
'v8_linux_arm_armv8a_rel': 'release_simulate_arm_trybot',
'v8_linux_arm_armv8a_dbg': 'debug_simulate_arm',
'v8_linux_arm64_compile_rel': 'release_simulate_arm64_trybot',
'v8_linux_arm64_compile_rel': 'release_simulate_arm64_gcmole_trybot',
'v8_linux_arm64_cfi_compile_rel' : 'release_simulate_arm64_cfi',
'v8_linux_arm64_compile_dbg': 'debug_simulate_arm64',
'v8_linux_arm64_gc_stress_compile_dbg': 'debug_simulate_arm64',
@ -403,18 +403,24 @@
'debug_bot', 'simulate_arm64', 'asan', 'lsan'],
# Release configs for simulators.
'release_simulate_arm': [
'release_bot', 'simulate_arm'],
'release_simulate_arm_gcmole': [
'release_bot', 'simulate_arm', 'gcmole'],
'release_simulate_arm_lite': [
'release_bot', 'simulate_arm', 'v8_enable_lite_mode'],
'release_simulate_arm_trybot': [
'release_trybot', 'simulate_arm'],
'release_simulate_arm_gcmole_trybot': [
'release_trybot', 'simulate_arm', 'gcmole'],
'release_simulate_arm_lite_trybot': [
'release_trybot', 'simulate_arm', 'v8_enable_lite_mode'],
'release_simulate_arm_trybot': [
'release_trybot', 'simulate_arm'],
'release_simulate_arm64': [
'release_bot', 'simulate_arm64'],
'release_simulate_arm64_cfi': [
'release_bot', 'simulate_arm64', 'v8_control_flow_integrity'],
'release_simulate_arm64_gcmole': [
'release_bot', 'simulate_arm64', 'gcmole'],
'release_simulate_arm64_gcmole_trybot': [
'release_trybot', 'simulate_arm64', 'gcmole'],
'release_simulate_arm64_no_pointer_compression': [
'release_bot', 'simulate_arm64', 'no_sandbox', 'dcheck_always_on',
'v8_enable_slow_dchecks', 'v8_disable_pointer_compression'],
@ -511,6 +517,8 @@
'release_x64_fuzzilli': [
'release_bot', 'x64', 'dcheck_always_on', 'v8_enable_slow_dchecks',
'v8_verify_heap', 'v8_verify_csa', 'fuzzilli'],
'release_x64_gcmole': [
'release_bot', 'x64', 'gcmole'],
'release_x64_msvc': [
'release_bot_no_goma', 'x64', 'minimal_symbols', 'msvc'],
'release_x64_correctness_fuzzer' : [
@ -543,6 +551,8 @@
'release_bot', 'x64', 'no_sandbox'],
'release_x64_trybot': [
'release_trybot', 'x64'],
'release_x64_test_features_gcmole_trybot': [
'release_trybot', 'x64', 'v8_enable_test_features', 'gcmole'],
'release_x64_test_features_trybot': [
'release_trybot', 'x64', 'v8_enable_test_features'],
'release_x64_tsan': [

63
infra/testing/builders.pyl
View File

@ -181,21 +181,6 @@
'shards': 4,
},
{'name': 'gcmole_v3', 'variant': 'ia32', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'x64', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'arm', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'arm64', 'shards': 4},
{
'name': 'gcmole_v2',
'variant': 'x64',
'suffix': 'test single host',
'test_args': ['--test-run'],
},
{
'name': 'gcmole_v3',
'variant': 'x64',
'suffix': 'test multi host',
'test_args': ['--test-run'],
},
],
},
'v8_linux_optional_rel': {
@ -345,6 +330,7 @@
{'name': 'test262', 'variant': 'default', 'shards': 2},
{'name': 'v8testing', 'shards': 10},
{'name': 'v8testing', 'variant': 'extra', 'shards': 10},
{'name': 'gcmole_v3', 'variant': 'arm', 'shards': 4},
],
},
##############################################################################
@ -627,6 +613,20 @@
{'name': 'v8testing', 'variant': 'stress_instruction_scheduling'},
# Maglev -- move to extra once more architectures are supported.
{'name': 'mjsunit', 'variant': 'maglev'},
# GCMole.
{'name': 'gcmole_v3', 'variant': 'x64', 'shards': 4},
{
'name': 'gcmole_v2',
'variant': 'x64',
'suffix': 'test single host',
'test_args': ['--test-run'],
},
{
'name': 'gcmole_v3',
'variant': 'x64',
'suffix': 'test multi host',
'test_args': ['--test-run'],
},
],
},
'v8_linux64_predictable_rel': {
@ -749,6 +749,7 @@
{'name': 'test262', 'variant': 'default', 'shards': 4},
{'name': 'v8testing', 'shards': 14},
{'name': 'v8testing', 'variant': 'extra', 'shards': 14},
{'name': 'gcmole_v3', 'variant': 'arm64', 'shards': 4},
],
},
'v8_linux_arm64_cfi_rel': {
@ -1099,21 +1100,6 @@
'shards': 2
},
{'name': 'gcmole_v3', 'variant': 'ia32', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'x64', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'arm', 'shards': 4},
{'name': 'gcmole_v3', 'variant': 'arm64', 'shards': 4},
{
'name': 'gcmole_v2',
'variant': 'x64',
'suffix': 'test single host',
'test_args': ['--test-run'],
},
{
'name': 'gcmole_v3',
'variant': 'x64',
'suffix': 'test multi host',
'test_args': ['--test-run'],
},
],
},
'V8 Linux - arm64 - sim - CFI': {
@ -1349,6 +1335,20 @@
'suffix': 'noavx',
'test_args': ['--extra-flags', '--noenable-avx']
},
# GCMole.
{'name': 'gcmole_v3', 'variant': 'x64', 'shards': 4},
{
'name': 'gcmole_v2',
'variant': 'x64',
'suffix': 'test single host',
'test_args': ['--test-run'],
},
{
'name': 'gcmole_v3',
'variant': 'x64',
'suffix': 'test multi host',
'test_args': ['--test-run'],
},
],
},
'V8 Linux64 - cfi': {
@ -2068,6 +2068,8 @@
'test_args': ['--novfp3'],
'shards': 6
},
# GCMole.
{'name': 'gcmole_v3', 'variant': 'arm', 'shards': 4},
],
},
'V8 Linux - arm - sim - debug': {
@ -2149,6 +2151,7 @@
{'name': 'test262', 'variant': 'default', 'shards': 2},
{'name': 'v8testing', 'shards': 3},
{'name': 'v8testing', 'variant': 'extra', 'shards': 2},
{'name': 'gcmole_v3', 'variant': 'arm64', 'shards': 4},
],
},
'V8 Linux - arm64 - sim - debug': {

13
tools/gcmole/BUILD.gn
View File

@ -8,6 +8,7 @@ if (v8_gcmole) {
group("v8_gcmole_files") {
testonly = true
data_deps = [
":v8_gcmole_args",
"../../:v8_dump_build_config",
"../../:v8_generated_cc_files",
]
@ -29,6 +30,7 @@ if (v8_gcmole) {
"../../test/cctest/",
"../../test/common/",
"../../testing/gtest/include/gtest/gtest_prod.h",
"../../third_party/glibc/",
"../../third_party/googletest/src/googletest/include/gtest/gtest_prod.h",
"../../third_party/icu/source/common/",
"../../third_party/icu/source/i18n/",
@ -44,4 +46,15 @@ if (v8_gcmole) {
"gcmole-tools/",
]
}
# Only prepare gcmole args if gcmole is activated by a gn arg.
action("v8_gcmole_args") {
script = "gcmole_args.py"
outputs = [ "$root_out_dir/v8_gcmole.args" ]
# We use an arbitrary v8 target as proxy for calculating globally used
# gcmole plugin arguments. Should be a target that's early in the chain
# and that enables icu.
deps = [ "../../:v8_base_without_compiler" ]
}
}

34
tools/gcmole/gcmole.py
View File

@ -25,17 +25,8 @@ import threading
import queue
ArchCfg = collections.namedtuple(
"ArchCfg", ["name", "cpu", "triple", "arch_define", "arch_options"])
# TODO(cbruni): use gn desc by default for platform-specific settings
OPTIONS_64BIT = [
"-DV8_COMPRESS_POINTERS",
"-DV8_COMPRESS_POINTERS_IN_SHARED_CAGE",
"-DV8_EXTERNAL_CODE_SPACE",
"-DV8_SHORT_BUILTIN_CALLS",
"-DV8_SHARED_RO_HEAP",
]
ArchCfg = collections.namedtuple("ArchCfg",
["name", "cpu", "triple", "arch_options"])
ARCHITECTURES = {
"ia32":
@ -43,7 +34,6 @@ ARCHITECTURES = {
name="ia32",
cpu="x86",
triple="i586-unknown-linux",
arch_define="V8_TARGET_ARCH_IA32",
arch_options=["-m32"],
),
"arm":
@ -51,24 +41,19 @@ ARCHITECTURES = {
name="arm",
cpu="arm",
triple="i586-unknown-linux",
arch_define="V8_TARGET_ARCH_ARM",
arch_options=["-m32"],
),
# TODO(cbruni): Use detailed settings:
# arch_options = OPTIONS_64BIT + [ "-DV8_WIN64_UNWINDING_INFO" ]
"x64":
ArchCfg(
name="x64",
cpu="x64",
triple="x86_64-unknown-linux",
arch_define="V8_TARGET_ARCH_X64",
arch_options=[]),
"arm64":
ArchCfg(
name="arm64",
cpu="arm64",
triple="x86_64-unknown-linux",
arch_define="V8_TARGET_ARCH_ARM64",
arch_options=[],
),
}
@ -90,6 +75,9 @@ def fatal(format):
def make_clang_command_line(plugin, plugin_args, options):
with open(options.v8_build_dir / 'v8_gcmole.args') as f:
generated_args = f.read().strip().split()
arch_cfg = ARCHITECTURES[options.v8_target_cpu]
prefixed_plugin_args = []
if plugin_args:
@ -101,7 +89,6 @@ def make_clang_command_line(plugin, plugin_args, options):
arg,
]
log("Using generated files in {}", options.v8_build_dir / 'gen')
icu_src_dir = options.v8_root_dir / 'third_party/icu/source'
return ([
options.clang_bin_dir / "clang++",
"-std=c++17",
@ -121,17 +108,8 @@ def make_clang_command_line(plugin, plugin_args, options):
arch_cfg.triple,
"-fno-exceptions",
"-Wno-everything",
"-D",
arch_cfg.arch_define,
"-DV8_ENABLE_WEBASSEMBLY",
"-DV8_GC_MOLE",
"-DV8_INTL_SUPPORT",
"-I{}".format(options.v8_root_dir),
"-I{}".format(options.v8_root_dir / 'include'),
"-I{}".format(options.v8_build_dir / 'gen'),
"-I{}".format(icu_src_dir / 'common'),
"-I{}".format(icu_src_dir / 'i18n'),
] + arch_cfg.arch_options)
] + generated_args + arch_cfg.arch_options)
def invoke_clang_plugin_for_file(filename, cmd_line, verbose):

59
tools/gcmole/gcmole_args.py Normal file
View File

@ -0,0 +1,59 @@
# Copyright 2023 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Calculate arguments for the gcmole plugin based on flags passed to the
compiler for a typical target in V8.
"""
from pathlib import Path
import os
import re
import sys
DEFINES_RE = re.compile(r'^defines = (.*)$', re.M)
INCLUDES_RE = re.compile(r'^include_dirs = (.*)$', re.M)
BASE_DIR = Path(__file__).resolve().parents[2].absolute()
# This script is always called relative to the build directory root
# by ninja.
BUILD_DIR_ABS = Path.cwd()
BUILD_DIR_REL = BUILD_DIR_ABS.relative_to(BASE_DIR)
def search_flags(regexp, ninja_config):
match = regexp.search(ninja_config)
assert match
result = match.group(1)
assert result
return result
def main():
assert BUILD_DIR_ABS.exists()
ninja_file = BUILD_DIR_ABS / 'obj' / 'v8_base_without_compiler.ninja'
assert ninja_file.exists()
with ninja_file.open() as f:
ninja_config = f.read()
defines = search_flags(DEFINES_RE, ninja_config)
includes = search_flags(INCLUDES_RE, ninja_config)
# Include flags are relative to the build root. Make them relative to the
# base directory for gcmole.
# E.g. BUILD_DIR_REL = out/build and -I../../include gives -Iinclude.
include_flags = []
for flag in includes.strip().split():
prefix, suffix = flag[:2], flag[2:]
assert prefix == '-I'
include_flags.append(prefix + os.path.normpath(BUILD_DIR_REL / suffix))
with open('v8_gcmole.args', 'w') as f:
f.write(' '.join([defines] + include_flags))
if __name__ == '__main__':
main()

43
tools/gcmole/gcmole_test.py
View File

@ -8,14 +8,17 @@ from pathlib import Path
import collections
import os
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
import unittest
import gcmole
TESTDATA_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'testdata', 'v8')
GCMOLE_PATH = Path(__file__).parent.absolute()
TESTDATA_PATH = GCMOLE_PATH / 'testdata' / 'v8'
Options = collections.namedtuple(
'Options', ['v8_root_dir', 'v8_target_cpu', 'shard_count', 'shard_index',
@ -23,19 +26,19 @@ Options = collections.namedtuple(
def abs_test_file(f):
return Path(os.path.join(TESTDATA_PATH, f))
return TESTDATA_PATH / f
class FilesTest(unittest.TestCase):
def testFileList_for_testing(self):
options = Options(Path(TESTDATA_PATH), 'x64', 1, 0, True)
options = Options(TESTDATA_PATH, 'x64', 1, 0, True)
self.assertEqual(
gcmole.build_file_list(options),
list(map(abs_test_file, ['tools/gcmole/gcmole-test.cc'])))
def testFileList_x64(self):
options = Options(Path(TESTDATA_PATH), 'x64', 1, 0, False)
options = Options(TESTDATA_PATH, 'x64', 1, 0, False)
expected = [
'file1.cc',
'file2.cc',
@ -51,7 +54,7 @@ class FilesTest(unittest.TestCase):
list(map(abs_test_file, expected)))
def testFileList_x64_shard0(self):
options = Options(Path(TESTDATA_PATH), 'x64', 2, 0, False)
options = Options(TESTDATA_PATH, 'x64', 2, 0, False)
expected = [
'file1.cc',
'x64/file1.cc',
@ -63,7 +66,7 @@ class FilesTest(unittest.TestCase):
list(map(abs_test_file, expected)))
def testFileList_x64_shard1(self):
options = Options(Path(TESTDATA_PATH), 'x64', 2, 1, False)
options = Options(TESTDATA_PATH, 'x64', 2, 1, False)
expected = [
'file2.cc',
'x64/file2.cc',
@ -75,7 +78,7 @@ class FilesTest(unittest.TestCase):
list(map(abs_test_file, expected)))
def testFileList_arm(self):
options = Options(Path(TESTDATA_PATH), 'arm', 1, 0, False)
options = Options(TESTDATA_PATH, 'arm', 1, 0, False)
expected = [
'file1.cc',
'file2.cc',
@ -402,5 +405,29 @@ class SuspectCollectorTest(unittest.TestCase):
with open(temp_dir / 'gccauses') as f:
self.assertEqual(f.read().strip(), gccauses_expected)
class ArgsTest(unittest.TestCase):
def testArgs(self):
"""Test argument retrieval using a fake v8 file system and build dir."""
with tempfile.TemporaryDirectory('gcmole_args_test') as temp_dir:
temp_dir = Path(temp_dir)
temp_out = temp_dir / 'out'
temp_gcmole = temp_dir / 'tools' / 'gcmole' / 'gcmole_args.py'
shutil.copytree(abs_test_file('out'), temp_out)
os.makedirs(temp_gcmole.parent)
shutil.copy(GCMOLE_PATH / 'gcmole_args.py', temp_gcmole)
# Simulate a ninja call relative to the build dir.
subprocess.check_call([sys.executable, temp_gcmole], cwd=temp_out)
with open(temp_dir / 'out' / 'v8_gcmole.args') as f:
self.assertEqual(f.read().split(), [
'-DUSE_GLIB=1', '-DV8_TARGET_ARCH_X64', '-I.', '-Iout/gen',
'-Iinclude', '-Iout/gen/include'
])
if __name__ == '__main__':
unittest.main()

7
tools/gcmole/testdata/v8/out/obj/v8_base_without_compiler.ninja vendored Normal file
View File

@ -0,0 +1,7 @@
defines = -DUSE_GLIB=1 -DV8_TARGET_ARCH_X64
include_dirs = -I.. -Igen -I../include -Igen/include
cflags = -Wall -Werror
cflags_cc = -isystem../../buildtools/third_party/libc++/trunk/include
label_name = v8_base_without_compiler
target_out_dir = obj
target_output_name = v8_base_without_compiler