AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

Revert "Reland "Reuse arch/mode discovery in deopt fuzzer""

Browse Source 
This reverts commit 5442e8b224.

Reason for revert: https://crbug.com/v8/7006

Original change's description:
> Reland "Reuse arch/mode discovery in deopt fuzzer"
> 
> This is a reland of a24c7c9a52
> Original change's description:
> > Reuse arch/mode discovery in deopt fuzzer
> > 
> > Bug: v8:6917
> > Change-Id: I1b7169c8702c8649812b17579d38d64de676ed60
> > Reviewed-on: https://chromium-review.googlesource.com/723420
> > Commit-Queue: Michał Majewski <majeski@google.com>
> > Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> > Cr-Commit-Position: refs/heads/master@{#48838}
> 
> Bug: v8:6917
> Change-Id: I03b2c288257d44c2df9d0fa6cf4750d1c5719d59
> Reviewed-on: https://chromium-review.googlesource.com/735719
> Commit-Queue: Michał Majewski <majeski@google.com>
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#48916}

TBR=machenbach@chromium.org,majeski@google.com

Change-Id: Ibbc16bb7115c8e8b93bd8f39065742d3c2d7739f
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:6917, v8:7006
Reviewed-on: https://chromium-review.googlesource.com/737807
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#48960}
This commit is contained in:
Michał Majewski 2017-10-26 09:43:03 +00:00 committed by Commit Bot
parent 0ff84265c6
commit 5b8702548f
3 changed files with 383 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
tools/testrunner/base_runner.py
View File

@ -73,6 +73,26 @@ MODES = {
},
}
SUPPORTED_ARCHS = [
"android_arm",
"android_arm64",
"android_ia32",
"android_x64",
"arm",
"ia32",
"mips",
"mipsel",
"mips64",
"mips64el",
"s390",
"s390x",
"ppc",
"ppc64",
"x64",
"x32",
"arm64",
]
# Map of test name synonyms to lists of test suites. Should be ordered by
# expected runtimes (suites with slow test cases first). These groups are
# invoked in separate steps on the bots.
@ -125,58 +145,37 @@ class TestRunnerError(Exception):
pass
class BuildConfig(object):
def __init__(self, build_config):
# In V8 land, GN's x86 is called ia32.
if build_config['v8_target_cpu'] == 'x86':
self.arch = 'ia32'
else:
self.arch = build_config['v8_target_cpu']
self.mode = 'debug' if build_config['is_debug'] else 'release'
self.asan = build_config['is_asan']
self.cfi_vptr = build_config['is_cfi']
self.dcheck_always_on = build_config['dcheck_always_on']
self.gcov_coverage = build_config['is_gcov_coverage']
self.msan = build_config['is_msan']
self.no_i18n = not build_config['v8_enable_i18n_support']
self.no_snap = not build_config['v8_use_snapshot']
self.predictable = build_config['v8_enable_verify_predictable']
self.tsan = build_config['is_tsan']
self.ubsan_vptr = build_config['is_ubsan_vptr']
class BaseTestRunner(object):
def __init__(self):
self.outdir = None
self.shell_dir = None
self.build_config = None
self.arch = None
self.mode = None
self.auto_detect = None
def execute(self):
try:
parser = self._create_parser()
options, args = self._parse_args(parser)
self._load_build_config(options)
try:
self._process_default_options(options)
self._process_options(options)
except TestRunnerError:
parser.print_help()
raise
options, args = self._parse_args()
return self._do_execute(options, args)
except TestRunnerError:
return 1
def _create_parser(self):
def _parse_args(self):
parser = optparse.OptionParser()
parser.usage = '%prog [options] [tests]'
parser.description = """TESTS: %s""" % (TEST_MAP["default"])
self._add_parser_default_options(parser)
self._add_parser_options(parser)
return parser
options, args = parser.parse_args()
try:
self._process_default_options(options)
self._process_options(options)
except TestRunnerError:
parser.print_help()
raise
return options, args
def _add_parser_default_options(self, parser):
parser.add_option("--gn", help="Scan out.gn for the last built"
@ -188,53 +187,48 @@ class BaseTestRunner(object):
help="Adapt to path structure used on buildbots",
default=False, action="store_true")
parser.add_option("--arch",
help="The architecture to run tests for: %s")
help=("The architecture to run tests for, "
"'auto' or 'native' for auto-detect: %s" %
SUPPORTED_ARCHS))
parser.add_option("-m", "--mode",
help="The test mode in which to run (uppercase for ninja"
" and buildbot builds): %s" % MODES.keys())
parser.add_option("--shell", help="DEPRECATED! use --shell-dir",
default="")
parser.add_option("--shell-dir", help="Directory containing executables",
default="")
def _add_parser_options(self, parser):
pass
def _parse_args(self, parser):
options, args = parser.parse_args()
def _process_default_options(self, options):
# Try to autodetect configuration based on the build if GN was used.
# This can't be ovveridden by cmd-line arguments.
if options.gn:
outdir = self._get_gn_outdir()
else:
outdir = options.outdir
if any(map(lambda v: v and ',' in v,
[options.arch, options.mode])):
print 'Multiple arch/mode are deprecated'
self.auto_detect = self._read_build_config(outdir, options)
if not self.auto_detect:
if any(map(lambda v: v and ',' in v,
[options.arch, options.mode])):
print 'Multiple arch/mode are deprecated'
raise TestRunnerError()
self.outdir = outdir
if not options.arch or not options.mode:
print('Autodetect mode is not available and therefore '
'--arch and --mode options are required')
raise TestRunnerError()
self.arch = options.arch
self.mode = options.mode
if not self._buildbot_to_v8_mode(self.mode) in MODES:
print "Unknown mode %s" % self.mode
raise TestRunnerError()
return options, args
def _load_build_config(self, options):
for outdir in self._possible_outdirs(options):
try:
self.build_config = self._do_load_build_config(
outdir, options.mode, options.buildbot)
except TestRunnerError:
pass
if not self.build_config:
print 'Failed to load build config'
raise TestRunnerError
print 'Build found: %s' % self.outdir
# Returns possible build paths in order: gn, outdir, outdir/arch.mode
def _possible_outdirs(self, options):
if options.gn:
yield self._get_gn_outdir()
return
yield options.outdir
if options.arch and options.mode:
yield os.path.join(options.outdir,
'%s.%s' % (options.arch, options.mode))
return
if self.arch in ["auto", "native"]:
self.arch = ARCH_GUESS
if not self.arch in SUPPORTED_ARCHS:
print "Unknown architecture %s" % self.arch
raise TestRunnerError()
def _get_gn_outdir(self):
gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN)
@ -251,21 +245,25 @@ class BaseTestRunner(object):
print(">>> Latest GN build found: %s" % latest_config)
return os.path.join(DEFAULT_OUT_GN, latest_config)
def _do_load_build_config(self, outdir, mode, is_buildbot):
if is_buildbot:
# Auto-detect test configurations based on the build (GN only).
# sets:
# - arch
# - mode
# - outdir
def _read_build_config(self, outdir, options):
if options.buildbot:
build_config_path = os.path.join(
BASE_DIR, outdir, mode, "v8_build_config.json")
BASE_DIR, outdir, options.mode, "v8_build_config.json")
else:
build_config_path = os.path.join(
BASE_DIR, outdir, "v8_build_config.json")
if not os.path.exists(build_config_path):
print('Missing build config in the output directory (%s)' %
build_config_path)
raise TestRunnerError()
return False
with open(build_config_path) as f:
try:
build_config_json = json.load(f)
build_config = json.load(f)
except Exception:
print("%s exists but contains invalid json. Is your build up-to-date?"
% build_config_path)
@ -276,24 +274,71 @@ class BaseTestRunner(object):
# This ensures that we'll also take the build products from there.
self.outdir = os.path.dirname(build_config_path)
return BuildConfig(build_config_json)
# In V8 land, GN's x86 is called ia32.
if build_config["v8_target_cpu"] == "x86":
build_config["v8_target_cpu"] = "ia32"
def _process_default_options(self, options):
# We don't use the mode for more path-magic.
# Therefore transform the buildbot mode here to fix build_config value.
if options.buildbot and options.mode:
if options.mode:
# In auto-detect mode we don't use the mode for more path-magic.
# Therefore transform the buildbot mode here to fit to the GN build
# config.
options.mode = self._buildbot_to_v8_mode(options.mode)
def check_consistency(name, option_val, cfg_val):
if option_val and option_val != cfg_val:
print('Attempted to set %s to %s while build config is %s.' % (
name, option_val, cfg_val))
# TODO(majeski): merge next two loops and put everything in self.
# Get options from the build config. Sanity check that we're not trying to
# use inconsistent options.
for param, value in (
('arch', build_config["v8_target_cpu"]),
('mode', 'debug' if build_config["is_debug"] else 'release'),
):
cmd_line_value = getattr(options, param)
if (cmd_line_value not in [None, True, False] and
cmd_line_value != value):
# TODO(machenbach): This is for string options only. Requires
# options to not have default values. We should make this more
# modular and implement it in our own version of the option parser.
print("Attempted to set %s to %s, while build is %s." %
(param, cmd_line_value, value))
raise TestRunnerError()
if cmd_line_value == True and value == False:
print("Attempted to turn on %s, but it's not available." % param)
raise TestRunnerError()
if cmd_line_value != value:
print(">>> Auto-detected %s=%s" % (param, value))
setattr(self, param, value)
check_consistency('arch', options.arch, self.build_config.arch)
check_consistency('mode', options.mode, self.build_config.mode)
# Update options based on the build config. Sanity check that we're not
# trying to use inconsistent options.
for param, value in (
('asan', build_config["is_asan"]),
('cfi_vptr', build_config["is_cfi"]),
('dcheck_always_on', build_config["dcheck_always_on"]),
('gcov_coverage', build_config["is_gcov_coverage"]),
('msan', build_config["is_msan"]),
('no_i18n', not build_config["v8_enable_i18n_support"]),
('no_snap', not build_config["v8_use_snapshot"]),
('predictable', build_config["v8_enable_verify_predictable"]),
('tsan', build_config["is_tsan"]),
('ubsan_vptr', build_config["is_ubsan_vptr"]),
):
cmd_line_value = getattr(options, param)
if (cmd_line_value not in [None, True, False] and
cmd_line_value != value):
# TODO(machenbach): This is for string options only. Requires
# options to not have default values. We should make this more
# modular and implement it in our own version of the option parser.
print("Attempted to set %s to %s, while build is %s." %
(param, cmd_line_value, value))
raise TestRunnerError()
if cmd_line_value == True and value == False:
print("Attempted to turn on %s, but it's not available." % param)
raise TestRunnerError()
if cmd_line_value != value:
print(">>> Auto-detected %s=%s" % (param, value))
setattr(options, param, value)
self._set_shell_dir(options)
return True
def _buildbot_to_v8_mode(self, config):
"""Convert buildbot build configs to configs understood by the v8 runner.
@ -304,20 +349,6 @@ class BaseTestRunner(object):
mode = config[:-4] if config.endswith('_x64') else config
return mode.lower()
def _set_shell_dir(self, options):
self.shell_dir = options.shell_dir
if not self.shell_dir:
# TODO(majeski): drop this option
if options.shell:
print "Warning: --shell is deprecated, use --shell-dir instead."
self.shell_dir = os.path.dirname(options.shell)
else:
# If an output dir with a build was passed, test directly in that
# directory.
self.shell_dir = os.path.join(BASE_DIR, self.outdir)
if not os.path.exists(self.shell_dir):
raise Exception('Could not find shell_dir: "%s"' % self.shell_dir)
def _process_options(self, options):
pass

195
tools/testrunner/deopt_fuzzer.py
View File

@ -9,9 +9,11 @@ from os.path import join
import json
import math
import multiprocessing
import optparse
import os
import random
import shlex
import subprocess
import sys
import time
@ -26,13 +28,38 @@ from testrunner.local import verbose
from testrunner.objects import context
# Base dir of the v8 checkout to be used as cwd.
BASE_DIR = (
os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.abspath(__file__)))))
ARCH_GUESS = utils.DefaultArch()
DEFAULT_TESTS = ["mjsunit", "webkit"]
TIMEOUT_DEFAULT = 60
TIMEOUT_SCALEFACTOR = {"debug" : 4,
"release" : 1 }
MODE_FLAGS = {
"debug" : ["--nohard-abort", "--enable-slow-asserts",
"--verify-heap", "--noconcurrent-recompilation"],
"release" : ["--nohard-abort", "--noconcurrent-recompilation"]}
SUPPORTED_ARCHS = ["android_arm",
"android_ia32",
"arm",
"ia32",
"ppc",
"ppc64",
"s390",
"s390x",
"mipsel",
"x64"]
# Double the timeout for these:
SLOW_ARCHS = ["arm",
SLOW_ARCHS = ["android_arm",
"android_ia32",
"arm",
"mipsel"]
MAX_DEOPT = 1000000000
DISTRIBUTION_MODES = ["smooth", "random"]
@ -105,57 +132,100 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
options.distribution_factor2)
def _add_parser_options(self, parser):
parser.add_option("--command-prefix",
def _build_options(self):
result = optparse.OptionParser()
result.add_option("--arch",
help=("The architecture to run tests for, "
"'auto' or 'native' for auto-detect"),
default="ia32,x64,arm")
result.add_option("--arch-and-mode",
help="Architecture and mode in the format 'arch.mode'",
default=None)
result.add_option("--asan",
help="Regard test expectations for ASAN",
default=False, action="store_true")
result.add_option("--buildbot",
help="Adapt to path structure used on buildbots",
default=False, action="store_true")
result.add_option("--dcheck-always-on",
help="Indicates that V8 was compiled with DCHECKs"
" enabled",
default=False, action="store_true")
result.add_option("--command-prefix",
help="Prepended to each shell command used to run a test",
default="")
parser.add_option("--coverage", help=("Exponential test coverage "
result.add_option("--coverage", help=("Exponential test coverage "
"(range 0.0, 1.0) - 0.0: one test, 1.0 all tests (slow)"),
default=0.4, type="float")
parser.add_option("--coverage-lift", help=("Lifts test coverage for tests "
result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
"with a small number of deopt points (range 0, inf)"),
default=20, type="int")
parser.add_option("--distribution-factor1", help=("Factor of the first "
result.add_option("--distribution-factor1", help=("Factor of the first "
"derivation of the distribution function"), default=2.0,
type="float")
parser.add_option("--distribution-factor2", help=("Factor of the second "
result.add_option("--distribution-factor2", help=("Factor of the second "
"derivation of the distribution function"), default=0.7,
type="float")
parser.add_option("--distribution-mode", help=("How to select deopt points "
result.add_option("--distribution-mode", help=("How to select deopt points "
"for a given test (smooth|random)"),
default="smooth")
parser.add_option("--dump-results-file", help=("Dump maximum number of "
result.add_option("--dump-results-file", help=("Dump maximum number of "
"deopt points per test to a file"))
parser.add_option("--extra-flags",
result.add_option("--extra-flags",
help="Additional flags to pass to each test command",
default="")
parser.add_option("--isolates", help="Whether to test isolates",
result.add_option("--isolates", help="Whether to test isolates",
default=False, action="store_true")
parser.add_option("-j", help="The number of parallel tasks to run",
result.add_option("-j", help="The number of parallel tasks to run",
default=0, type="int")
parser.add_option("-p", "--progress",
result.add_option("-m", "--mode",
help="The test modes in which to run (comma-separated)",
default="release,debug")
result.add_option("--outdir", help="Base directory with compile output",
default="out")
result.add_option("-p", "--progress",
help=("The style of progress indicator"
" (verbose, dots, color, mono)"),
choices=progress.PROGRESS_INDICATORS.keys(),
default="mono")
parser.add_option("--shard-count",
result.add_option("--shard-count",
help="Split testsuites into this number of shards",
default=1, type="int")
parser.add_option("--shard-run",
result.add_option("--shard-run",
help="Run this shard from the split up tests.",
default=1, type="int")
parser.add_option("--seed", help="The seed for the random distribution",
result.add_option("--shell-dir", help="Directory containing executables",
default="")
result.add_option("--seed", help="The seed for the random distribution",
type="int")
parser.add_option("-t", "--timeout", help="Timeout in seconds",
result.add_option("-t", "--timeout", help="Timeout in seconds",
default= -1, type="int")
parser.add_option("-v", "--verbose", help="Verbose output",
result.add_option("-v", "--verbose", help="Verbose output",
default=False, action="store_true")
parser.add_option("--random-seed", default=0, dest="random_seed",
result.add_option("--random-seed", default=0, dest="random_seed",
help="Default seed for initializing random generator")
return parser
return result
def _process_options(self, options):
# Architecture and mode related stuff.
if options.arch_and_mode:
tokens = options.arch_and_mode.split(".")
options.arch = tokens[0]
options.mode = tokens[1]
options.mode = options.mode.split(",")
for mode in options.mode:
if not mode.lower() in ["debug", "release"]:
print "Unknown mode %s" % mode
return False
if options.arch in ["auto", "native"]:
options.arch = ARCH_GUESS
options.arch = options.arch.split(",")
for arch in options.arch:
if not arch in SUPPORTED_ARCHS:
print "Unknown architecture %s" % arch
return False
# Special processing of other options, sorted alphabetically.
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = shlex.split(options.extra_flags)
@ -200,11 +270,20 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
count += 1
return shard
def _do_execute(self, options, args):
# TODO(majeski): reuse baseclass code
def execute(self):
# Use the v8 root as cwd as some test cases use "load" with relative paths.
os.chdir(base_runner.BASE_DIR)
os.chdir(BASE_DIR)
suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test"))
parser = self._build_options()
(options, args) = parser.parse_args()
if not self._process_options(options):
parser.print_help()
return 1
exit_code = 0
suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
if len(args) == 0:
suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
@ -219,14 +298,18 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
suites = []
for root in suite_paths:
suite = testsuite.TestSuite.LoadTestSuite(
os.path.join(base_runner.BASE_DIR, "test", root))
os.path.join(BASE_DIR, "test", root))
if suite:
suites.append(suite)
try:
return self._execute(args, options, suites)
except KeyboardInterrupt:
return 2
for mode in options.mode:
for arch in options.arch:
try:
code = self._execute(arch, mode, args, options, suites, BASE_DIR)
exit_code = exit_code or code
except KeyboardInterrupt:
return 2
return exit_code
def _calculate_n_tests(self, m, options):
@ -240,26 +323,33 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
return int(math.pow(m, (m * c + l) / (m + l)))
def _execute(self, args, options, suites):
print(">>> Running tests for %s.%s" % (self.build_config.arch,
self.build_config.mode))
def _execute(self, arch, mode, args, options, suites, workspace):
print(">>> Running tests for %s.%s" % (arch, mode))
dist = self._distribution(options)
shell_dir = options.shell_dir
if not shell_dir:
if options.buildbot:
shell_dir = os.path.join(workspace, options.outdir, mode)
mode = mode.lower()
else:
shell_dir = os.path.join(workspace, options.outdir,
"%s.%s" % (arch, mode))
shell_dir = os.path.relpath(shell_dir)
# Populate context object.
mode_flags = base_runner.MODES[self.build_config.mode]['flags']
mode_flags = MODE_FLAGS[mode]
timeout = options.timeout
if timeout == -1:
# Simulators are slow, therefore allow a longer default timeout.
if self.build_config.arch in SLOW_ARCHS:
if arch in SLOW_ARCHS:
timeout = 2 * TIMEOUT_DEFAULT;
else:
timeout = TIMEOUT_DEFAULT;
timeout *= TIMEOUT_SCALEFACTOR[self.build_config.mode]
ctx = context.Context(self.build_config.arch,
self.build_config.mode,
self.shell_dir,
timeout *= TIMEOUT_SCALEFACTOR[mode]
ctx = context.Context(arch, mode, shell_dir,
mode_flags, options.verbose,
timeout, options.isolates,
options.command_prefix,
@ -276,26 +366,25 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
# Find available test suites and read test cases from them.
variables = {
"arch": self.build_config.arch,
"asan": self.build_config.asan,
"byteorder": sys.byteorder,
"dcheck_always_on": self.build_config.dcheck_always_on,
"arch": arch,
"asan": options.asan,
"deopt_fuzzer": True,
"gc_stress": False,
"gcov_coverage": self.build_config.gcov_coverage,
"gcov_coverage": False,
"isolates": options.isolates,
"mode": base_runner.MODES[self.build_config.mode]["status_mode"],
"msan": self.build_config.msan,
"no_harness": False,
"no_i18n": self.build_config.no_i18n,
"no_snap": self.build_config.no_snap,
"novfp3": False,
"predictable": self.build_config.predictable,
"simulator": utils.UseSimulator(self.build_config.arch),
"simulator_run": False,
"mode": mode,
"no_i18n": False,
"no_snap": False,
"simulator": utils.UseSimulator(arch),
"system": utils.GuessOS(),
"tsan": self.build_config.tsan,
"ubsan_vptr": self.build_config.ubsan_vptr,
"tsan": False,
"msan": False,
"dcheck_always_on": options.dcheck_always_on,
"novfp3": False,
"predictable": False,
"byteorder": sys.byteorder,
"no_harness": False,
"ubsan_vptr": False,
}
num_tests = 0
test_id = 0

151
tools/testrunner/standard_runner.py
View File

@ -28,6 +28,7 @@ from testrunner.network import network_execution
from testrunner.objects import context
TIMEOUT_DEFAULT = 60
# Variants ordered by expected runtime (slowest first).
@ -59,7 +60,11 @@ GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
"--concurrent-recompilation"]
# Double the timeout for these:
SLOW_ARCHS = ["arm",
SLOW_ARCHS = ["android_arm",
"android_arm64",
"android_ia32",
"android_x64",
"arm",
"mips",
"mipsel",
"mips64",
@ -128,16 +133,23 @@ class StandardTestRunner(base_runner.BaseTestRunner):
s.PrepareSources()
try:
return self._execute(args, options, suites)
return self._execute(self.arch, self.mode, args, options, suites)
except KeyboardInterrupt:
return 2
def _add_parser_options(self, parser):
parser.add_option("--asan",
help="Regard test expectations for ASAN",
default=False, action="store_true")
parser.add_option("--sancov-dir",
help="Directory where to collect coverage data")
parser.add_option("--cfi-vptr",
help="Run tests with UBSAN cfi_vptr option.",
default=False, action="store_true")
parser.add_option("--dcheck-always-on",
help="Indicates that V8 was compiled with DCHECKs"
" enabled",
default=False, action="store_true")
parser.add_option("--novfp3",
help="Indicates that V8 was compiled without VFP3"
" support",
@ -153,6 +165,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
parser.add_option("--gc-stress",
help="Switch on GC stress mode",
default=False, action="store_true")
parser.add_option("--gcov-coverage",
help="Uses executables instrumented for gcov coverage",
default=False, action="store_true")
parser.add_option("--command-prefix",
help="Prepended to each shell command used to run a"
" test",
@ -167,6 +182,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
parser.add_option("--no-harness", "--noharness",
help="Run without test harness of a given suite",
default=False, action="store_true")
parser.add_option("--no-i18n", "--noi18n",
help="Skip internationalization tests",
default=False, action="store_true")
parser.add_option("--network", help="Distribute tests on the network",
default=False, dest="network", action="store_true")
parser.add_option("--no-network", "--nonetwork",
@ -175,6 +193,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
parser.add_option("--no-presubmit", "--nopresubmit",
help='Skip presubmit checks (deprecated)',
default=False, dest="no_presubmit", action="store_true")
parser.add_option("--no-snap", "--nosnap",
help='Test a build compiled without snapshot.',
default=False, dest="no_snap", action="store_true")
parser.add_option("--no-sorting", "--nosorting",
help="Don't sort tests according to duration of last"
" run.",
@ -189,6 +210,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
default=False, action="store_true",
help="Use exhaustive set of default variants:"
" \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS))
parser.add_option("--predictable",
help="Compare output of several reruns of each test",
default=False, action="store_true")
parser.add_option("-p", "--progress",
help=("The style of progress indicator"
" (verbose, dots, color, mono)"),
@ -216,6 +240,10 @@ class StandardTestRunner(base_runner.BaseTestRunner):
parser.add_option("--shard-run",
help="Run this shard from the split up tests.",
default=1, type="int")
parser.add_option("--shell", help="DEPRECATED! use --shell-dir",
default="")
parser.add_option("--shell-dir", help="Directory containing executables",
default="")
parser.add_option("--dont-skip-slow-simulator-tests",
help="Don't skip more slow tests when using a"
" simulator.",
@ -228,6 +256,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
default=False, action="store_true")
parser.add_option("-t", "--timeout", help="Timeout in seconds",
default=TIMEOUT_DEFAULT, type="int")
parser.add_option("--tsan",
help="Regard test expectations for TSAN",
default=False, action="store_true")
parser.add_option("-v", "--verbose", help="Verbose output",
default=False, action="store_true")
parser.add_option("--valgrind", help="Run tests through valgrind",
@ -244,6 +275,12 @@ class StandardTestRunner(base_runner.BaseTestRunner):
parser.add_option("--random-seed-stress-count", default=1, type="int",
dest="random_seed_stress_count",
help="Number of runs with different random seeds")
parser.add_option("--ubsan-vptr",
help="Regard test expectations for UBSanVptr",
default=False, action="store_true")
parser.add_option("--msan",
help="Regard test expectations for UBSanVptr",
default=False, action="store_true")
def _process_options(self, options):
global VARIANTS
@ -262,7 +299,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if self.build_config.asan:
if options.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
@ -277,7 +314,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
if options.msan:
VARIANTS = ["default"]
if options.j == 0:
@ -312,7 +349,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
if not set(VARIANTS).issubset(ALL_VARIANTS):
print "All variants must be in %s" % str(ALL_VARIANTS)
raise base_runner.TestRunnerError()
if self.build_config.predictable:
if options.predictable:
VARIANTS = ["default"]
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
@ -321,6 +358,10 @@ class StandardTestRunner(base_runner.BaseTestRunner):
# Dedupe.
VARIANTS = list(set(VARIANTS))
if not options.shell_dir:
if options.shell:
print "Warning: --shell is deprecated, use --shell-dir instead."
options.shell_dir = os.path.dirname(options.shell)
if options.valgrind:
run_valgrind = os.path.join("tools", "run-valgrind.py")
# This is OK for distributed running, so we don't need to disable
@ -333,7 +374,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
if options.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
@ -356,7 +397,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
external_symbolizer_path = '"%s.exe"' % external_symbolizer_path
symbolizer = 'external_symbolizer_path=%s' % external_symbolizer_path
if self.build_config.asan:
if options.asan:
asan_options = [symbolizer, "allow_user_segv_handler=1"]
if not utils.GuessOS() in ['macos', 'windows']:
# LSAN is not available on mac and windows.
@ -372,7 +413,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
"allow_user_segv_handler=1",
])
if self.build_config.cfi_vptr:
if options.cfi_vptr:
os.environ['UBSAN_OPTIONS'] = ":".join([
'print_stacktrace=1',
'print_summary=1',
@ -380,16 +421,16 @@ class StandardTestRunner(base_runner.BaseTestRunner):
symbolizer,
])
if self.build_config.ubsan_vptr:
if options.ubsan_vptr:
os.environ['UBSAN_OPTIONS'] = ":".join([
'print_stacktrace=1',
symbolizer,
])
if self.build_config.msan:
if options.msan:
os.environ['MSAN_OPTIONS'] = symbolizer
if self.build_config.tsan:
if options.tsan:
suppressions_file = os.path.join(
base_runner.BASE_DIR,
'tools',
@ -410,40 +451,57 @@ class StandardTestRunner(base_runner.BaseTestRunner):
seed = random.SystemRandom().randint(-2147483648, 2147483647)
return seed
def _execute(self, args, options, suites):
print(">>> Running tests for %s.%s" % (self.build_config.arch,
self.build_config.mode))
def _execute(self, arch, mode, args, options, suites):
print(">>> Running tests for %s.%s" % (arch, mode))
mode_options = base_runner.MODES[self.build_config.mode]
shell_dir = options.shell_dir
if not shell_dir:
if self.auto_detect:
# If an output dir with a build was passed, test directly in that
# directory.
shell_dir = os.path.join(base_runner.BASE_DIR, self.outdir)
elif options.buildbot:
# TODO(machenbach): Get rid of different output folder location on
# buildbot. Currently this is capitalized Release and Debug.
shell_dir = os.path.join(base_runner.BASE_DIR, self.outdir, mode)
mode = self._buildbot_to_v8_mode(mode)
else:
shell_dir = os.path.join(
base_runner.BASE_DIR,
self.outdir,
"%s.%s" % (arch, base_runner.MODES[mode]["output_folder"]),
)
if not os.path.exists(shell_dir):
raise Exception('Could not find shell_dir: "%s"' % shell_dir)
# Populate context object.
mode_flags = mode_options["flags"]
mode_flags = base_runner.MODES[mode]["flags"]
# Simulators are slow, therefore allow a longer timeout.
if self.build_config.arch in SLOW_ARCHS:
if arch in SLOW_ARCHS:
options.timeout *= 2
options.timeout *= mode_options["timeout_scalefactor"]
options.timeout *= base_runner.MODES[mode]["timeout_scalefactor"]
if self.build_config.predictable:
if options.predictable:
# Predictable mode is slower.
options.timeout *= 2
ctx = context.Context(self.build_config.arch,
mode_options["execution_mode"],
self.shell_dir,
ctx = context.Context(arch,
base_runner.MODES[mode]["execution_mode"],
shell_dir,
mode_flags,
options.verbose,
options.timeout,
options.isolates,
options.command_prefix,
options.extra_flags,
self.build_config.no_i18n,
options.no_i18n,
options.random_seed,
options.no_sorting,
options.rerun_failures_count,
options.rerun_failures_max,
self.build_config.predictable,
options.predictable,
options.no_harness,
use_perf_data=not options.swarming,
sancov_dir=options.sancov_dir)
@ -451,35 +509,32 @@ class StandardTestRunner(base_runner.BaseTestRunner):
# TODO(all): Combine "simulator" and "simulator_run".
# TODO(machenbach): In GN we can derive simulator run from
# target_arch != v8_target_arch in the dumped build config.
simulator_run = (
not options.dont_skip_simulator_slow_tests and
self.build_config.arch in [
'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
'ppc64', 's390', 's390x'] and
bool(base_runner.ARCH_GUESS) and
self.build_config.arch != base_runner.ARCH_GUESS)
simulator_run = not options.dont_skip_simulator_slow_tests and \
arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \
'ppc', 'ppc64', 's390', 's390x'] and \
bool(base_runner.ARCH_GUESS) and arch != base_runner.ARCH_GUESS
# Find available test suites and read test cases from them.
variables = {
"arch": self.build_config.arch,
"asan": self.build_config.asan,
"byteorder": sys.byteorder,
"dcheck_always_on": self.build_config.dcheck_always_on,
"arch": arch,
"asan": options.asan,
"deopt_fuzzer": False,
"gc_stress": options.gc_stress,
"gcov_coverage": self.build_config.gcov_coverage,
"gcov_coverage": options.gcov_coverage,
"isolates": options.isolates,
"mode": mode_options["status_mode"],
"msan": self.build_config.msan,
"no_harness": options.no_harness,
"no_i18n": self.build_config.no_i18n,
"no_snap": self.build_config.no_snap,
"novfp3": options.novfp3,
"predictable": self.build_config.predictable,
"simulator": utils.UseSimulator(self.build_config.arch),
"mode": base_runner.MODES[mode]["status_mode"],
"no_i18n": options.no_i18n,
"no_snap": options.no_snap,
"simulator_run": simulator_run,
"simulator": utils.UseSimulator(arch),
"system": utils.GuessOS(),
"tsan": self.build_config.tsan,
"ubsan_vptr": self.build_config.ubsan_vptr,
"tsan": options.tsan,
"msan": options.msan,
"dcheck_always_on": options.dcheck_always_on,
"novfp3": options.novfp3,
"predictable": options.predictable,
"byteorder": sys.byteorder,
"no_harness": options.no_harness,
"ubsan_vptr": options.ubsan_vptr,
}
all_tests = []
num_tests = 0
@ -546,8 +601,8 @@ class StandardTestRunner(base_runner.BaseTestRunner):
if options.json_test_results:
progress_indicator.Register(progress.JsonTestProgressIndicator(
options.json_test_results,
self.build_config.arch,
mode_options["execution_mode"],
arch,
base_runner.MODES[mode]["execution_mode"],
ctx.random_seed))
if options.flakiness_results:
progress_indicator.Register(progress.FlakinessTestProgressIndicator(