v8/tools/testrunner/standard_runner.py
Michael Achenbach ba30988cbc Revert "[test] Random seed processor"
This reverts commit 0db74d4974.

Reason for revert: https://chromium-swarm.appspot.com/task?id=3b609f9976bac610&refresh=10&show_raw=1

Original change's description:
> [test] Random seed processor
> 
> 1. --total-timeout-sec now available for ./run-tests.py. It can be
> useful with infinite seed stressing
> 2. random seed dropped from the context. Now JSON progress indicator
> gets it from the list of command args.
> 
> Bug: v8:6917
> Cq-Include-Trybots: luci.v8.try:v8_linux_noi18n_rel_ng
> Change-Id: I73e535bc8face9b913c696b8d5e3a246fa231004
> Reviewed-on: https://chromium-review.googlesource.com/888524
> Commit-Queue: Michał Majewski <majeski@google.com>
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#50964}

TBR=machenbach@chromium.org,sergiyb@chromium.org,majeski@google.com

Change-Id: I2d96ea328cda2d09b01ff455e47c77d567fafe00
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:6917
Cq-Include-Trybots: luci.v8.try:v8_linux_noi18n_rel_ng
Reviewed-on: https://chromium-review.googlesource.com/894522
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50970}
2018-01-30 22:05:03 +00:00

646 lines
25 KiB
Python
Executable File

#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from collections import OrderedDict
from os.path import join
import multiprocessing
import os
import random
import re
import shlex
import subprocess
import sys
import time
# Adds testrunner to the path hence it has to be imported at the beggining.
import base_runner
from testrunner.local import execution
from testrunner.local import progress
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.local import verbose
from testrunner.local.variants import ALL_VARIANTS
from testrunner.objects import context
from testrunner.objects import predictable
from testrunner.testproc.execution import ExecutionProc
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
from testrunner.testproc.progress import (VerboseProgressIndicator,
ResultsTracker,
TestsCounter)
from testrunner.testproc.rerun import RerunProc
from testrunner.testproc.variant import VariantProc
TIMEOUT_DEFAULT = 60
VARIANTS = ["default"]
MORE_VARIANTS = [
"nooptimization",
"stress",
"stress_background_compile",
"stress_incremental_marking",
"wasm_traps",
]
VARIANT_ALIASES = {
# The default for developer workstations.
"dev": VARIANTS,
# Additional variants, run on all bots.
"more": MORE_VARIANTS,
# Shortcut for the two above ("more" first - it has the longer running tests).
"exhaustive": MORE_VARIANTS + VARIANTS,
# Additional variants, run on a subset of bots.
"extra": ["future", "liftoff", "trusted"],
}
GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
"--concurrent-recompilation-queue-length=64",
"--concurrent-recompilation-delay=500",
"--concurrent-recompilation"]
RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000",
"--stress-compaction-random"]
# Double the timeout for these:
SLOW_ARCHS = ["arm",
"mips",
"mipsel",
"mips64",
"mips64el",
"s390",
"s390x",
"arm64"]
PREDICTABLE_WRAPPER = os.path.join(
base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')
# Staging default. When set to True it overwrites the two options below.
USE_STAGING = True
# Specifies which builders should use the staging test-runner.
# Mapping from mastername to list of buildernames. Buildernames can be strings
# or compiled regexps which will be matched.
BUILDER_WHITELIST_STAGING = {
}
_RE_TYPE = type(re.compile(''))
# Specifies which architectures are whitelisted to use the staging test-runner.
# List of arch strings, e.g. "x64".
ARCH_WHITELIST_STAGING = [
]
class StandardTestRunner(base_runner.BaseTestRunner):
def __init__(self, *args, **kwargs):
super(StandardTestRunner, self).__init__(*args, **kwargs)
self.sancov_dir = None
self._variants = None
def _get_default_suite_names(self):
return ['default']
def _do_execute(self, suites, args, options):
if options.swarming:
# Swarming doesn't print how isolated commands are called. Lets make
# this less cryptic by printing it ourselves.
print ' '.join(sys.argv)
return self._execute(args, options, suites)
def _add_parser_options(self, parser):
parser.add_option("--sancov-dir",
help="Directory where to collect coverage data")
parser.add_option("--cfi-vptr",
help="Run tests with UBSAN cfi_vptr option.",
default=False, action="store_true")
parser.add_option("--novfp3",
help="Indicates that V8 was compiled without VFP3"
" support",
default=False, action="store_true")
parser.add_option("--cat", help="Print the source of the tests",
default=False, action="store_true")
parser.add_option("--slow-tests",
help="Regard slow tests (run|skip|dontcare)",
default="dontcare")
parser.add_option("--pass-fail-tests",
help="Regard pass|fail tests (run|skip|dontcare)",
default="dontcare")
parser.add_option("--gc-stress",
help="Switch on GC stress mode",
default=False, action="store_true")
parser.add_option("--random-gc-stress",
help="Switch on random GC stress mode",
default=False, action="store_true")
parser.add_option("--command-prefix",
help="Prepended to each shell command used to run a"
" test",
default="")
parser.add_option("--extra-flags",
help="Additional flags to pass to each test command",
action="append", default=[])
parser.add_option("--infra-staging", help="Use new test runner features",
dest='infra_staging', default=None,
action="store_true")
parser.add_option("--no-infra-staging",
help="Opt out of new test runner features",
dest='infra_staging', default=None,
action="store_false")
parser.add_option("--isolates", help="Whether to test isolates",
default=False, action="store_true")
parser.add_option("-j", help="The number of parallel tasks to run",
default=0, type="int")
parser.add_option("--no-harness", "--noharness",
help="Run without test harness of a given suite",
default=False, action="store_true")
parser.add_option("--no-presubmit", "--nopresubmit",
help='Skip presubmit checks (deprecated)',
default=False, dest="no_presubmit", action="store_true")
parser.add_option("--no-sorting", "--nosorting",
help="Don't sort tests according to duration of last"
" run.",
default=False, dest="no_sorting", action="store_true")
parser.add_option("--no-variants", "--novariants",
help="Deprecated. "
"Equivalent to passing --variants=default",
default=False, dest="no_variants", action="store_true")
parser.add_option("--variants",
help="Comma-separated list of testing variants;"
" default: \"%s\"" % ",".join(VARIANTS))
parser.add_option("--exhaustive-variants",
default=False, action="store_true",
help="Deprecated. "
"Equivalent to passing --variants=exhaustive")
parser.add_option("-p", "--progress",
help=("The style of progress indicator"
" (verbose, dots, color, mono)"),
choices=progress.PROGRESS_INDICATORS.keys(),
default="mono")
parser.add_option("--quickcheck", default=False, action="store_true",
help=("Quick check mode (skip slow tests)"))
parser.add_option("--report", help="Print a summary of the tests to be"
" run",
default=False, action="store_true")
parser.add_option("--json-test-results",
help="Path to a file for storing json results.")
parser.add_option("--flakiness-results",
help="Path to a file for storing flakiness json.")
parser.add_option("--rerun-failures-count",
help=("Number of times to rerun each failing test case."
" Very slow tests will be rerun only once."),
default=0, type="int")
parser.add_option("--rerun-failures-max",
help="Maximum number of failing test cases to rerun.",
default=100, type="int")
parser.add_option("--dont-skip-slow-simulator-tests",
help="Don't skip more slow tests when using a"
" simulator.",
default=False, action="store_true",
dest="dont_skip_simulator_slow_tests")
parser.add_option("--swarming",
help="Indicates running test driver on swarming.",
default=False, action="store_true")
parser.add_option("--time", help="Print timing information after running",
default=False, action="store_true")
parser.add_option("-t", "--timeout", help="Timeout in seconds",
default=TIMEOUT_DEFAULT, type="int")
parser.add_option("--warn-unused", help="Report unused rules",
default=False, action="store_true")
parser.add_option("--junitout", help="File name of the JUnit output")
parser.add_option("--junittestsuite",
help="The testsuite name in the JUnit output file",
default="v8tests")
parser.add_option("--random-seed", default=0, dest="random_seed",
help="Default seed for initializing random generator",
type=int)
parser.add_option("--random-seed-stress-count", default=1, type="int",
dest="random_seed_stress_count",
help="Number of runs with different random seeds")
def _use_staging(self, options):
if options.infra_staging is not None:
# True or False are used to explicitly opt in or out.
return options.infra_staging
if USE_STAGING:
return True
builder_configs = BUILDER_WHITELIST_STAGING.get(options.mastername, [])
for builder_config in builder_configs:
if (isinstance(builder_config, _RE_TYPE) and
builder_config.match(options.buildername)):
return True
if builder_config == options.buildername:
return True
for arch in ARCH_WHITELIST_STAGING:
if self.build_config.arch == arch:
return True
return False
def _process_options(self, options):
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print("sancov-dir %s doesn't exist" % self.sancov_dir)
raise base_runner.TestRunnerError()
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = sum(map(shlex.split, options.extra_flags), [])
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if options.random_gc_stress:
options.extra_flags += RANDOM_GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
if options.novfp3:
options.extra_flags.append("--noenable-vfp3")
if options.no_variants: # pragma: no cover
print ("Option --no-variants is deprecated. "
"Pass --variants=default instead.")
assert not options.variants
options.variants = "default"
if options.exhaustive_variants: # pragma: no cover
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print ("Option --exhaustive-variants is deprecated. "
"Pass --variants=exhaustive instead.")
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = "exhaustive"
if options.quickcheck:
assert not options.variants
options.variants = "stress,default"
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
if self.build_config.predictable:
options.variants = "default"
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
options.extra_flags.append("--no-inline-new")
# Add predictable wrapper to command prefix.
options.command_prefix = (
[sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = "default"
if options.j == 0:
options.j = multiprocessing.cpu_count()
if options.random_seed_stress_count <= 1 and options.random_seed == 0:
options.random_seed = self._random_seed()
if options.variants == "infra_staging":
options.variants = "exhaustive"
options.infra_staging = True
# Use staging on whitelisted masters/builders.
options.infra_staging = self._use_staging(options)
self._variants = self._parse_variants(options.variants)
def CheckTestMode(name, option): # pragma: no cover
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
# TODO(machenbach): uncomment after infra side lands.
# base_runner.TEST_MAP["d8_default"].remove("intl")
def _parse_variants(self, aliases_str):
# Use developer defaults if no variant was specified.
aliases_str = aliases_str or 'dev'
aliases = aliases_str.split(',')
user_variants = set(reduce(
list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))
result = [v for v in ALL_VARIANTS if v in user_variants]
if len(result) == len(user_variants):
return result
for v in user_variants:
if v not in ALL_VARIANTS:
print 'Unknown variant: %s' % v
raise base_runner.TestRunnerError()
assert False, 'Unreachable'
def _setup_env(self):
super(StandardTestRunner, self)._setup_env()
symbolizer_option = self._get_external_symbolizer_option()
if self.sancov_dir:
os.environ['ASAN_OPTIONS'] = ":".join([
'coverage=1',
'coverage_dir=%s' % self.sancov_dir,
symbolizer_option,
"allow_user_segv_handler=1",
])
def _random_seed(self):
seed = 0
while not seed:
seed = random.SystemRandom().randint(-2147483648, 2147483647)
return seed
def _execute(self, args, options, suites):
print(">>> Running tests for %s.%s" % (self.build_config.arch,
self.mode_name))
# Populate context object.
# Simulators are slow, therefore allow a longer timeout.
if self.build_config.arch in SLOW_ARCHS:
options.timeout *= 2
options.timeout *= self.mode_options.timeout_scalefactor
if self.build_config.predictable:
# Predictable mode is slower.
options.timeout *= 2
ctx = context.Context(self.build_config.arch,
self.mode_options.execution_mode,
self.outdir,
self.mode_options.flags,
options.verbose,
options.timeout,
options.isolates,
options.command_prefix,
options.extra_flags,
self.build_config.no_i18n,
options.random_seed,
options.no_sorting,
options.rerun_failures_count,
options.rerun_failures_max,
options.no_harness,
use_perf_data=not options.swarming,
sancov_dir=self.sancov_dir,
infra_staging=options.infra_staging)
# TODO(all): Combine "simulator" and "simulator_run".
# TODO(machenbach): In GN we can derive simulator run from
# target_arch != v8_target_arch in the dumped build config.
simulator_run = (
not options.dont_skip_simulator_slow_tests and
self.build_config.arch in [
'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
'ppc64', 's390', 's390x'] and
bool(base_runner.ARCH_GUESS) and
self.build_config.arch != base_runner.ARCH_GUESS)
# Find available test suites and read test cases from them.
variables = {
"arch": self.build_config.arch,
"asan": self.build_config.asan,
"byteorder": sys.byteorder,
"dcheck_always_on": self.build_config.dcheck_always_on,
"deopt_fuzzer": False,
"gc_fuzzer": False,
"gc_stress": options.gc_stress or options.random_gc_stress,
"gcov_coverage": self.build_config.gcov_coverage,
"isolates": options.isolates,
"mode": self.mode_options.status_mode,
"msan": self.build_config.msan,
"no_harness": options.no_harness,
"no_i18n": self.build_config.no_i18n,
"no_snap": self.build_config.no_snap,
"novfp3": options.novfp3,
"predictable": self.build_config.predictable,
"simulator": utils.UseSimulator(self.build_config.arch),
"simulator_run": simulator_run,
"system": utils.GuessOS(),
"tsan": self.build_config.tsan,
"ubsan_vptr": self.build_config.ubsan_vptr,
}
progress_indicator = progress.IndicatorNotifier()
progress_indicator.Register(
progress.PROGRESS_INDICATORS[options.progress]())
if options.junitout: # pragma: no cover
progress_indicator.Register(progress.JUnitTestProgressIndicator(
options.junitout, options.junittestsuite))
if options.json_test_results:
progress_indicator.Register(progress.JsonTestProgressIndicator(
options.json_test_results,
self.build_config.arch,
self.mode_options.execution_mode,
ctx.random_seed))
if options.flakiness_results: # pragma: no cover
progress_indicator.Register(progress.FlakinessTestProgressIndicator(
options.flakiness_results))
if options.infra_staging:
for s in suites:
s.ReadStatusFile(variables)
s.ReadTestCases(ctx)
return self._run_test_procs(suites, args, options, progress_indicator,
ctx)
all_tests = []
num_tests = 0
for s in suites:
s.ReadStatusFile(variables)
s.ReadTestCases(ctx)
if len(args) > 0:
s.FilterTestCasesByArgs(args)
all_tests += s.tests
# First filtering by status applying the generic rules (tests without
# variants)
if options.warn_unused:
tests = [(t.name, t.variant) for t in s.tests]
s.statusfile.warn_unused_rules(tests, check_variant_rules=False)
s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests)
if options.cat:
verbose.PrintTestSource(s.tests)
continue
variant_gen = s.CreateLegacyVariantsGenerator(self._variants)
variant_tests = [ t.create_variant(v, flags)
for t in s.tests
for v in variant_gen.FilterVariantsByTest(t)
for flags in variant_gen.GetFlagSets(t, v) ]
if options.random_seed_stress_count > 1:
# Duplicate test for random seed stress mode.
def iter_seed_flags():
for _ in range(0, options.random_seed_stress_count):
# Use given random seed for all runs (set by default in
# execution.py) or a new random seed if none is specified.
if options.random_seed:
yield []
else:
yield ["--random-seed=%d" % self._random_seed()]
s.tests = [
t.create_variant(t.variant, flags, 'seed-stress-%d' % n)
for t in variant_tests
for n, flags in enumerate(iter_seed_flags())
]
else:
s.tests = variant_tests
# Second filtering by status applying also the variant-dependent rules.
if options.warn_unused:
tests = [(t.name, t.variant) for t in s.tests]
s.statusfile.warn_unused_rules(tests, check_variant_rules=True)
s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests)
s.tests = self._shard_tests(s.tests, options)
for t in s.tests:
t.cmd = t.get_command(ctx)
num_tests += len(s.tests)
if options.cat:
return 0 # We're done here.
if options.report:
verbose.PrintReport(all_tests)
# Run the tests.
start_time = time.time()
if self.build_config.predictable:
outproc_factory = predictable.get_outproc
else:
outproc_factory = None
runner = execution.Runner(suites, progress_indicator, ctx,
outproc_factory)
exit_code = runner.Run(options.j)
overall_duration = time.time() - start_time
if options.time:
verbose.PrintTestDurations(suites, runner.outputs, overall_duration)
if num_tests == 0:
exit_code = 3
print("Warning: no tests were run!")
if exit_code == 1 and options.json_test_results:
print("Force exit code 0 after failures. Json test results file "
"generated with failure information.")
exit_code = 0
if self.sancov_dir:
# If tests ran with sanitizer coverage, merge coverage files in the end.
try:
print "Merging sancov files."
subprocess.check_call([
sys.executable,
join(self.basedir, "tools", "sanitizers", "sancov_merger.py"),
"--coverage-dir=%s" % self.sancov_dir])
except:
print >> sys.stderr, "Error: Merging sancov files failed."
exit_code = 1
return exit_code
def _shard_tests(self, tests, options):
shard_run, shard_count = self._get_shard_info(options)
if shard_count < 2:
return tests
count = 0
shard = []
for test in tests:
if count % shard_count == shard_run - 1:
shard.append(test)
count += 1
return shard
def _run_test_procs(self, suites, args, options, progress_indicator,
context):
jobs = options.j
print '>>> Running with test processors'
loader = LoadProc()
tests_counter = TestsCounter()
results = ResultsTracker()
indicators = progress_indicator.ToProgressIndicatorProcs()
outproc_factory = None
if self.build_config.predictable:
outproc_factory = predictable.get_outproc
execproc = ExecutionProc(jobs, context, outproc_factory)
procs = [
loader,
NameFilterProc(args) if args else None,
StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
self._create_shard_proc(options),
tests_counter,
VariantProc(self._variants),
StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
] + indicators + [
results,
self._create_rerun_proc(context),
execproc,
]
procs = filter(None, procs)
for i in xrange(0, len(procs) - 1):
procs[i].connect_to(procs[i + 1])
tests = [t for s in suites for t in s.tests]
tests.sort(key=lambda t: t.is_slow, reverse=True)
loader.setup()
loader.load_tests(tests)
print '>>> Running %d base tests' % tests_counter.total
tests_counter.remove_from_chain()
execproc.start()
for indicator in indicators:
indicator.finished()
print '>>> %d tests ran' % results.total
exit_code = 0
if results.failed:
exit_code = 1
if results.remaining:
exit_code = 2
if not results.total:
exit_code = 3
if exit_code == 1 and options.json_test_results:
print("Force exit code 0 after failures. Json test results file "
"generated with failure information.")
exit_code = 0
return exit_code
def _create_rerun_proc(self, ctx):
if not ctx.rerun_failures_count:
return None
return RerunProc(ctx.rerun_failures_count,
ctx.rerun_failures_max)
if __name__ == '__main__':
sys.exit(StandardTestRunner().execute())