#!/usr/bin/env python # # Copyright 2017 the V8 project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from collections import OrderedDict from os.path import join import multiprocessing import os import random import re import shlex import subprocess import sys import time # Adds testrunner to the path hence it has to be imported at the beggining. import base_runner from testrunner.local import execution from testrunner.local import progress from testrunner.local import testsuite from testrunner.local import utils from testrunner.local import verbose from testrunner.local.variants import ALL_VARIANTS from testrunner.objects import context from testrunner.objects import predictable from testrunner.testproc.execution import ExecutionProc from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc from testrunner.testproc.loader import LoadProc from testrunner.testproc.progress import (VerboseProgressIndicator, ResultsTracker, TestsCounter) from testrunner.testproc.rerun import RerunProc from testrunner.testproc.variant import VariantProc TIMEOUT_DEFAULT = 60 # Variants ordered by expected runtime (slowest first). VARIANTS = ["default"] MORE_VARIANTS = [ "stress", "stress_incremental_marking", "nooptimization", "stress_background_compile", "wasm_traps", ] VARIANT_ALIASES = { # The default for developer workstations. "dev": VARIANTS, # Additional variants, run on all bots. "more": MORE_VARIANTS, # Shortcut for the two above ("more" first - it has the longer running tests). "exhaustive": MORE_VARIANTS + VARIANTS, # Additional variants, run on a subset of bots. "extra": ["future", "liftoff", "trusted"], } GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", "--concurrent-recompilation-queue-length=64", "--concurrent-recompilation-delay=500", "--concurrent-recompilation"] RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000", "--stress-compaction-random"] # Double the timeout for these: SLOW_ARCHS = ["arm", "mips", "mipsel", "mips64", "mips64el", "s390", "s390x", "arm64"] PREDICTABLE_WRAPPER = os.path.join( base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py') # Specifies which builders should use the staging test-runner. # Mapping from mastername to list of buildernames. Buildernames can be strings # or compiled regexps which will be matched. BUILDER_WHITELIST_STAGING = { 'client.v8': [ re.compile(r'V8 Linux64.*'), re.compile(r'V8 Mac.*'), re.compile(r'V8 Win.*'), ], 'client.v8.ports': [ re.compile(r'V8 Linux - arm64.*'), ], 'tryserver.v8': [ re.compile(r'v8_linux_arm64.*'), re.compile(r'v8_linux64.*'), re.compile(r'v8_mac.*'), re.compile(r'v8_win.*'), ], } _RE_TYPE = type(re.compile('')) # Specifies which architectures are whitelisted to use the staging test-runner. # List of arch strings, e.g. "x64". ARCH_WHITELIST_STAGING = [ ] class StandardTestRunner(base_runner.BaseTestRunner): def __init__(self, *args, **kwargs): super(StandardTestRunner, self).__init__(*args, **kwargs) self.sancov_dir = None def _get_default_suite_names(self): return ['default'] def _do_execute(self, suites, args, options): if options.swarming: # Swarming doesn't print how isolated commands are called. Lets make # this less cryptic by printing it ourselves. print ' '.join(sys.argv) return self._execute(args, options, suites) def _add_parser_options(self, parser): parser.add_option("--sancov-dir", help="Directory where to collect coverage data") parser.add_option("--cfi-vptr", help="Run tests with UBSAN cfi_vptr option.", default=False, action="store_true") parser.add_option("--novfp3", help="Indicates that V8 was compiled without VFP3" " support", default=False, action="store_true") parser.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") parser.add_option("--slow-tests", help="Regard slow tests (run|skip|dontcare)", default="dontcare") parser.add_option("--pass-fail-tests", help="Regard pass|fail tests (run|skip|dontcare)", default="dontcare") parser.add_option("--gc-stress", help="Switch on GC stress mode", default=False, action="store_true") parser.add_option("--random-gc-stress", help="Switch on random GC stress mode", default=False, action="store_true") parser.add_option("--command-prefix", help="Prepended to each shell command used to run a" " test", default="") parser.add_option("--extra-flags", help="Additional flags to pass to each test command", action="append", default=[]) parser.add_option("--infra-staging", help="Use new test runner features", dest='infra_staging', default=None, action="store_true") parser.add_option("--no-infra-staging", help="Opt out of new test runner features", dest='infra_staging', default=None, action="store_false") parser.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") parser.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") parser.add_option("--no-harness", "--noharness", help="Run without test harness of a given suite", default=False, action="store_true") parser.add_option("--no-presubmit", "--nopresubmit", help='Skip presubmit checks (deprecated)', default=False, dest="no_presubmit", action="store_true") parser.add_option("--no-sorting", "--nosorting", help="Don't sort tests according to duration of last" " run.", default=False, dest="no_sorting", action="store_true") parser.add_option("--no-variants", "--novariants", help="Deprecated. " "Equivalent to passing --variants=default", default=False, dest="no_variants", action="store_true") parser.add_option("--variants", help="Comma-separated list of testing variants;" " default: \"%s\"" % ",".join(VARIANTS)) parser.add_option("--exhaustive-variants", default=False, action="store_true", help="Deprecated. " "Equivalent to passing --variants=exhaustive") parser.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") parser.add_option("--quickcheck", default=False, action="store_true", help=("Quick check mode (skip slow tests)")) parser.add_option("--report", help="Print a summary of the tests to be" " run", default=False, action="store_true") parser.add_option("--json-test-results", help="Path to a file for storing json results.") parser.add_option("--flakiness-results", help="Path to a file for storing flakiness json.") parser.add_option("--rerun-failures-count", help=("Number of times to rerun each failing test case." " Very slow tests will be rerun only once."), default=0, type="int") parser.add_option("--rerun-failures-max", help="Maximum number of failing test cases to rerun.", default=100, type="int") parser.add_option("--dont-skip-slow-simulator-tests", help="Don't skip more slow tests when using a" " simulator.", default=False, action="store_true", dest="dont_skip_simulator_slow_tests") parser.add_option("--swarming", help="Indicates running test driver on swarming.", default=False, action="store_true") parser.add_option("--time", help="Print timing information after running", default=False, action="store_true") parser.add_option("-t", "--timeout", help="Timeout in seconds", default=TIMEOUT_DEFAULT, type="int") parser.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") parser.add_option("--junitout", help="File name of the JUnit output") parser.add_option("--junittestsuite", help="The testsuite name in the JUnit output file", default="v8tests") parser.add_option("--random-seed", default=0, dest="random_seed", help="Default seed for initializing random generator", type=int) parser.add_option("--random-seed-stress-count", default=1, type="int", dest="random_seed_stress_count", help="Number of runs with different random seeds") def _use_staging(self, options): if options.infra_staging is not None: # True or False are used to explicitly opt in or out. return options.infra_staging builder_configs = BUILDER_WHITELIST_STAGING.get(options.mastername, []) for builder_config in builder_configs: if (isinstance(builder_config, _RE_TYPE) and builder_config.match(options.buildername)): return True if builder_config == options.buildername: return True for arch in ARCH_WHITELIST_STAGING: if self.build_config.arch == arch: return True return False def _process_options(self, options): global VARIANTS if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print("sancov-dir %s doesn't exist" % self.sancov_dir) raise base_runner.TestRunnerError() options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = sum(map(shlex.split, options.extra_flags), []) if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append("--invoke-weak-callbacks") options.extra_flags.append("--omit-quit") if options.novfp3: options.extra_flags.append("--noenable-vfp3") if options.no_variants: # pragma: no cover print ("Option --no-variants is deprecated. " "Pass --variants=default instead.") assert not options.variants options.variants = "default" if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print ("Option --exhaustive-variants is deprecated. " "Pass --variants=exhaustive instead.") # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = "exhaustive" if options.quickcheck: assert not options.variants options.variants = "stress,default" options.slow_tests = "skip" options.pass_fail_tests = "skip" if self.build_config.predictable: options.variants = "default" options.extra_flags.append("--predictable") options.extra_flags.append("--verify_predictable") options.extra_flags.append("--no-inline-new") # Add predictable wrapper to command prefix. options.command_prefix = ( [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = "default" if options.j == 0: options.j = multiprocessing.cpu_count() if options.random_seed_stress_count <= 1 and options.random_seed == 0: options.random_seed = self._random_seed() # Use developer defaults if no variant was specified. options.variants = options.variants or "dev" if options.variants == "infra_staging": options.variants = "exhaustive" options.infra_staging = True # Use staging on whitelisted masters/builders. options.infra_staging = self._use_staging(options) # Resolve variant aliases and dedupe. # TODO(machenbach): Don't mutate global variable. Rather pass mutated # version as local variable. VARIANTS = list(set(reduce( list.__add__, (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")), [], ))) if not set(VARIANTS).issubset(ALL_VARIANTS): print "All variants must be in %s" % str(ALL_VARIANTS) raise base_runner.TestRunnerError() def CheckTestMode(name, option): # pragma: no cover if not option in ["run", "skip", "dontcare"]: print "Unknown %s mode %s" % (name, option) raise base_runner.TestRunnerError() CheckTestMode("slow test", options.slow_tests) CheckTestMode("pass|fail test", options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP["bot_default"].remove("intl") base_runner.TEST_MAP["default"].remove("intl") # TODO(machenbach): uncomment after infra side lands. # base_runner.TEST_MAP["d8_default"].remove("intl") def _setup_env(self): super(StandardTestRunner, self)._setup_env() symbolizer_option = self._get_external_symbolizer_option() if self.sancov_dir: os.environ['ASAN_OPTIONS'] = ":".join([ 'coverage=1', 'coverage_dir=%s' % self.sancov_dir, symbolizer_option, "allow_user_segv_handler=1", ]) def _random_seed(self): seed = 0 while not seed: seed = random.SystemRandom().randint(-2147483648, 2147483647) return seed def _execute(self, args, options, suites): print(">>> Running tests for %s.%s" % (self.build_config.arch, self.mode_name)) # Populate context object. # Simulators are slow, therefore allow a longer timeout. if self.build_config.arch in SLOW_ARCHS: options.timeout *= 2 options.timeout *= self.mode_options.timeout_scalefactor if self.build_config.predictable: # Predictable mode is slower. options.timeout *= 2 ctx = context.Context(self.build_config.arch, self.mode_options.execution_mode, self.outdir, self.mode_options.flags, options.verbose, options.timeout, options.isolates, options.command_prefix, options.extra_flags, self.build_config.no_i18n, options.random_seed, options.no_sorting, options.rerun_failures_count, options.rerun_failures_max, options.no_harness, use_perf_data=not options.swarming, sancov_dir=self.sancov_dir, infra_staging=options.infra_staging) # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from # target_arch != v8_target_arch in the dumped build config. simulator_run = ( not options.dont_skip_simulator_slow_tests and self.build_config.arch in [ 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', 'ppc64', 's390', 's390x'] and bool(base_runner.ARCH_GUESS) and self.build_config.arch != base_runner.ARCH_GUESS) # Find available test suites and read test cases from them. variables = { "arch": self.build_config.arch, "asan": self.build_config.asan, "byteorder": sys.byteorder, "dcheck_always_on": self.build_config.dcheck_always_on, "deopt_fuzzer": False, "gc_fuzzer": False, "gc_stress": options.gc_stress or options.random_gc_stress, "gcov_coverage": self.build_config.gcov_coverage, "isolates": options.isolates, "mode": self.mode_options.status_mode, "msan": self.build_config.msan, "no_harness": options.no_harness, "no_i18n": self.build_config.no_i18n, "no_snap": self.build_config.no_snap, "novfp3": options.novfp3, "predictable": self.build_config.predictable, "simulator": utils.UseSimulator(self.build_config.arch), "simulator_run": simulator_run, "system": utils.GuessOS(), "tsan": self.build_config.tsan, "ubsan_vptr": self.build_config.ubsan_vptr, } progress_indicator = progress.IndicatorNotifier() progress_indicator.Register( progress.PROGRESS_INDICATORS[options.progress]()) if options.junitout: # pragma: no cover progress_indicator.Register(progress.JUnitTestProgressIndicator( options.junitout, options.junittestsuite)) if options.json_test_results: progress_indicator.Register(progress.JsonTestProgressIndicator( options.json_test_results, self.build_config.arch, self.mode_options.execution_mode, ctx.random_seed)) if options.flakiness_results: # pragma: no cover progress_indicator.Register(progress.FlakinessTestProgressIndicator( options.flakiness_results)) if options.infra_staging: for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) return self._run_test_procs(suites, args, options, progress_indicator, ctx) all_tests = [] num_tests = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests # First filtering by status applying the generic rules (tests without # variants) if options.warn_unused: tests = [(t.name, t.variant) for t in s.tests] s.statusfile.warn_unused_rules(tests, check_variant_rules=False) s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) if options.cat: verbose.PrintTestSource(s.tests) continue variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS) variant_tests = [ t.create_variant(v, flags) for t in s.tests for v in variant_gen.FilterVariantsByTest(t) for flags in variant_gen.GetFlagSets(t, v) ] if options.random_seed_stress_count > 1: # Duplicate test for random seed stress mode. def iter_seed_flags(): for _ in range(0, options.random_seed_stress_count): # Use given random seed for all runs (set by default in # execution.py) or a new random seed if none is specified. if options.random_seed: yield [] else: yield ["--random-seed=%d" % self._random_seed()] s.tests = [ t.create_variant(t.variant, flags, 'seed-stress-%d' % n) for t in variant_tests for n, flags in enumerate(iter_seed_flags()) ] else: s.tests = variant_tests # Second filtering by status applying also the variant-dependent rules. if options.warn_unused: tests = [(t.name, t.variant) for t in s.tests] s.statusfile.warn_unused_rules(tests, check_variant_rules=True) s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) s.tests = self._shard_tests(s.tests, options) for t in s.tests: t.cmd = t.get_command(ctx) num_tests += len(s.tests) if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) # Run the tests. start_time = time.time() if self.build_config.predictable: outproc_factory = predictable.get_outproc else: outproc_factory = None runner = execution.Runner(suites, progress_indicator, ctx, outproc_factory) exit_code = runner.Run(options.j) overall_duration = time.time() - start_time if options.time: verbose.PrintTestDurations(suites, runner.outputs, overall_duration) if num_tests == 0: exit_code = 3 print("Warning: no tests were run!") if exit_code == 1 and options.json_test_results: print("Force exit code 0 after failures. Json test results file " "generated with failure information.") exit_code = 0 if self.sancov_dir: # If tests ran with sanitizer coverage, merge coverage files in the end. try: print "Merging sancov files." subprocess.check_call([ sys.executable, join(self.basedir, "tools", "sanitizers", "sancov_merger.py"), "--coverage-dir=%s" % self.sancov_dir]) except: print >> sys.stderr, "Error: Merging sancov files failed." exit_code = 1 return exit_code def _shard_tests(self, tests, options): shard_run, shard_count = self._get_shard_info(options) if shard_count < 2: return tests count = 0 shard = [] for test in tests: if count % shard_count == shard_run - 1: shard.append(test) count += 1 return shard def _run_test_procs(self, suites, args, options, progress_indicator, context): jobs = options.j print '>>> Running with test processors' loader = LoadProc() tests_counter = TestsCounter() results = ResultsTracker() indicators = progress_indicator.ToProgressIndicatorProcs() execproc = ExecutionProc(jobs, context) procs = [ loader, NameFilterProc(args) if args else None, StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), self._create_shard_proc(options), tests_counter, VariantProc(VARIANTS), StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), ] + indicators + [ results, self._create_rerun_proc(context), execproc, ] procs = filter(None, procs) for i in xrange(0, len(procs) - 1): procs[i].connect_to(procs[i + 1]) tests = [t for s in suites for t in s.tests] tests.sort(key=lambda t: t.is_slow, reverse=True) loader.setup() loader.load_tests(tests) print '>>> Running %d base tests' % tests_counter.total tests_counter.remove_from_chain() execproc.start() for indicator in indicators: indicator.finished() print '>>> %d tests ran' % results.total exit_code = 0 if results.failed: exit_code = 1 if results.remaining: exit_code = 2 if not results.total: exit_code = 3 if exit_code == 1 and options.json_test_results: print("Force exit code 0 after failures. Json test results file " "generated with failure information.") exit_code = 0 return exit_code def _create_rerun_proc(self, ctx): if not ctx.rerun_failures_count: return None return RerunProc(ctx.rerun_failures_count, ctx.rerun_failures_max) if __name__ == '__main__': sys.exit(StandardTestRunner().execute())