#!/usr/bin/env python # # Copyright 2017 the V8 project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # for py2/py3 compatibility from __future__ import print_function from functools import reduce import datetime import json import os import sys import tempfile # Adds testrunner to the path hence it has to be imported at the beggining. import base_runner from testrunner.local import utils from testrunner.local.variants import ALL_VARIANTS from testrunner.objects import predictable from testrunner.testproc.execution import ExecutionProc from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc from testrunner.testproc.loader import LoadProc from testrunner.testproc.seed import SeedProc from testrunner.testproc.variant import VariantProc ARCH_GUESS = utils.DefaultArch() VARIANTS = ['default'] MORE_VARIANTS = [ 'jitless', 'stress', 'stress_js_bg_compile_wasm_code_gc', 'stress_incremental_marking', ] VARIANT_ALIASES = { # The default for developer workstations. 'dev': VARIANTS, # Additional variants, run on all bots. 'more': MORE_VARIANTS, # Shortcut for the two above ('more' first - it has the longer running tests) 'exhaustive': MORE_VARIANTS + VARIANTS, # Additional variants, run on a subset of bots. 'extra': ['nooptimization', 'future', 'no_wasm_traps', 'turboprop', 'instruction_scheduling'], } # Extra flags passed to all tests using the standard test runner. EXTRA_DEFAULT_FLAGS = ['--testing-d8-test-runner'] GC_STRESS_FLAGS = ['--gc-interval=500', '--stress-compaction', '--concurrent-recompilation-queue-length=64', '--concurrent-recompilation-delay=500', '--concurrent-recompilation', '--stress-flush-bytecode', '--wasm-code-gc', '--stress-wasm-code-gc'] RANDOM_GC_STRESS_FLAGS = ['--random-gc-interval=5000', '--stress-compaction-random'] PREDICTABLE_WRAPPER = os.path.join( base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py') class StandardTestRunner(base_runner.BaseTestRunner): def __init__(self, *args, **kwargs): super(StandardTestRunner, self).__init__(*args, **kwargs) self.sancov_dir = None self._variants = None @property def framework_name(self): return 'standard_runner' def _get_default_suite_names(self): return ['default'] def _add_parser_options(self, parser): parser.add_option('--novfp3', help='Indicates that V8 was compiled without VFP3' ' support', default=False, action='store_true') # Variants parser.add_option('--no-variants', '--novariants', help='Deprecated. ' 'Equivalent to passing --variants=default', default=False, dest='no_variants', action='store_true') parser.add_option('--variants', help='Comma-separated list of testing variants;' ' default: "%s"' % ','.join(VARIANTS)) parser.add_option('--exhaustive-variants', default=False, action='store_true', help='Deprecated. ' 'Equivalent to passing --variants=exhaustive') # Filters parser.add_option('--slow-tests', default='dontcare', help='Regard slow tests (run|skip|dontcare)') parser.add_option('--pass-fail-tests', default='dontcare', help='Regard pass|fail tests (run|skip|dontcare)') parser.add_option('--quickcheck', default=False, action='store_true', help=('Quick check mode (skip slow tests)')) parser.add_option('--dont-skip-slow-simulator-tests', help='Don\'t skip more slow tests when using a' ' simulator.', default=False, action='store_true', dest='dont_skip_simulator_slow_tests') # Stress modes parser.add_option('--gc-stress', help='Switch on GC stress mode', default=False, action='store_true') parser.add_option('--random-gc-stress', help='Switch on random GC stress mode', default=False, action='store_true') parser.add_option('--random-seed-stress-count', default=1, type='int', dest='random_seed_stress_count', help='Number of runs with different random seeds. Only ' 'with test processors: 0 means infinite ' 'generation.') # Extra features. parser.add_option('--time', help='Print timing information after running', default=False, action='store_true') # Noop parser.add_option('--cfi-vptr', help='Run tests with UBSAN cfi_vptr option.', default=False, action='store_true') parser.add_option('--infra-staging', help='Use new test runner features', dest='infra_staging', default=None, action='store_true') parser.add_option('--no-infra-staging', help='Opt out of new test runner features', dest='infra_staging', default=None, action='store_false') parser.add_option('--no-sorting', '--nosorting', help='Don\'t sort tests according to duration of last' ' run.', default=False, dest='no_sorting', action='store_true') parser.add_option('--no-presubmit', '--nopresubmit', help='Skip presubmit checks (deprecated)', default=False, dest='no_presubmit', action='store_true') # Unimplemented for test processors parser.add_option('--sancov-dir', help='Directory where to collect coverage data') parser.add_option('--cat', help='Print the source of the tests', default=False, action='store_true') parser.add_option('--flakiness-results', help='Path to a file for storing flakiness json.') parser.add_option('--warn-unused', help='Report unused rules', default=False, action='store_true') parser.add_option('--report', default=False, action='store_true', help='Print a summary of the tests to be run') def _process_options(self, options): if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print('sancov-dir %s doesn\'t exist' % self.sancov_dir) raise base_runner.TestRunnerError() if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append('--invoke-weak-callbacks') if options.novfp3: options.extra_flags.append('--noenable-vfp3') if options.no_variants: # pragma: no cover print ('Option --no-variants is deprecated. ' 'Pass --variants=default instead.') assert not options.variants options.variants = 'default' if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print ('Option --exhaustive-variants is deprecated. ' 'Pass --variants=exhaustive instead.') # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = 'exhaustive' if options.quickcheck: assert not options.variants options.variants = 'stress,default' options.slow_tests = 'skip' options.pass_fail_tests = 'skip' if self.build_config.predictable: options.variants = 'default' options.extra_flags.append('--predictable') options.extra_flags.append('--verify-predictable') options.extra_flags.append('--no-inline-new') # Add predictable wrapper to command prefix. options.command_prefix = ( [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = 'default' if options.variants == 'infra_staging': options.variants = 'exhaustive' self._variants = self._parse_variants(options.variants) def CheckTestMode(name, option): # pragma: no cover if option not in ['run', 'skip', 'dontcare']: print('Unknown %s mode %s' % (name, option)) raise base_runner.TestRunnerError() CheckTestMode('slow test', options.slow_tests) CheckTestMode('pass|fail test', options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP['bot_default'].remove('intl') base_runner.TEST_MAP['default'].remove('intl') # TODO(machenbach): uncomment after infra side lands. # base_runner.TEST_MAP['d8_default'].remove('intl') if options.time and not options.json_test_results: # We retrieve the slowest tests from the JSON output file, so create # a temporary output file (which will automatically get deleted on exit) # if the user didn't specify one. self._temporary_json_output_file = tempfile.NamedTemporaryFile( prefix="v8-test-runner-") options.json_test_results = self._temporary_json_output_file.name def _runner_flags(self): return EXTRA_DEFAULT_FLAGS def _parse_variants(self, aliases_str): # Use developer defaults if no variant was specified. aliases_str = aliases_str or 'dev' aliases = aliases_str.split(',') user_variants = set(reduce( list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases])) result = [v for v in ALL_VARIANTS if v in user_variants] if len(result) == len(user_variants): return result for v in user_variants: if v not in ALL_VARIANTS: print('Unknown variant: %s' % v) print(' Available variants: %s' % ALL_VARIANTS) print(' Available variant aliases: %s' % VARIANT_ALIASES.keys()); raise base_runner.TestRunnerError() assert False, 'Unreachable' def _setup_env(self): super(StandardTestRunner, self)._setup_env() symbolizer_option = self._get_external_symbolizer_option() if self.sancov_dir: os.environ['ASAN_OPTIONS'] = ':'.join([ 'coverage=1', 'coverage_dir=%s' % self.sancov_dir, symbolizer_option, 'allow_user_segv_handler=1', ]) def _get_statusfile_variables(self, options): variables = ( super(StandardTestRunner, self)._get_statusfile_variables(options)) simulator_run = ( not options.dont_skip_simulator_slow_tests and self.build_config.arch in [ 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', 'ppc64', 's390', 's390x'] and bool(ARCH_GUESS) and self.build_config.arch != ARCH_GUESS) variables.update({ 'gc_stress': options.gc_stress or options.random_gc_stress, 'gc_fuzzer': options.random_gc_stress, 'novfp3': options.novfp3, 'simulator_run': simulator_run, }) return variables def _do_execute(self, tests, args, options): jobs = options.j print('>>> Running with test processors') loader = LoadProc(tests) results = self._create_result_tracker(options) indicators = self._create_progress_indicators( tests.test_count_estimate, options) outproc_factory = None if self.build_config.predictable: outproc_factory = predictable.get_outproc execproc = ExecutionProc(jobs, outproc_factory) sigproc = self._create_signal_proc() procs = [ loader, NameFilterProc(args) if args else None, StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), VariantProc(self._variants), StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), self._create_predictable_filter(), self._create_shard_proc(options), self._create_seed_proc(options), sigproc, ] + indicators + [ results, self._create_timeout_proc(options), self._create_rerun_proc(options), execproc, ] self._prepare_procs(procs) loader.load_initial_tests(initial_batch_size=options.j * 2) # This starts up worker processes and blocks until all tests are # processed. execproc.run() for indicator in indicators: indicator.finished() if tests.test_count_estimate: percentage = float(results.total) / tests.test_count_estimate * 100 else: percentage = 0 print (('>>> %d base tests produced %d (%d%s)' ' non-filtered tests') % ( tests.test_count_estimate, results.total, percentage, '%')) print('>>> %d tests ran' % (results.total - results.remaining)) exit_code = utils.EXIT_CODE_PASS if results.failed: exit_code = utils.EXIT_CODE_FAILURES if not results.total: exit_code = utils.EXIT_CODE_NO_TESTS if options.time: self._print_durations(options) # Indicate if a SIGINT or SIGTERM happened. return max(exit_code, sigproc.exit_code) def _print_durations(self, options): def format_duration(duration_in_seconds): duration = datetime.timedelta(seconds=duration_in_seconds) time = (datetime.datetime.min + duration).time() return time.strftime('%M:%S:') + '%03i' % int(time.microsecond / 1000) def _duration_results_text(test): return [ 'Test: %s' % test['name'], 'Flags: %s' % ' '.join(test['flags']), 'Command: %s' % test['command'], 'Duration: %s' % format_duration(test['duration']), ] assert os.path.exists(options.json_test_results) complete_results = [] with open(options.json_test_results, "r") as f: complete_results = json.loads(f.read()) output = complete_results[0] lines = [] for test in output['slowest_tests']: suffix = '' if test.get('marked_slow') is False: suffix = ' *' lines.append( '%s %s%s' % (format_duration(test['duration']), test['name'], suffix)) # Slowest tests duration details. lines.extend(['', 'Details:', '']) for test in output['slowest_tests']: lines.extend(_duration_results_text(test)) print("\n".join(lines)) def _create_predictable_filter(self): if not self.build_config.predictable: return None return predictable.PredictableFilterProc() def _create_seed_proc(self, options): if options.random_seed_stress_count == 1: return None return SeedProc(options.random_seed_stress_count, options.random_seed, options.j * 4) if __name__ == '__main__': sys.exit(StandardTestRunner().execute())