From 5b8702548f8468ac2f508af9c3ac6b8be6ebbc46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Majewski?= Date: Thu, 26 Oct 2017 09:43:03 +0000 Subject: [PATCH] Revert "Reland "Reuse arch/mode discovery in deopt fuzzer"" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 5442e8b22453fa39b3119e2226d676aaa278e43e. Reason for revert: https://crbug.com/v8/7006 Original change's description: > Reland "Reuse arch/mode discovery in deopt fuzzer" > > This is a reland of a24c7c9a52f593e45f115c0480dc94f341fa5d52 > Original change's description: > > Reuse arch/mode discovery in deopt fuzzer > > > > Bug: v8:6917 > > Change-Id: I1b7169c8702c8649812b17579d38d64de676ed60 > > Reviewed-on: https://chromium-review.googlesource.com/723420 > > Commit-Queue: Michał Majewski > > Reviewed-by: Michael Achenbach > > Cr-Commit-Position: refs/heads/master@{#48838} > > Bug: v8:6917 > Change-Id: I03b2c288257d44c2df9d0fa6cf4750d1c5719d59 > Reviewed-on: https://chromium-review.googlesource.com/735719 > Commit-Queue: Michał Majewski > Reviewed-by: Michael Achenbach > Cr-Commit-Position: refs/heads/master@{#48916} TBR=machenbach@chromium.org,majeski@google.com Change-Id: Ibbc16bb7115c8e8b93bd8f39065742d3c2d7739f No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: v8:6917, v8:7006 Reviewed-on: https://chromium-review.googlesource.com/737807 Commit-Queue: Michał Majewski Reviewed-by: Michael Achenbach Cr-Commit-Position: refs/heads/master@{#48960} --- tools/testrunner/base_runner.py | 245 ++++++++++++++++------------ tools/testrunner/deopt_fuzzer.py | 195 ++++++++++++++++------ tools/testrunner/standard_runner.py | 151 +++++++++++------ 3 files changed, 383 insertions(+), 208 deletions(-) diff --git a/tools/testrunner/base_runner.py b/tools/testrunner/base_runner.py index 2e6fc35eab..7ed7c033e5 100644 --- a/tools/testrunner/base_runner.py +++ b/tools/testrunner/base_runner.py @@ -73,6 +73,26 @@ MODES = { }, } +SUPPORTED_ARCHS = [ + "android_arm", + "android_arm64", + "android_ia32", + "android_x64", + "arm", + "ia32", + "mips", + "mipsel", + "mips64", + "mips64el", + "s390", + "s390x", + "ppc", + "ppc64", + "x64", + "x32", + "arm64", +] + # Map of test name synonyms to lists of test suites. Should be ordered by # expected runtimes (suites with slow test cases first). These groups are # invoked in separate steps on the bots. @@ -125,58 +145,37 @@ class TestRunnerError(Exception): pass -class BuildConfig(object): - def __init__(self, build_config): - # In V8 land, GN's x86 is called ia32. - if build_config['v8_target_cpu'] == 'x86': - self.arch = 'ia32' - else: - self.arch = build_config['v8_target_cpu'] - - self.mode = 'debug' if build_config['is_debug'] else 'release' - self.asan = build_config['is_asan'] - self.cfi_vptr = build_config['is_cfi'] - self.dcheck_always_on = build_config['dcheck_always_on'] - self.gcov_coverage = build_config['is_gcov_coverage'] - self.msan = build_config['is_msan'] - self.no_i18n = not build_config['v8_enable_i18n_support'] - self.no_snap = not build_config['v8_use_snapshot'] - self.predictable = build_config['v8_enable_verify_predictable'] - self.tsan = build_config['is_tsan'] - self.ubsan_vptr = build_config['is_ubsan_vptr'] - - class BaseTestRunner(object): def __init__(self): self.outdir = None - self.shell_dir = None - self.build_config = None + + self.arch = None + self.mode = None + + self.auto_detect = None def execute(self): try: - parser = self._create_parser() - options, args = self._parse_args(parser) - - self._load_build_config(options) - - try: - self._process_default_options(options) - self._process_options(options) - except TestRunnerError: - parser.print_help() - raise - + options, args = self._parse_args() return self._do_execute(options, args) except TestRunnerError: return 1 - def _create_parser(self): + def _parse_args(self): parser = optparse.OptionParser() parser.usage = '%prog [options] [tests]' parser.description = """TESTS: %s""" % (TEST_MAP["default"]) self._add_parser_default_options(parser) self._add_parser_options(parser) - return parser + options, args = parser.parse_args() + try: + self._process_default_options(options) + self._process_options(options) + except TestRunnerError: + parser.print_help() + raise + + return options, args def _add_parser_default_options(self, parser): parser.add_option("--gn", help="Scan out.gn for the last built" @@ -188,53 +187,48 @@ class BaseTestRunner(object): help="Adapt to path structure used on buildbots", default=False, action="store_true") parser.add_option("--arch", - help="The architecture to run tests for: %s") + help=("The architecture to run tests for, " + "'auto' or 'native' for auto-detect: %s" % + SUPPORTED_ARCHS)) parser.add_option("-m", "--mode", help="The test mode in which to run (uppercase for ninja" " and buildbot builds): %s" % MODES.keys()) - parser.add_option("--shell", help="DEPRECATED! use --shell-dir", - default="") - parser.add_option("--shell-dir", help="Directory containing executables", - default="") def _add_parser_options(self, parser): pass - def _parse_args(self, parser): - options, args = parser.parse_args() + def _process_default_options(self, options): + # Try to autodetect configuration based on the build if GN was used. + # This can't be ovveridden by cmd-line arguments. + if options.gn: + outdir = self._get_gn_outdir() + else: + outdir = options.outdir - if any(map(lambda v: v and ',' in v, - [options.arch, options.mode])): - print 'Multiple arch/mode are deprecated' + self.auto_detect = self._read_build_config(outdir, options) + if not self.auto_detect: + if any(map(lambda v: v and ',' in v, + [options.arch, options.mode])): + print 'Multiple arch/mode are deprecated' + raise TestRunnerError() + + self.outdir = outdir + if not options.arch or not options.mode: + print('Autodetect mode is not available and therefore ' + '--arch and --mode options are required') + raise TestRunnerError() + self.arch = options.arch + self.mode = options.mode + + if not self._buildbot_to_v8_mode(self.mode) in MODES: + print "Unknown mode %s" % self.mode raise TestRunnerError() - return options, args - - def _load_build_config(self, options): - for outdir in self._possible_outdirs(options): - try: - self.build_config = self._do_load_build_config( - outdir, options.mode, options.buildbot) - except TestRunnerError: - pass - - if not self.build_config: - print 'Failed to load build config' - raise TestRunnerError - - print 'Build found: %s' % self.outdir - - # Returns possible build paths in order: gn, outdir, outdir/arch.mode - def _possible_outdirs(self, options): - if options.gn: - yield self._get_gn_outdir() - return - - yield options.outdir - if options.arch and options.mode: - yield os.path.join(options.outdir, - '%s.%s' % (options.arch, options.mode)) - return + if self.arch in ["auto", "native"]: + self.arch = ARCH_GUESS + if not self.arch in SUPPORTED_ARCHS: + print "Unknown architecture %s" % self.arch + raise TestRunnerError() def _get_gn_outdir(self): gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN) @@ -251,21 +245,25 @@ class BaseTestRunner(object): print(">>> Latest GN build found: %s" % latest_config) return os.path.join(DEFAULT_OUT_GN, latest_config) - def _do_load_build_config(self, outdir, mode, is_buildbot): - if is_buildbot: + # Auto-detect test configurations based on the build (GN only). + # sets: + # - arch + # - mode + # - outdir + def _read_build_config(self, outdir, options): + if options.buildbot: build_config_path = os.path.join( - BASE_DIR, outdir, mode, "v8_build_config.json") + BASE_DIR, outdir, options.mode, "v8_build_config.json") else: build_config_path = os.path.join( BASE_DIR, outdir, "v8_build_config.json") + if not os.path.exists(build_config_path): - print('Missing build config in the output directory (%s)' % - build_config_path) - raise TestRunnerError() + return False with open(build_config_path) as f: try: - build_config_json = json.load(f) + build_config = json.load(f) except Exception: print("%s exists but contains invalid json. Is your build up-to-date?" % build_config_path) @@ -276,24 +274,71 @@ class BaseTestRunner(object): # This ensures that we'll also take the build products from there. self.outdir = os.path.dirname(build_config_path) - return BuildConfig(build_config_json) + # In V8 land, GN's x86 is called ia32. + if build_config["v8_target_cpu"] == "x86": + build_config["v8_target_cpu"] = "ia32" - def _process_default_options(self, options): - # We don't use the mode for more path-magic. - # Therefore transform the buildbot mode here to fix build_config value. - if options.buildbot and options.mode: + if options.mode: + # In auto-detect mode we don't use the mode for more path-magic. + # Therefore transform the buildbot mode here to fit to the GN build + # config. options.mode = self._buildbot_to_v8_mode(options.mode) - def check_consistency(name, option_val, cfg_val): - if option_val and option_val != cfg_val: - print('Attempted to set %s to %s while build config is %s.' % ( - name, option_val, cfg_val)) + # TODO(majeski): merge next two loops and put everything in self. + + # Get options from the build config. Sanity check that we're not trying to + # use inconsistent options. + for param, value in ( + ('arch', build_config["v8_target_cpu"]), + ('mode', 'debug' if build_config["is_debug"] else 'release'), + ): + cmd_line_value = getattr(options, param) + if (cmd_line_value not in [None, True, False] and + cmd_line_value != value): + # TODO(machenbach): This is for string options only. Requires + # options to not have default values. We should make this more + # modular and implement it in our own version of the option parser. + print("Attempted to set %s to %s, while build is %s." % + (param, cmd_line_value, value)) raise TestRunnerError() + if cmd_line_value == True and value == False: + print("Attempted to turn on %s, but it's not available." % param) + raise TestRunnerError() + if cmd_line_value != value: + print(">>> Auto-detected %s=%s" % (param, value)) + setattr(self, param, value) - check_consistency('arch', options.arch, self.build_config.arch) - check_consistency('mode', options.mode, self.build_config.mode) + # Update options based on the build config. Sanity check that we're not + # trying to use inconsistent options. + for param, value in ( + ('asan', build_config["is_asan"]), + ('cfi_vptr', build_config["is_cfi"]), + ('dcheck_always_on', build_config["dcheck_always_on"]), + ('gcov_coverage', build_config["is_gcov_coverage"]), + ('msan', build_config["is_msan"]), + ('no_i18n', not build_config["v8_enable_i18n_support"]), + ('no_snap', not build_config["v8_use_snapshot"]), + ('predictable', build_config["v8_enable_verify_predictable"]), + ('tsan', build_config["is_tsan"]), + ('ubsan_vptr', build_config["is_ubsan_vptr"]), + ): + cmd_line_value = getattr(options, param) + if (cmd_line_value not in [None, True, False] and + cmd_line_value != value): + # TODO(machenbach): This is for string options only. Requires + # options to not have default values. We should make this more + # modular and implement it in our own version of the option parser. + print("Attempted to set %s to %s, while build is %s." % + (param, cmd_line_value, value)) + raise TestRunnerError() + if cmd_line_value == True and value == False: + print("Attempted to turn on %s, but it's not available." % param) + raise TestRunnerError() + if cmd_line_value != value: + print(">>> Auto-detected %s=%s" % (param, value)) + setattr(options, param, value) - self._set_shell_dir(options) + return True def _buildbot_to_v8_mode(self, config): """Convert buildbot build configs to configs understood by the v8 runner. @@ -304,20 +349,6 @@ class BaseTestRunner(object): mode = config[:-4] if config.endswith('_x64') else config return mode.lower() - def _set_shell_dir(self, options): - self.shell_dir = options.shell_dir - if not self.shell_dir: - # TODO(majeski): drop this option - if options.shell: - print "Warning: --shell is deprecated, use --shell-dir instead." - self.shell_dir = os.path.dirname(options.shell) - else: - # If an output dir with a build was passed, test directly in that - # directory. - self.shell_dir = os.path.join(BASE_DIR, self.outdir) - if not os.path.exists(self.shell_dir): - raise Exception('Could not find shell_dir: "%s"' % self.shell_dir) - def _process_options(self, options): pass diff --git a/tools/testrunner/deopt_fuzzer.py b/tools/testrunner/deopt_fuzzer.py index 9f1711299a..ef778c632d 100755 --- a/tools/testrunner/deopt_fuzzer.py +++ b/tools/testrunner/deopt_fuzzer.py @@ -9,9 +9,11 @@ from os.path import join import json import math import multiprocessing +import optparse import os import random import shlex +import subprocess import sys import time @@ -26,13 +28,38 @@ from testrunner.local import verbose from testrunner.objects import context +# Base dir of the v8 checkout to be used as cwd. +BASE_DIR = ( + os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.abspath(__file__))))) + +ARCH_GUESS = utils.DefaultArch() DEFAULT_TESTS = ["mjsunit", "webkit"] TIMEOUT_DEFAULT = 60 TIMEOUT_SCALEFACTOR = {"debug" : 4, "release" : 1 } +MODE_FLAGS = { + "debug" : ["--nohard-abort", "--enable-slow-asserts", + "--verify-heap", "--noconcurrent-recompilation"], + "release" : ["--nohard-abort", "--noconcurrent-recompilation"]} + +SUPPORTED_ARCHS = ["android_arm", + "android_ia32", + "arm", + "ia32", + "ppc", + "ppc64", + "s390", + "s390x", + "mipsel", + "x64"] # Double the timeout for these: -SLOW_ARCHS = ["arm", +SLOW_ARCHS = ["android_arm", + "android_ia32", + "arm", "mipsel"] MAX_DEOPT = 1000000000 DISTRIBUTION_MODES = ["smooth", "random"] @@ -105,57 +132,100 @@ class DeoptFuzzer(base_runner.BaseTestRunner): options.distribution_factor2) - def _add_parser_options(self, parser): - parser.add_option("--command-prefix", + def _build_options(self): + result = optparse.OptionParser() + result.add_option("--arch", + help=("The architecture to run tests for, " + "'auto' or 'native' for auto-detect"), + default="ia32,x64,arm") + result.add_option("--arch-and-mode", + help="Architecture and mode in the format 'arch.mode'", + default=None) + result.add_option("--asan", + help="Regard test expectations for ASAN", + default=False, action="store_true") + result.add_option("--buildbot", + help="Adapt to path structure used on buildbots", + default=False, action="store_true") + result.add_option("--dcheck-always-on", + help="Indicates that V8 was compiled with DCHECKs" + " enabled", + default=False, action="store_true") + result.add_option("--command-prefix", help="Prepended to each shell command used to run a test", default="") - parser.add_option("--coverage", help=("Exponential test coverage " + result.add_option("--coverage", help=("Exponential test coverage " "(range 0.0, 1.0) - 0.0: one test, 1.0 all tests (slow)"), default=0.4, type="float") - parser.add_option("--coverage-lift", help=("Lifts test coverage for tests " + result.add_option("--coverage-lift", help=("Lifts test coverage for tests " "with a small number of deopt points (range 0, inf)"), default=20, type="int") - parser.add_option("--distribution-factor1", help=("Factor of the first " + result.add_option("--distribution-factor1", help=("Factor of the first " "derivation of the distribution function"), default=2.0, type="float") - parser.add_option("--distribution-factor2", help=("Factor of the second " + result.add_option("--distribution-factor2", help=("Factor of the second " "derivation of the distribution function"), default=0.7, type="float") - parser.add_option("--distribution-mode", help=("How to select deopt points " + result.add_option("--distribution-mode", help=("How to select deopt points " "for a given test (smooth|random)"), default="smooth") - parser.add_option("--dump-results-file", help=("Dump maximum number of " + result.add_option("--dump-results-file", help=("Dump maximum number of " "deopt points per test to a file")) - parser.add_option("--extra-flags", + result.add_option("--extra-flags", help="Additional flags to pass to each test command", default="") - parser.add_option("--isolates", help="Whether to test isolates", + result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") - parser.add_option("-j", help="The number of parallel tasks to run", + result.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") - parser.add_option("-p", "--progress", + result.add_option("-m", "--mode", + help="The test modes in which to run (comma-separated)", + default="release,debug") + result.add_option("--outdir", help="Base directory with compile output", + default="out") + result.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") - parser.add_option("--shard-count", + result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") - parser.add_option("--shard-run", + result.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") - parser.add_option("--seed", help="The seed for the random distribution", + result.add_option("--shell-dir", help="Directory containing executables", + default="") + result.add_option("--seed", help="The seed for the random distribution", type="int") - parser.add_option("-t", "--timeout", help="Timeout in seconds", + result.add_option("-t", "--timeout", help="Timeout in seconds", default= -1, type="int") - parser.add_option("-v", "--verbose", help="Verbose output", + result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") - parser.add_option("--random-seed", default=0, dest="random_seed", + result.add_option("--random-seed", default=0, dest="random_seed", help="Default seed for initializing random generator") - return parser + return result def _process_options(self, options): + # Architecture and mode related stuff. + if options.arch_and_mode: + tokens = options.arch_and_mode.split(".") + options.arch = tokens[0] + options.mode = tokens[1] + options.mode = options.mode.split(",") + for mode in options.mode: + if not mode.lower() in ["debug", "release"]: + print "Unknown mode %s" % mode + return False + if options.arch in ["auto", "native"]: + options.arch = ARCH_GUESS + options.arch = options.arch.split(",") + for arch in options.arch: + if not arch in SUPPORTED_ARCHS: + print "Unknown architecture %s" % arch + return False + # Special processing of other options, sorted alphabetically. options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = shlex.split(options.extra_flags) @@ -200,11 +270,20 @@ class DeoptFuzzer(base_runner.BaseTestRunner): count += 1 return shard - def _do_execute(self, options, args): + # TODO(majeski): reuse baseclass code + def execute(self): # Use the v8 root as cwd as some test cases use "load" with relative paths. - os.chdir(base_runner.BASE_DIR) + os.chdir(BASE_DIR) - suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test")) + parser = self._build_options() + (options, args) = parser.parse_args() + if not self._process_options(options): + parser.print_help() + return 1 + + exit_code = 0 + + suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) if len(args) == 0: suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] @@ -219,14 +298,18 @@ class DeoptFuzzer(base_runner.BaseTestRunner): suites = [] for root in suite_paths: suite = testsuite.TestSuite.LoadTestSuite( - os.path.join(base_runner.BASE_DIR, "test", root)) + os.path.join(BASE_DIR, "test", root)) if suite: suites.append(suite) - try: - return self._execute(args, options, suites) - except KeyboardInterrupt: - return 2 + for mode in options.mode: + for arch in options.arch: + try: + code = self._execute(arch, mode, args, options, suites, BASE_DIR) + exit_code = exit_code or code + except KeyboardInterrupt: + return 2 + return exit_code def _calculate_n_tests(self, m, options): @@ -240,26 +323,33 @@ class DeoptFuzzer(base_runner.BaseTestRunner): return int(math.pow(m, (m * c + l) / (m + l))) - def _execute(self, args, options, suites): - print(">>> Running tests for %s.%s" % (self.build_config.arch, - self.build_config.mode)) + def _execute(self, arch, mode, args, options, suites, workspace): + print(">>> Running tests for %s.%s" % (arch, mode)) dist = self._distribution(options) + shell_dir = options.shell_dir + if not shell_dir: + if options.buildbot: + shell_dir = os.path.join(workspace, options.outdir, mode) + mode = mode.lower() + else: + shell_dir = os.path.join(workspace, options.outdir, + "%s.%s" % (arch, mode)) + shell_dir = os.path.relpath(shell_dir) + # Populate context object. - mode_flags = base_runner.MODES[self.build_config.mode]['flags'] + mode_flags = MODE_FLAGS[mode] timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. - if self.build_config.arch in SLOW_ARCHS: + if arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT; else: timeout = TIMEOUT_DEFAULT; - timeout *= TIMEOUT_SCALEFACTOR[self.build_config.mode] - ctx = context.Context(self.build_config.arch, - self.build_config.mode, - self.shell_dir, + timeout *= TIMEOUT_SCALEFACTOR[mode] + ctx = context.Context(arch, mode, shell_dir, mode_flags, options.verbose, timeout, options.isolates, options.command_prefix, @@ -276,26 +366,25 @@ class DeoptFuzzer(base_runner.BaseTestRunner): # Find available test suites and read test cases from them. variables = { - "arch": self.build_config.arch, - "asan": self.build_config.asan, - "byteorder": sys.byteorder, - "dcheck_always_on": self.build_config.dcheck_always_on, + "arch": arch, + "asan": options.asan, "deopt_fuzzer": True, "gc_stress": False, - "gcov_coverage": self.build_config.gcov_coverage, + "gcov_coverage": False, "isolates": options.isolates, - "mode": base_runner.MODES[self.build_config.mode]["status_mode"], - "msan": self.build_config.msan, - "no_harness": False, - "no_i18n": self.build_config.no_i18n, - "no_snap": self.build_config.no_snap, - "novfp3": False, - "predictable": self.build_config.predictable, - "simulator": utils.UseSimulator(self.build_config.arch), - "simulator_run": False, + "mode": mode, + "no_i18n": False, + "no_snap": False, + "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), - "tsan": self.build_config.tsan, - "ubsan_vptr": self.build_config.ubsan_vptr, + "tsan": False, + "msan": False, + "dcheck_always_on": options.dcheck_always_on, + "novfp3": False, + "predictable": False, + "byteorder": sys.byteorder, + "no_harness": False, + "ubsan_vptr": False, } num_tests = 0 test_id = 0 diff --git a/tools/testrunner/standard_runner.py b/tools/testrunner/standard_runner.py index 030044c042..2b143f4adc 100755 --- a/tools/testrunner/standard_runner.py +++ b/tools/testrunner/standard_runner.py @@ -28,6 +28,7 @@ from testrunner.network import network_execution from testrunner.objects import context + TIMEOUT_DEFAULT = 60 # Variants ordered by expected runtime (slowest first). @@ -59,7 +60,11 @@ GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", "--concurrent-recompilation"] # Double the timeout for these: -SLOW_ARCHS = ["arm", +SLOW_ARCHS = ["android_arm", + "android_arm64", + "android_ia32", + "android_x64", + "arm", "mips", "mipsel", "mips64", @@ -128,16 +133,23 @@ class StandardTestRunner(base_runner.BaseTestRunner): s.PrepareSources() try: - return self._execute(args, options, suites) + return self._execute(self.arch, self.mode, args, options, suites) except KeyboardInterrupt: return 2 def _add_parser_options(self, parser): + parser.add_option("--asan", + help="Regard test expectations for ASAN", + default=False, action="store_true") parser.add_option("--sancov-dir", help="Directory where to collect coverage data") parser.add_option("--cfi-vptr", help="Run tests with UBSAN cfi_vptr option.", default=False, action="store_true") + parser.add_option("--dcheck-always-on", + help="Indicates that V8 was compiled with DCHECKs" + " enabled", + default=False, action="store_true") parser.add_option("--novfp3", help="Indicates that V8 was compiled without VFP3" " support", @@ -153,6 +165,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): parser.add_option("--gc-stress", help="Switch on GC stress mode", default=False, action="store_true") + parser.add_option("--gcov-coverage", + help="Uses executables instrumented for gcov coverage", + default=False, action="store_true") parser.add_option("--command-prefix", help="Prepended to each shell command used to run a" " test", @@ -167,6 +182,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): parser.add_option("--no-harness", "--noharness", help="Run without test harness of a given suite", default=False, action="store_true") + parser.add_option("--no-i18n", "--noi18n", + help="Skip internationalization tests", + default=False, action="store_true") parser.add_option("--network", help="Distribute tests on the network", default=False, dest="network", action="store_true") parser.add_option("--no-network", "--nonetwork", @@ -175,6 +193,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): parser.add_option("--no-presubmit", "--nopresubmit", help='Skip presubmit checks (deprecated)', default=False, dest="no_presubmit", action="store_true") + parser.add_option("--no-snap", "--nosnap", + help='Test a build compiled without snapshot.', + default=False, dest="no_snap", action="store_true") parser.add_option("--no-sorting", "--nosorting", help="Don't sort tests according to duration of last" " run.", @@ -189,6 +210,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): default=False, action="store_true", help="Use exhaustive set of default variants:" " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS)) + parser.add_option("--predictable", + help="Compare output of several reruns of each test", + default=False, action="store_true") parser.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), @@ -216,6 +240,10 @@ class StandardTestRunner(base_runner.BaseTestRunner): parser.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") + parser.add_option("--shell", help="DEPRECATED! use --shell-dir", + default="") + parser.add_option("--shell-dir", help="Directory containing executables", + default="") parser.add_option("--dont-skip-slow-simulator-tests", help="Don't skip more slow tests when using a" " simulator.", @@ -228,6 +256,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): default=False, action="store_true") parser.add_option("-t", "--timeout", help="Timeout in seconds", default=TIMEOUT_DEFAULT, type="int") + parser.add_option("--tsan", + help="Regard test expectations for TSAN", + default=False, action="store_true") parser.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") parser.add_option("--valgrind", help="Run tests through valgrind", @@ -244,6 +275,12 @@ class StandardTestRunner(base_runner.BaseTestRunner): parser.add_option("--random-seed-stress-count", default=1, type="int", dest="random_seed_stress_count", help="Number of runs with different random seeds") + parser.add_option("--ubsan-vptr", + help="Regard test expectations for UBSanVptr", + default=False, action="store_true") + parser.add_option("--msan", + help="Regard test expectations for UBSanVptr", + default=False, action="store_true") def _process_options(self, options): global VARIANTS @@ -262,7 +299,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS - if self.build_config.asan: + if options.asan: options.extra_flags.append("--invoke-weak-callbacks") options.extra_flags.append("--omit-quit") @@ -277,7 +314,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. - if self.build_config.msan: + if options.msan: VARIANTS = ["default"] if options.j == 0: @@ -312,7 +349,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): if not set(VARIANTS).issubset(ALL_VARIANTS): print "All variants must be in %s" % str(ALL_VARIANTS) raise base_runner.TestRunnerError() - if self.build_config.predictable: + if options.predictable: VARIANTS = ["default"] options.extra_flags.append("--predictable") options.extra_flags.append("--verify_predictable") @@ -321,6 +358,10 @@ class StandardTestRunner(base_runner.BaseTestRunner): # Dedupe. VARIANTS = list(set(VARIANTS)) + if not options.shell_dir: + if options.shell: + print "Warning: --shell is deprecated, use --shell-dir instead." + options.shell_dir = os.path.dirname(options.shell) if options.valgrind: run_valgrind = os.path.join("tools", "run-valgrind.py") # This is OK for distributed running, so we don't need to disable @@ -333,7 +374,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): raise base_runner.TestRunnerError() CheckTestMode("slow test", options.slow_tests) CheckTestMode("pass|fail test", options.pass_fail_tests) - if self.build_config.no_i18n: + if options.no_i18n: base_runner.TEST_MAP["bot_default"].remove("intl") base_runner.TEST_MAP["default"].remove("intl") @@ -356,7 +397,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): external_symbolizer_path = '"%s.exe"' % external_symbolizer_path symbolizer = 'external_symbolizer_path=%s' % external_symbolizer_path - if self.build_config.asan: + if options.asan: asan_options = [symbolizer, "allow_user_segv_handler=1"] if not utils.GuessOS() in ['macos', 'windows']: # LSAN is not available on mac and windows. @@ -372,7 +413,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): "allow_user_segv_handler=1", ]) - if self.build_config.cfi_vptr: + if options.cfi_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', 'print_summary=1', @@ -380,16 +421,16 @@ class StandardTestRunner(base_runner.BaseTestRunner): symbolizer, ]) - if self.build_config.ubsan_vptr: + if options.ubsan_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', symbolizer, ]) - if self.build_config.msan: + if options.msan: os.environ['MSAN_OPTIONS'] = symbolizer - if self.build_config.tsan: + if options.tsan: suppressions_file = os.path.join( base_runner.BASE_DIR, 'tools', @@ -410,40 +451,57 @@ class StandardTestRunner(base_runner.BaseTestRunner): seed = random.SystemRandom().randint(-2147483648, 2147483647) return seed - def _execute(self, args, options, suites): - print(">>> Running tests for %s.%s" % (self.build_config.arch, - self.build_config.mode)) + def _execute(self, arch, mode, args, options, suites): + print(">>> Running tests for %s.%s" % (arch, mode)) - mode_options = base_runner.MODES[self.build_config.mode] + shell_dir = options.shell_dir + if not shell_dir: + if self.auto_detect: + # If an output dir with a build was passed, test directly in that + # directory. + shell_dir = os.path.join(base_runner.BASE_DIR, self.outdir) + elif options.buildbot: + # TODO(machenbach): Get rid of different output folder location on + # buildbot. Currently this is capitalized Release and Debug. + shell_dir = os.path.join(base_runner.BASE_DIR, self.outdir, mode) + mode = self._buildbot_to_v8_mode(mode) + else: + shell_dir = os.path.join( + base_runner.BASE_DIR, + self.outdir, + "%s.%s" % (arch, base_runner.MODES[mode]["output_folder"]), + ) + if not os.path.exists(shell_dir): + raise Exception('Could not find shell_dir: "%s"' % shell_dir) # Populate context object. - mode_flags = mode_options["flags"] + mode_flags = base_runner.MODES[mode]["flags"] # Simulators are slow, therefore allow a longer timeout. - if self.build_config.arch in SLOW_ARCHS: + if arch in SLOW_ARCHS: options.timeout *= 2 - options.timeout *= mode_options["timeout_scalefactor"] + options.timeout *= base_runner.MODES[mode]["timeout_scalefactor"] - if self.build_config.predictable: + if options.predictable: # Predictable mode is slower. options.timeout *= 2 - ctx = context.Context(self.build_config.arch, - mode_options["execution_mode"], - self.shell_dir, + ctx = context.Context(arch, + base_runner.MODES[mode]["execution_mode"], + shell_dir, mode_flags, options.verbose, options.timeout, options.isolates, options.command_prefix, options.extra_flags, - self.build_config.no_i18n, + options.no_i18n, options.random_seed, options.no_sorting, options.rerun_failures_count, options.rerun_failures_max, - self.build_config.predictable, + options.predictable, options.no_harness, use_perf_data=not options.swarming, sancov_dir=options.sancov_dir) @@ -451,35 +509,32 @@ class StandardTestRunner(base_runner.BaseTestRunner): # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from # target_arch != v8_target_arch in the dumped build config. - simulator_run = ( - not options.dont_skip_simulator_slow_tests and - self.build_config.arch in [ - 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', - 'ppc64', 's390', 's390x'] and - bool(base_runner.ARCH_GUESS) and - self.build_config.arch != base_runner.ARCH_GUESS) + simulator_run = not options.dont_skip_simulator_slow_tests and \ + arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ + 'ppc', 'ppc64', 's390', 's390x'] and \ + bool(base_runner.ARCH_GUESS) and arch != base_runner.ARCH_GUESS # Find available test suites and read test cases from them. variables = { - "arch": self.build_config.arch, - "asan": self.build_config.asan, - "byteorder": sys.byteorder, - "dcheck_always_on": self.build_config.dcheck_always_on, + "arch": arch, + "asan": options.asan, "deopt_fuzzer": False, "gc_stress": options.gc_stress, - "gcov_coverage": self.build_config.gcov_coverage, + "gcov_coverage": options.gcov_coverage, "isolates": options.isolates, - "mode": mode_options["status_mode"], - "msan": self.build_config.msan, - "no_harness": options.no_harness, - "no_i18n": self.build_config.no_i18n, - "no_snap": self.build_config.no_snap, - "novfp3": options.novfp3, - "predictable": self.build_config.predictable, - "simulator": utils.UseSimulator(self.build_config.arch), + "mode": base_runner.MODES[mode]["status_mode"], + "no_i18n": options.no_i18n, + "no_snap": options.no_snap, "simulator_run": simulator_run, + "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), - "tsan": self.build_config.tsan, - "ubsan_vptr": self.build_config.ubsan_vptr, + "tsan": options.tsan, + "msan": options.msan, + "dcheck_always_on": options.dcheck_always_on, + "novfp3": options.novfp3, + "predictable": options.predictable, + "byteorder": sys.byteorder, + "no_harness": options.no_harness, + "ubsan_vptr": options.ubsan_vptr, } all_tests = [] num_tests = 0 @@ -546,8 +601,8 @@ class StandardTestRunner(base_runner.BaseTestRunner): if options.json_test_results: progress_indicator.Register(progress.JsonTestProgressIndicator( options.json_test_results, - self.build_config.arch, - mode_options["execution_mode"], + arch, + base_runner.MODES[mode]["execution_mode"], ctx.random_seed)) if options.flakiness_results: progress_indicator.Register(progress.FlakinessTestProgressIndicator(