From 874eadd1ea3d817078ec3e59d196e49d5bca74be Mon Sep 17 00:00:00 2001 From: Michael Achenbach Date: Fri, 26 Jan 2018 13:04:22 +0000 Subject: [PATCH] Revert "[test] Keep the order of variants" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8f4407fbd6fe0915a40ded5f5f9f8fd7c06d0366. Reason for revert: Failing in old test runner. See comment. Original change's description: > [test] Keep the order of variants > > Bug: v8:6917 > Change-Id: I7e04255649f62a28f0ed07ccb4fbaf4825498bf6 > Reviewed-on: https://chromium-review.googlesource.com/886841 > Commit-Queue: Michał Majewski > Reviewed-by: Michael Achenbach > Cr-Commit-Position: refs/heads/master@{#50874} TBR=machenbach@chromium.org,sergiyb@chromium.org,majeski@google.com Change-Id: Ia8608053e080c70d2ac09d7921a6abb6cd0e338f No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: v8:6917 Reviewed-on: https://chromium-review.googlesource.com/888749 Reviewed-by: Michael Achenbach Commit-Queue: Michael Achenbach Cr-Commit-Position: refs/heads/master@{#50895} --- tools/testrunner/local/testsuite.py | 2 +- tools/testrunner/local/variants.py | 25 +--------------- tools/testrunner/standard_runner.py | 45 ++++++++++++++--------------- 3 files changed, 24 insertions(+), 48 deletions(-) diff --git a/tools/testrunner/local/testsuite.py b/tools/testrunner/local/testsuite.py index 299c4b31ad..7b6791d733 100644 --- a/tools/testrunner/local/testsuite.py +++ b/tools/testrunner/local/testsuite.py @@ -43,7 +43,7 @@ STANDARD_VARIANT = set(["default"]) class LegacyVariantsGenerator(object): def __init__(self, suite, variants): self.suite = suite - self.all_variants = variants + self.all_variants = ALL_VARIANTS & variants self.standard_variant = STANDARD_VARIANT & variants def FilterVariantsByTest(self, test): diff --git a/tools/testrunner/local/variants.py b/tools/testrunner/local/variants.py index 60f5e63679..73e9a6e644 100644 --- a/tools/testrunner/local/variants.py +++ b/tools/testrunner/local/variants.py @@ -25,27 +25,4 @@ ALL_VARIANT_FLAGS = { "wasm_traps": [["--wasm_trap_handler", "--invoke-weak-callbacks", "--wasm-jit-to-native"]], } -SLOW_VARIANTS = set([ - 'stress', - 'nooptimization', -]) - -FAST_VARIANTS = set([ - 'default' -]) - - -def _variant_order_key(v): - if v in SLOW_VARIANTS: - return 0 - if v in FAST_VARIANTS: - return 100 - return 50 - -ALL_VARIANTS = sorted(ALL_VARIANT_FLAGS.keys(), - key=_variant_order_key) - -# Check {SLOW,FAST}_VARIANTS entries -for variants in [SLOW_VARIANTS, FAST_VARIANTS]: - for v in variants: - assert v in ALL_VARIANT_FLAGS +ALL_VARIANTS = set(ALL_VARIANT_FLAGS.keys()) diff --git a/tools/testrunner/standard_runner.py b/tools/testrunner/standard_runner.py index e16926ce77..65333a0b13 100755 --- a/tools/testrunner/standard_runner.py +++ b/tools/testrunner/standard_runner.py @@ -39,13 +39,14 @@ from testrunner.testproc.variant import VariantProc TIMEOUT_DEFAULT = 60 +# Variants ordered by expected runtime (slowest first). VARIANTS = ["default"] MORE_VARIANTS = [ - "nooptimization", "stress", - "stress_background_compile", "stress_incremental_marking", + "nooptimization", + "stress_background_compile", "wasm_traps", ] @@ -111,7 +112,6 @@ class StandardTestRunner(base_runner.BaseTestRunner): super(StandardTestRunner, self).__init__(*args, **kwargs) self.sancov_dir = None - self._variants = None def _get_default_suite_names(self): return ['default'] @@ -249,6 +249,8 @@ class StandardTestRunner(base_runner.BaseTestRunner): return False def _process_options(self, options): + global VARIANTS + if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): @@ -313,6 +315,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): if options.random_seed_stress_count <= 1 and options.random_seed == 0: options.random_seed = self._random_seed() + # Use developer defaults if no variant was specified. + options.variants = options.variants or "dev" + if options.variants == "infra_staging": options.variants = "exhaustive" options.infra_staging = True @@ -320,7 +325,18 @@ class StandardTestRunner(base_runner.BaseTestRunner): # Use staging on whitelisted masters/builders. options.infra_staging = self._use_staging(options) - self._variants = self._parse_variants(options.variants) + # Resolve variant aliases and dedupe. + # TODO(machenbach): Don't mutate global variable. Rather pass mutated + # version as local variable. + VARIANTS = list(set(reduce( + list.__add__, + (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")), + [], + ))) + + if not set(VARIANTS).issubset(ALL_VARIANTS): + print "All variants must be in %s" % str(ALL_VARIANTS) + raise base_runner.TestRunnerError() def CheckTestMode(name, option): # pragma: no cover if not option in ["run", "skip", "dontcare"]: @@ -334,23 +350,6 @@ class StandardTestRunner(base_runner.BaseTestRunner): # TODO(machenbach): uncomment after infra side lands. # base_runner.TEST_MAP["d8_default"].remove("intl") - def _parse_variants(self, aliases_str): - # Use developer defaults if no variant was specified. - aliases_str = aliases_str or 'dev' - aliases = aliases_str.split(',') - user_variants = set(reduce( - list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases])) - - result = [v for v in ALL_VARIANTS if v in user_variants] - if len(result) == len(user_variants): - return result - - for v in user_variants: - if v not in ALL_VARIANTS: - print 'Unknown variant: %s' % v - raise base_runner.TestRunnerError() - assert False, 'Unreachable' - def _setup_env(self): super(StandardTestRunner, self)._setup_env() @@ -482,7 +481,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): if options.cat: verbose.PrintTestSource(s.tests) continue - variant_gen = s.CreateLegacyVariantsGenerator(self._variants) + variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS) variant_tests = [ t.create_variant(v, flags) for t in s.tests for v in variant_gen.FilterVariantsByTest(t) @@ -598,7 +597,7 @@ class StandardTestRunner(base_runner.BaseTestRunner): StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), self._create_shard_proc(options), tests_counter, - VariantProc(self._variants), + VariantProc(VARIANTS), StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), ] + indicators + [ results,