[test] Organize flags into sections and share more code

Bug: v8:6917
Change-Id: I4267900b6beed44eeae3df9b3b7a3e58402d6e6e
Reviewed-on: https://chromium-review.googlesource.com/899366
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51084}

tools/testrunner/base_runner.py

@@ -5,6 +5,7 @@
 
 from collections import OrderedDict
 import json
+import multiprocessing
 import optparse
 import os
 import shlex
@@ -227,6 +228,10 @@ class BaseTestRunner(object):
     try:
       parser = self._create_parser()
       options, args = self._parse_args(parser, sys_args)
+      if options.swarming:
+        # Swarming doesn't print how isolated commands are called. Lets make
+        # this less cryptic by printing it ourselves.
+        print ' '.join(sys.argv)
 
       self._load_build_config(options)
 
@@ -239,10 +244,14 @@ class BaseTestRunner(object):
 
       args = self._parse_test_args(args)
       suites = self._get_suites(args, options)
-      self._load_status_files(suites, options)
+      self._prepare_suites(suites, options)
 
       self._setup_env()
-      return self._do_execute(suites, args, options)
+
+      print(">>> Running tests for %s.%s" % (self.build_config.arch,
+                                            self.mode_name))
+      tests = [t for s in suites for t in s.tests]
+      return self._do_execute(tests, args, options)
     except TestRunnerError:
       return 1
     except KeyboardInterrupt:
@@ -273,6 +282,11 @@ class BaseTestRunner(object):
                       "directory will be used")
     parser.add_option("--total-timeout-sec", default=0, type="int",
                       help="How long should fuzzer run")
+    parser.add_option("--swarming", default=False, action="store_true",
+                      help="Indicates running test driver on swarming.")
+
+    parser.add_option("-j", help="The number of parallel tasks to run",
+                      default=0, type=int)
 
     # Shard
     parser.add_option("--shard-count", default=1, type=int,
@@ -446,6 +460,9 @@ class BaseTestRunner(object):
       print('Warning: --shell-dir is deprecated. Searching for executables in '
             'build directory (%s) instead.' % self.outdir)
 
+    if options.j == 0:
+      options.j = multiprocessing.cpu_count()
+
     options.command_prefix = shlex.split(options.command_prefix)
     options.extra_flags = sum(map(shlex.split, options.extra_flags), [])
 
@@ -565,6 +582,11 @@ class BaseTestRunner(object):
           test_config)
     return map(load_suite, names)
 
+  def _prepare_suites(self, suites, options):
+    self._load_status_files(suites, options)
+    for s in suites:
+      s.ReadTestCases()
+
   def _load_status_files(self, suites, options):
     # simd_mips is true if SIMD is fully supported on MIPS
     variables = self._get_statusfile_variables(options)
@@ -638,6 +660,12 @@ class BaseTestRunner(object):
   def _do_execute(self, suites, args, options):
     raise NotImplementedError()
 
+  def _prepare_procs(self, procs):
+    procs = filter(None, procs)
+    for i in xrange(0, len(procs) - 1):
+      procs[i].connect_to(procs[i + 1])
+    procs[0].setup()
+
   def _create_shard_proc(self, options):
     myid, count = self._get_shard_info(options)
     if count == 1:

tools/testrunner/num_fuzzer.py

@@ -5,7 +5,6 @@
 # found in the LICENSE file.
 
 
-import multiprocessing
 import random
 import sys
 
@@ -32,15 +31,9 @@ class NumFuzzer(base_runner.BaseTestRunner):
     super(NumFuzzer, self).__init__(*args, **kwargs)
 
   def _add_parser_options(self, parser):
-    parser.add_option("--dump-results-file", help="Dump maximum limit reached")
-    parser.add_option("-j", help="The number of parallel tasks to run",
-                      default=0, type="int")
     parser.add_option("--fuzzer-random-seed", default=0,
                       help="Default seed for initializing fuzzer random "
                       "generator")
-    parser.add_option("--swarming",
-                      help="Indicates running test driver on swarming.",
-                      default=False, action="store_true")
     parser.add_option("--tests-count", default=5, type="int",
                       help="Number of tests to generate from each base test. "
                            "Can be combined with --total-timeout-sec with "
@@ -88,8 +81,6 @@ class NumFuzzer(base_runner.BaseTestRunner):
 
 
   def _process_options(self, options):
-    if options.j == 0:
-      options.j = multiprocessing.cpu_count()
     if not options.fuzzer_random_seed:
       options.fuzzer_random_seed = random_utils.random_seed()
 
@@ -129,13 +120,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
     })
     return variables
 
-  def _do_execute(self, suites, args, options):
+  def _do_execute(self, tests, args, options):
-    print(">>> Running tests for %s.%s" % (self.build_config.arch,
-                                           self.mode_name))
-
-    self._setup_suites(options, suites)
-    tests = self._load_tests(options, suites)
-
     loader = LoadProc()
     fuzzer_rng = random.Random(options.fuzzer_random_seed)
 
@@ -178,31 +163,21 @@ class NumFuzzer(base_runner.BaseTestRunner):
       return 1
     return 0
 
-  def _setup_suites(self, options, suites):
+  def _load_suites(self, names, options):
+    suites = super(NumFuzzer, self)._load_suites(names, options)
+    if options.combine_tests:
+      suites = [s for s in suites if s.test_combiner_available()]
+    return suites
+
+  def _prepare_suites(self, suites, options):
     """Sets additional configurations on test suites based on options."""
+    super(NumFuzzer, self)._prepare_suites(suites, options)
+
     if options.stress_interrupt_budget:
       # Changing interrupt budget forces us to suppress certain test assertions.
       for suite in suites:
         suite.do_suppress_internals()
 
-  def _load_tests(self, options, suites):
-    if options.combine_tests:
-      suites = [s for s in suites if s.test_combiner_available()]
-
-    # Find available test suites and read test cases from them.
-
-    tests = []
-    for s in suites:
-      s.ReadTestCases()
-      tests += s.tests
-    return tests
-
-  def _prepare_procs(self, procs):
-    procs = filter(None, procs)
-    for i in xrange(0, len(procs) - 1):
-      procs[i].connect_to(procs[i + 1])
-    procs[0].setup()
-
   def _create_combiner(self, rng, options):
     if not options.combine_tests:
       return None

tools/testrunner/standard_runner.py

@@ -5,7 +5,6 @@
 # found in the LICENSE file.
 
 
-import multiprocessing
 import os
 import re
 import sys
@@ -71,53 +70,13 @@ class StandardTestRunner(base_runner.BaseTestRunner):
     def _get_default_suite_names(self):
       return ['default']
 
-    def _do_execute(self, suites, args, options):
-      if options.swarming:
-        # Swarming doesn't print how isolated commands are called. Lets make
-        # this less cryptic by printing it ourselves.
-        print ' '.join(sys.argv)
-      return self._execute(args, options, suites)
-
     def _add_parser_options(self, parser):
-      parser.add_option("--sancov-dir",
-                        help="Directory where to collect coverage data")
-      parser.add_option("--cfi-vptr",
-                        help="Run tests with UBSAN cfi_vptr option.",
-                        default=False, action="store_true")
       parser.add_option("--novfp3",
                         help="Indicates that V8 was compiled without VFP3"
                         " support",
                         default=False, action="store_true")
-      parser.add_option("--cat", help="Print the source of the tests",
+
-                        default=False, action="store_true")
+      # Variants
-      parser.add_option("--slow-tests",
-                        help="Regard slow tests (run|skip|dontcare)",
-                        default="dontcare")
-      parser.add_option("--pass-fail-tests",
-                        help="Regard pass|fail tests (run|skip|dontcare)",
-                        default="dontcare")
-      parser.add_option("--gc-stress",
-                        help="Switch on GC stress mode",
-                        default=False, action="store_true")
-      parser.add_option("--random-gc-stress",
-                        help="Switch on random GC stress mode",
-                        default=False, action="store_true")
-      parser.add_option("--infra-staging", help="Use new test runner features",
-                        dest='infra_staging', default=None,
-                        action="store_true")
-      parser.add_option("--no-infra-staging",
-                        help="Opt out of new test runner features",
-                        dest='infra_staging', default=None,
-                        action="store_false")
-      parser.add_option("-j", help="The number of parallel tasks to run",
-                        default=0, type="int")
-      parser.add_option("--no-presubmit", "--nopresubmit",
-                        help='Skip presubmit checks (deprecated)',
-                        default=False, dest="no_presubmit", action="store_true")
-      parser.add_option("--no-sorting", "--nosorting",
-                        help="Don't sort tests according to duration of last"
-                        " run.",
-                        default=False, dest="no_sorting", action="store_true")
       parser.add_option("--no-variants", "--novariants",
                         help="Deprecated. "
                              "Equivalent to passing --variants=default",
@@ -129,24 +88,26 @@ class StandardTestRunner(base_runner.BaseTestRunner):
                         default=False, action="store_true",
                         help="Deprecated. "
                              "Equivalent to passing --variants=exhaustive")
+
+      # Filters
+      parser.add_option("--slow-tests", default="dontcare",
+                        help="Regard slow tests (run|skip|dontcare)")
+      parser.add_option("--pass-fail-tests", default="dontcare",
+                        help="Regard pass|fail tests (run|skip|dontcare)")
       parser.add_option("--quickcheck", default=False, action="store_true",
                         help=("Quick check mode (skip slow tests)"))
-      parser.add_option("--report", help="Print a summary of the tests to be"
-                        " run",
-                        default=False, action="store_true")
-      parser.add_option("--flakiness-results",
-                        help="Path to a file for storing flakiness json.")
       parser.add_option("--dont-skip-slow-simulator-tests",
                         help="Don't skip more slow tests when using a"
                         " simulator.",
                         default=False, action="store_true",
                         dest="dont_skip_simulator_slow_tests")
-      parser.add_option("--swarming",
+
-                        help="Indicates running test driver on swarming.",
+      # Stress modes
+      parser.add_option("--gc-stress",
+                        help="Switch on GC stress mode",
                         default=False, action="store_true")
-      parser.add_option("--time", help="Print timing information after running",
+      parser.add_option("--random-gc-stress",
-                        default=False, action="store_true")
+                        help="Switch on random GC stress mode",
-      parser.add_option("--warn-unused", help="Report unused rules",
                         default=False, action="store_true")
       parser.add_option("--random-seed-stress-count", default=1, type="int",
                         dest="random_seed_stress_count",
@@ -154,6 +115,40 @@ class StandardTestRunner(base_runner.BaseTestRunner):
                              "with test processors: 0 means infinite "
                              "generation.")
 
+      # Noop
+      parser.add_option("--cfi-vptr",
+                        help="Run tests with UBSAN cfi_vptr option.",
+                        default=False, action="store_true")
+      parser.add_option("--infra-staging", help="Use new test runner features",
+                        dest='infra_staging', default=None,
+                        action="store_true")
+      parser.add_option("--no-infra-staging",
+                        help="Opt out of new test runner features",
+                        dest='infra_staging', default=None,
+                        action="store_false")
+      parser.add_option("--no-sorting", "--nosorting",
+                        help="Don't sort tests according to duration of last"
+                        " run.",
+                        default=False, dest="no_sorting", action="store_true")
+      parser.add_option("--no-presubmit", "--nopresubmit",
+                        help='Skip presubmit checks (deprecated)',
+                        default=False, dest="no_presubmit", action="store_true")
+
+      # Unimplemented for test processors
+      parser.add_option("--sancov-dir",
+                        help="Directory where to collect coverage data")
+      parser.add_option("--cat", help="Print the source of the tests",
+                        default=False, action="store_true")
+      parser.add_option("--flakiness-results",
+                        help="Path to a file for storing flakiness json.")
+      parser.add_option("--time", help="Print timing information after running",
+                        default=False, action="store_true")
+      parser.add_option("--warn-unused", help="Report unused rules",
+                        default=False, action="store_true")
+      parser.add_option("--report", default=False, action="store_true",
+                        help="Print a summary of the tests to be run")
+
+
     def _process_options(self, options):
       if options.sancov_dir:
         self.sancov_dir = options.sancov_dir
@@ -210,9 +205,6 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       if self.build_config.msan:
         options.variants = "default"
 
-      if options.j == 0:
-        options.j = multiprocessing.cpu_count()
-
       if options.variants == "infra_staging":
         options.variants = "exhaustive"
 
@@ -279,16 +271,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       })
       return variables
 
-    def _execute(self, args, options, suites):
+    def _do_execute(self, tests, args, options):
-      print(">>> Running tests for %s.%s" % (self.build_config.arch,
-                                             self.mode_name))
-
-      for s in suites:
-        s.ReadTestCases()
-
-      return self._run_test_procs(suites, args, options)
-
-    def _run_test_procs(self, suites, args, options):
       jobs = options.j
 
       print '>>> Running with test processors'
@@ -320,15 +303,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         execproc,
       ]
 
-      procs = filter(None, procs)
+      self._prepare_procs(procs)
-
-      for i in xrange(0, len(procs) - 1):
-        procs[i].connect_to(procs[i + 1])
-
-      tests = [t for s in suites for t in s.tests]
       tests.sort(key=lambda t: t.is_slow, reverse=True)
 
-      loader.setup()
       loader.load_tests(tests)
 
       print '>>> Running %d base tests' % tests_counter.total