v8/tools/testrunner/standard_runner.py

#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


from collections import OrderedDict
from os.path import join
import multiprocessing
import os
import random
import shlex
import subprocess
import sys
import time

# Adds testrunner to the path hence it has to be imported at the beggining.
import base_runner

from testrunner.local import execution
from testrunner.local import progress
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.local import verbose
from testrunner.local.variants import ALL_VARIANTS
from testrunner.objects import context
from testrunner.objects import predictable
from testrunner.testproc.execution import ExecutionProc
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
from testrunner.testproc.progress import (VerboseProgressIndicator,
                                          ResultsTracker,
                                          TestsCounter)
from testrunner.testproc.rerun import RerunProc
from testrunner.testproc.variant import VariantProc


TIMEOUT_DEFAULT = 60

# Variants ordered by expected runtime (slowest first).
VARIANTS = ["default"]

MORE_VARIANTS = [
  "stress",
  "stress_incremental_marking",
  "nooptimization",
  "stress_background_compile",
  "wasm_traps",
]

VARIANT_ALIASES = {
  # The default for developer workstations.
  "dev": VARIANTS,
  # Additional variants, run on all bots.
  "more": MORE_VARIANTS,
  # Shortcut for the two above ("more" first - it has the longer running tests).
  "exhaustive": MORE_VARIANTS + VARIANTS,
  # Additional variants, run on a subset of bots.
  "extra": ["future", "liftoff", "trusted"],
}

GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
                   "--concurrent-recompilation-queue-length=64",
                   "--concurrent-recompilation-delay=500",
                   "--concurrent-recompilation"]

# Double the timeout for these:
SLOW_ARCHS = ["arm",
              "mips",
              "mipsel",
              "mips64",
              "mips64el",
              "s390",
              "s390x",
              "arm64"]

PREDICTABLE_WRAPPER = os.path.join(
    base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')


class StandardTestRunner(base_runner.BaseTestRunner):
    def __init__(self, *args, **kwargs):
        super(StandardTestRunner, self).__init__(*args, **kwargs)

        self.sancov_dir = None

    def _get_default_suite_names(self):
      return ['default']

    def _do_execute(self, suites, args, options):
      if options.swarming:
        # Swarming doesn't print how isolated commands are called. Lets make
        # this less cryptic by printing it ourselves.
        print ' '.join(sys.argv)

        if utils.GuessOS() == "macos":
          # TODO(machenbach): Temporary output for investigating hanging test
          # driver on mac.
          print "V8 related processes running on this host:"
          try:
            print subprocess.check_output(
              "ps -e | egrep 'd8|cctest|unittests'", shell=True)
          except Exception:
            pass

      return self._execute(args, options, suites)

    def _add_parser_options(self, parser):
      parser.add_option("--sancov-dir",
                        help="Directory where to collect coverage data")
      parser.add_option("--cfi-vptr",
                        help="Run tests with UBSAN cfi_vptr option.",
                        default=False, action="store_true")
      parser.add_option("--novfp3",
                        help="Indicates that V8 was compiled without VFP3"
                        " support",
                        default=False, action="store_true")
      parser.add_option("--cat", help="Print the source of the tests",
                        default=False, action="store_true")
      parser.add_option("--slow-tests",
                        help="Regard slow tests (run|skip|dontcare)",
                        default="dontcare")
      parser.add_option("--pass-fail-tests",
                        help="Regard pass|fail tests (run|skip|dontcare)",
                        default="dontcare")
      parser.add_option("--gc-stress",
                        help="Switch on GC stress mode",
                        default=False, action="store_true")
      parser.add_option("--command-prefix",
                        help="Prepended to each shell command used to run a"
                        " test",
                        default="")
      parser.add_option("--extra-flags",
                        help="Additional flags to pass to each test command",
                        action="append", default=[])
      parser.add_option("--infra-staging", help="Use new test runner features",
                        default=False, action="store_true")
      parser.add_option("--isolates", help="Whether to test isolates",
                        default=False, action="store_true")
      parser.add_option("-j", help="The number of parallel tasks to run",
                        default=0, type="int")
      parser.add_option("--no-harness", "--noharness",
                        help="Run without test harness of a given suite",
                        default=False, action="store_true")
      parser.add_option("--no-presubmit", "--nopresubmit",
                        help='Skip presubmit checks (deprecated)',
                        default=False, dest="no_presubmit", action="store_true")
      parser.add_option("--no-sorting", "--nosorting",
                        help="Don't sort tests according to duration of last"
                        " run.",
                        default=False, dest="no_sorting", action="store_true")
      parser.add_option("--no-variants", "--novariants",
                        help="Deprecated. "
                             "Equivalent to passing --variants=default",
                        default=False, dest="no_variants", action="store_true")
      parser.add_option("--variants",
                        help="Comma-separated list of testing variants;"
                        " default: \"%s\"" % ",".join(VARIANTS))
      parser.add_option("--exhaustive-variants",
                        default=False, action="store_true",
                        help="Deprecated. "
                             "Equivalent to passing --variants=exhaustive")
      parser.add_option("-p", "--progress",
                        help=("The style of progress indicator"
                              " (verbose, dots, color, mono)"),
                        choices=progress.PROGRESS_INDICATORS.keys(),
                        default="mono")
      parser.add_option("--quickcheck", default=False, action="store_true",
                        help=("Quick check mode (skip slow tests)"))
      parser.add_option("--report", help="Print a summary of the tests to be"
                        " run",
                        default=False, action="store_true")
      parser.add_option("--json-test-results",
                        help="Path to a file for storing json results.")
      parser.add_option("--flakiness-results",
                        help="Path to a file for storing flakiness json.")
      parser.add_option("--rerun-failures-count",
                        help=("Number of times to rerun each failing test case."
                              " Very slow tests will be rerun only once."),
                        default=0, type="int")
      parser.add_option("--rerun-failures-max",
                        help="Maximum number of failing test cases to rerun.",
                        default=100, type="int")
      parser.add_option("--dont-skip-slow-simulator-tests",
                        help="Don't skip more slow tests when using a"
                        " simulator.",
                        default=False, action="store_true",
                        dest="dont_skip_simulator_slow_tests")
      parser.add_option("--swarming",
                        help="Indicates running test driver on swarming.",
                        default=False, action="store_true")
      parser.add_option("--time", help="Print timing information after running",
                        default=False, action="store_true")
      parser.add_option("-t", "--timeout", help="Timeout in seconds",
                        default=TIMEOUT_DEFAULT, type="int")
      parser.add_option("--warn-unused", help="Report unused rules",
                        default=False, action="store_true")
      parser.add_option("--junitout", help="File name of the JUnit output")
      parser.add_option("--junittestsuite",
                        help="The testsuite name in the JUnit output file",
                        default="v8tests")
      parser.add_option("--random-seed", default=0, dest="random_seed",
                        help="Default seed for initializing random generator",
                        type=int)
      parser.add_option("--random-seed-stress-count", default=1, type="int",
                        dest="random_seed_stress_count",
                        help="Number of runs with different random seeds")

    def _process_options(self, options):
      global VARIANTS

      if options.sancov_dir:
        self.sancov_dir = options.sancov_dir
        if not os.path.exists(self.sancov_dir):
          print("sancov-dir %s doesn't exist" % self.sancov_dir)
          raise base_runner.TestRunnerError()

      options.command_prefix = shlex.split(options.command_prefix)
      options.extra_flags = sum(map(shlex.split, options.extra_flags), [])

      if options.gc_stress:
        options.extra_flags += GC_STRESS_FLAGS

      if self.build_config.asan:
        options.extra_flags.append("--invoke-weak-callbacks")
        options.extra_flags.append("--omit-quit")

      if options.novfp3:
        options.extra_flags.append("--noenable-vfp3")

      if options.no_variants:  # pragma: no cover
        print ("Option --no-variants is deprecated. "
               "Pass --variants=default instead.")
        assert not options.variants
        options.variants = "default"

      if options.exhaustive_variants:  # pragma: no cover
        # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
        print ("Option --exhaustive-variants is deprecated. "
               "Pass --variants=exhaustive instead.")
        # This is used on many bots. It includes a larger set of default
        # variants.
        # Other options for manipulating variants still apply afterwards.
        assert not options.variants
        options.variants = "exhaustive"

      if options.quickcheck:
        assert not options.variants
        options.variants = "stress,default"
        options.slow_tests = "skip"
        options.pass_fail_tests = "skip"

      if self.build_config.predictable:
        options.variants = "default"
        options.extra_flags.append("--predictable")
        options.extra_flags.append("--verify_predictable")
        options.extra_flags.append("--no-inline-new")
        # Add predictable wrapper to command prefix.
        options.command_prefix = (
            [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)

      # TODO(machenbach): Figure out how to test a bigger subset of variants on
      # msan.
      if self.build_config.msan:
        options.variants = "default"

      if options.j == 0:
        options.j = multiprocessing.cpu_count()

      if options.random_seed_stress_count <= 1 and options.random_seed == 0:
        options.random_seed = self._random_seed()

      # Use developer defaults if no variant was specified.
      options.variants = options.variants or "dev"

      if options.variants == "infra_staging":
        options.variants = "exhaustive"
        options.infra_staging = True

      # Resolve variant aliases and dedupe.
      # TODO(machenbach): Don't mutate global variable. Rather pass mutated
      # version as local variable.
      VARIANTS = list(set(reduce(
          list.__add__,
          (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")),
          [],
      )))

      if not set(VARIANTS).issubset(ALL_VARIANTS):
        print "All variants must be in %s" % str(ALL_VARIANTS)
        raise base_runner.TestRunnerError()

      def CheckTestMode(name, option):  # pragma: no cover
        if not option in ["run", "skip", "dontcare"]:
          print "Unknown %s mode %s" % (name, option)
          raise base_runner.TestRunnerError()
      CheckTestMode("slow test", options.slow_tests)
      CheckTestMode("pass|fail test", options.pass_fail_tests)
      if self.build_config.no_i18n:
        base_runner.TEST_MAP["bot_default"].remove("intl")
        base_runner.TEST_MAP["default"].remove("intl")
        # TODO(machenbach): uncomment after infra side lands.
        # base_runner.TEST_MAP["d8_default"].remove("intl")

    def _setup_env(self):
      super(StandardTestRunner, self)._setup_env()

      symbolizer_option = self._get_external_symbolizer_option()

      if self.sancov_dir:
        os.environ['ASAN_OPTIONS'] = ":".join([
          'coverage=1',
          'coverage_dir=%s' % self.sancov_dir,
          symbolizer_option,
          "allow_user_segv_handler=1",
        ])

    def _random_seed(self):
      seed = 0
      while not seed:
        seed = random.SystemRandom().randint(-2147483648, 2147483647)
      return seed

    def _execute(self, args, options, suites):
      print(">>> Running tests for %s.%s" % (self.build_config.arch,
                                             self.mode_name))
      # Populate context object.

      # Simulators are slow, therefore allow a longer timeout.
      if self.build_config.arch in SLOW_ARCHS:
        options.timeout *= 2

      options.timeout *= self.mode_options.timeout_scalefactor

      if self.build_config.predictable:
        # Predictable mode is slower.
        options.timeout *= 2

      ctx = context.Context(self.build_config.arch,
                            self.mode_options.execution_mode,
                            self.outdir,
                            self.mode_options.flags,
                            options.verbose,
                            options.timeout,
                            options.isolates,
                            options.command_prefix,
                            options.extra_flags,
                            self.build_config.no_i18n,
                            options.random_seed,
                            options.no_sorting,
                            options.rerun_failures_count,
                            options.rerun_failures_max,
                            options.no_harness,
                            use_perf_data=not options.swarming,
                            sancov_dir=self.sancov_dir,
                            infra_staging=options.infra_staging)

      # TODO(all): Combine "simulator" and "simulator_run".
      # TODO(machenbach): In GN we can derive simulator run from
      # target_arch != v8_target_arch in the dumped build config.
      simulator_run = (
        not options.dont_skip_simulator_slow_tests and
        self.build_config.arch in [
          'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
          'ppc64', 's390', 's390x'] and
        bool(base_runner.ARCH_GUESS) and
        self.build_config.arch != base_runner.ARCH_GUESS)
      # Find available test suites and read test cases from them.
      variables = {
        "arch": self.build_config.arch,
        "asan": self.build_config.asan,
        "byteorder": sys.byteorder,
        "dcheck_always_on": self.build_config.dcheck_always_on,
        "deopt_fuzzer": False,
        "gc_fuzzer": False,
        "gc_stress": options.gc_stress,
        "gcov_coverage": self.build_config.gcov_coverage,
        "isolates": options.isolates,
        "mode": self.mode_options.status_mode,
        "msan": self.build_config.msan,
        "no_harness": options.no_harness,
        "no_i18n": self.build_config.no_i18n,
        "no_snap": self.build_config.no_snap,
        "novfp3": options.novfp3,
        "predictable": self.build_config.predictable,
        "simulator": utils.UseSimulator(self.build_config.arch),
        "simulator_run": simulator_run,
        "system": utils.GuessOS(),
        "tsan": self.build_config.tsan,
        "ubsan_vptr": self.build_config.ubsan_vptr,
      }

      progress_indicator = progress.IndicatorNotifier()
      progress_indicator.Register(
        progress.PROGRESS_INDICATORS[options.progress]())
      if options.junitout:  # pragma: no cover
        progress_indicator.Register(progress.JUnitTestProgressIndicator(
            options.junitout, options.junittestsuite))
      if options.json_test_results:
        progress_indicator.Register(progress.JsonTestProgressIndicator(
          options.json_test_results,
          self.build_config.arch,
          self.mode_options.execution_mode,
          ctx.random_seed))
      if options.flakiness_results:  # pragma: no cover
        progress_indicator.Register(progress.FlakinessTestProgressIndicator(
            options.flakiness_results))

      if options.infra_staging:
        for s in suites:
          s.ReadStatusFile(variables)
          s.ReadTestCases(ctx)

        return self._run_test_procs(suites, args, options, progress_indicator,
                                    ctx)

      all_tests = []
      num_tests = 0
      for s in suites:
        s.ReadStatusFile(variables)
        s.ReadTestCases(ctx)
        if len(args) > 0:
          s.FilterTestCasesByArgs(args)
        all_tests += s.tests

        # First filtering by status applying the generic rules (tests without
        # variants)
        if options.warn_unused:
          tests = [(t.name, t.variant) for t in s.tests]
          s.statusfile.warn_unused_rules(tests, check_variant_rules=False)
        s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests)

        if options.cat:
          verbose.PrintTestSource(s.tests)
          continue
        variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS)
        variant_tests = [ t.create_variant(v, flags)
                          for t in s.tests
                          for v in variant_gen.FilterVariantsByTest(t)
                          for flags in variant_gen.GetFlagSets(t, v) ]

        if options.random_seed_stress_count > 1:
          # Duplicate test for random seed stress mode.
          def iter_seed_flags():
            for _ in range(0, options.random_seed_stress_count):
              # Use given random seed for all runs (set by default in
              # execution.py) or a new random seed if none is specified.
              if options.random_seed:
                yield []
              else:
                yield ["--random-seed=%d" % self._random_seed()]
          s.tests = [
            t.create_variant(t.variant, flags, 'seed-stress-%d' % n)
            for t in variant_tests
            for n, flags in enumerate(iter_seed_flags())
          ]
        else:
          s.tests = variant_tests

        # Second filtering by status applying also the variant-dependent rules.
        if options.warn_unused:
          tests = [(t.name, t.variant) for t in s.tests]
          s.statusfile.warn_unused_rules(tests, check_variant_rules=True)

        s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests)
        s.tests = self._shard_tests(s.tests, options)

        for t in s.tests:
          t.cmd = t.get_command(ctx)

        num_tests += len(s.tests)

      if options.cat:
        return 0  # We're done here.

      if options.report:
        verbose.PrintReport(all_tests)

      # Run the tests.
      start_time = time.time()

      if self.build_config.predictable:
        outproc_factory = predictable.get_outproc
      else:
        outproc_factory = None

      runner = execution.Runner(suites, progress_indicator, ctx,
                                outproc_factory)
      exit_code = runner.Run(options.j)
      overall_duration = time.time() - start_time

      if options.time:
        verbose.PrintTestDurations(suites, runner.outputs, overall_duration)

      if num_tests == 0:
        print("Warning: no tests were run!")

      if exit_code == 1 and options.json_test_results:
        print("Force exit code 0 after failures. Json test results file "
              "generated with failure information.")
        exit_code = 0

      if self.sancov_dir:
        # If tests ran with sanitizer coverage, merge coverage files in the end.
        try:
          print "Merging sancov files."
          subprocess.check_call([
            sys.executable,
            join(self.basedir, "tools", "sanitizers", "sancov_merger.py"),
            "--coverage-dir=%s" % self.sancov_dir])
        except:
          print >> sys.stderr, "Error: Merging sancov files failed."
          exit_code = 1

      return exit_code

    def _shard_tests(self, tests, options):
      shard_run, shard_count = self._get_shard_info(options)

      if shard_count < 2:
        return tests
      count = 0
      shard = []
      for test in tests:
        if count % shard_count == shard_run - 1:
          shard.append(test)
        count += 1
      return shard

    def _run_test_procs(self, suites, args, options, progress_indicator,
                        context):
      jobs = options.j

      print '>>> Running with test processors'
      loader = LoadProc()
      tests_counter = TestsCounter()
      results = ResultsTracker()
      indicators = progress_indicator.ToProgressIndicatorProcs()
      execproc = ExecutionProc(jobs, context)

      procs = [
        loader,
        NameFilterProc(args) if args else None,
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
        self._create_shard_proc(options),
        tests_counter,
        VariantProc(VARIANTS),
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
      ] + indicators + [
        results,
        self._create_rerun_proc(context),
        execproc,
      ]

      procs = filter(None, procs)

      for i in xrange(0, len(procs) - 1):
        procs[i].connect_to(procs[i + 1])

      tests = [t for s in suites for t in s.tests]
      tests.sort(key=lambda t: t.is_slow, reverse=True)

      loader.setup()
      loader.load_tests(tests)

      print '>>> Running %d base tests' % tests_counter.total
      tests_counter.remove_from_chain()

      execproc.start()

      for indicator in indicators:
        indicator.finished()

      print '>>> %d tests ran' % results.total

      exit_code = 0
      if results.failed:
        exit_code = 1
      if results.remaining:
        exit_code = 2


      if exit_code == 1 and options.json_test_results:
        print("Force exit code 0 after failures. Json test results file "
              "generated with failure information.")
        exit_code = 0
      return exit_code

    def _create_rerun_proc(self, ctx):
      if not ctx.rerun_failures_count:
        return None
      return RerunProc(ctx.rerun_failures_count,
                       ctx.rerun_failures_max)



if __name__ == '__main__':
  sys.exit(StandardTestRunner().execute())