#!/usr/bin/env python # # Copyright 2017 the V8 project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from os.path import join import itertools import json import math import multiprocessing import os import random import shlex import sys import time # Adds testrunner to the path hence it has to be imported at the beggining. import base_runner from testrunner.local import execution from testrunner.local import progress from testrunner.local import testsuite from testrunner.local import utils from testrunner.local import verbose from testrunner.objects import context DEFAULT_TESTS = ["mjsunit", "webkit"] TIMEOUT_DEFAULT = 60 # Double the timeout for these: SLOW_ARCHS = ["arm", "mipsel"] class GCFuzzer(base_runner.BaseTestRunner): def __init__(self): super(GCFuzzer, self).__init__() self.fuzzer_rng = None def _add_parser_options(self, parser): parser.add_option("--command-prefix", help="Prepended to each shell command used to run a test", default="") parser.add_option("--coverage", help=("Exponential test coverage " "(range 0.0, 1.0) - 0.0: one test, 1.0 all tests (slow)"), default=0.4, type="float") parser.add_option("--coverage-lift", help=("Lifts test coverage for tests " "with a low memory size reached (range 0, inf)"), default=20, type="int") parser.add_option("--dump-results-file", help="Dump maximum limit reached") parser.add_option("--extra-flags", help="Additional flags to pass to each test command", default="") parser.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") parser.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") parser.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") parser.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") parser.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") parser.add_option("-t", "--timeout", help="Timeout in seconds", default= -1, type="int") parser.add_option("--random-seed", default=0, help="Default seed for initializing random generator") parser.add_option("--fuzzer-random-seed", default=0, help="Default seed for initializing fuzzer random " "generator") parser.add_option("--stress-compaction", default=False, action="store_true", help="Enable stress_compaction_percentage flag") parser.add_option("--distribution-factor1", help="DEPRECATED") parser.add_option("--distribution-factor2", help="DEPRECATED") parser.add_option("--distribution-mode", help="DEPRECATED") parser.add_option("--seed", help="DEPRECATED") return parser def _process_options(self, options): # Special processing of other options, sorted alphabetically. options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = shlex.split(options.extra_flags) if options.j == 0: options.j = multiprocessing.cpu_count() while options.random_seed == 0: options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) while options.fuzzer_random_seed == 0: options.fuzzer_random_seed = random.SystemRandom().randint(-2147483648, 2147483647) self.fuzzer_rng = random.Random(options.fuzzer_random_seed) return True def _shard_tests(self, tests, shard_count, shard_run): if shard_count < 2: return tests if shard_run < 1 or shard_run > shard_count: print "shard-run not a valid number, should be in [1:shard-count]" print "defaulting back to running all tests" return tests count = 0 shard = [] for test in tests: if count % shard_count == shard_run - 1: shard.append(test) count += 1 return shard def _do_execute(self, options, args): suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test")) if len(args) == 0: suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] else: args_suites = set() for arg in args: suite = arg.split(os.path.sep)[0] if not suite in args_suites: args_suites.add(suite) suite_paths = [ s for s in suite_paths if s in args_suites ] suites = [] for root in suite_paths: suite = testsuite.TestSuite.LoadTestSuite( os.path.join(base_runner.BASE_DIR, "test", root)) if suite: suites.append(suite) try: return self._execute(args, options, suites) except KeyboardInterrupt: return 2 def _calculate_n_tests(self, m, options): """Calculates the number of tests from m points with exponential coverage. The coverage is expected to be between 0.0 and 1.0. The 'coverage lift' lifts the coverage for tests with smaller m values. """ c = float(options.coverage) l = float(options.coverage_lift) return int(math.pow(m, (m * c + l) / (m + l))) def _execute(self, args, options, suites): print(">>> Running tests for %s.%s" % (self.build_config.arch, self.mode_name)) # Populate context object. timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. if self.build_config.arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT; else: timeout = TIMEOUT_DEFAULT; timeout *= self.mode_options.timeout_scalefactor ctx = context.Context(self.build_config.arch, self.mode_options.execution_mode, self.outdir, self.mode_options.flags, options.verbose, timeout, options.isolates, options.command_prefix, options.extra_flags, False, # Keep i18n on by default. options.random_seed, True, # No sorting of test cases. 0, # Don't rerun failing tests. 0, # No use of a rerun-failing-tests maximum. False, # No predictable mode. False, # No no_harness mode. False, # Don't use perf data. False) # Coverage not supported. num_tests = self._load_tests(args, options, suites, ctx) if num_tests == 0: print "No tests to run." return 0 test_backup = dict(map(lambda s: (s, s.tests), suites)) print('>>> Collection phase') for s in suites: analysis_flags = [ # > 100% to not influence default incremental marking, but we need this # flag to print reached incremental marking limit. '--stress_marking', '1000', '--trace_incremental_marking', ] s.tests = map(lambda t: t.CopyAddingFlags(t.variant, analysis_flags), s.tests) for t in s.tests: t.cmd = s.GetCommand(t, ctx) progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) print('>>> Analysis phase') test_results = dict() for s in suites: for t in s.tests: # Skip failed tests. if s.HasUnexpectedOutput(t): print '%s failed, skipping' % t.path continue max_limit = self._get_max_limit_reached(t) if max_limit: test_results[t.path] = max_limit if options.dump_results_file: with file("%s.%d.txt" % (options.dump_results_file, time.time()), "w") as f: f.write(json.dumps(test_results)) num_tests = 0 for s in suites: s.tests = [] for t in test_backup[s]: max_percent = test_results.get(t.path, 0) if not max_percent or max_percent < 1.0: continue max_percent = int(max_percent) subtests_count = self._calculate_n_tests(max_percent, options) if options.verbose: print ('%s [x%d] (max marking limit=%.02f)' % (t.path, subtests_count, max_percent)) for _ in xrange(0, subtests_count): fuzzer_seed = self._next_fuzzer_seed() fuzzing_flags = [ '--stress_marking', str(max_percent), '--fuzzer_random_seed', str(fuzzer_seed), ] if options.stress_compaction: fuzzing_flags.append('--stress_compaction_random') s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags)) for t in s.tests: t.cmd = s.GetCommand(t, ctx) num_tests += len(s.tests) if num_tests == 0: print "No tests to run." return 0 print(">>> Fuzzing phase (%d test cases)" % num_tests) progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) return runner.Run(options.j) or exit_code def _load_tests(self, args, options, suites, ctx): # Find available test suites and read test cases from them. variables = { "arch": self.build_config.arch, "asan": self.build_config.asan, "byteorder": sys.byteorder, "dcheck_always_on": self.build_config.dcheck_always_on, "deopt_fuzzer": False, "gc_fuzzer": True, "gc_stress": False, "gcov_coverage": self.build_config.gcov_coverage, "isolates": options.isolates, "mode": self.mode_options.status_mode, "msan": self.build_config.msan, "no_harness": False, "no_i18n": self.build_config.no_i18n, "no_snap": self.build_config.no_snap, "novfp3": False, "predictable": self.build_config.predictable, "simulator": utils.UseSimulator(self.build_config.arch), "simulator_run": False, "system": utils.GuessOS(), "tsan": self.build_config.tsan, "ubsan_vptr": self.build_config.ubsan_vptr, } num_tests = 0 test_id = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) s.FilterTestCasesByStatus(False) for t in s.tests: t.flags += s.GetStatusfileFlags(t) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 return num_tests # Parses test stdout and returns what was the highest reached percent of the # incremental marking limit (0-100). # Skips values >=100% since they already trigger incremental marking. @staticmethod def _get_max_limit_reached(test): def is_im_line(l): return 'IncrementalMarking' in l and '% of the memory limit reached' in l def line_to_percent(l): return filter(lambda part: '%' in part, l.split(' '))[0] def percent_str_to_float(s): return float(s[:-1]) if not (test.output and test.output.stdout): return None im_lines = filter(is_im_line, test.output.stdout.splitlines()) percents_str = map(line_to_percent, im_lines) percents = map(percent_str_to_float, percents_str) # Skip >= 100%. percents = filter(lambda p: p < 100, percents) if not percents: return None return max(percents) def _next_fuzzer_seed(self): fuzzer_seed = None while not fuzzer_seed: fuzzer_seed = self.fuzzer_rng.randint(-2147483648, 2147483647) return fuzzer_seed if __name__ == '__main__': sys.exit(GCFuzzer().execute())