2018-01-19 16:13:05 +00:00
|
|
|
#!/usr/bin/env python
|
|
|
|
#
|
|
|
|
# Copyright 2017 the V8 project authors. All rights reserved.
|
|
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
|
|
# found in the LICENSE file.
|
|
|
|
|
2019-02-18 14:01:33 +00:00
|
|
|
# for py2/py3 compatibility
|
|
|
|
from __future__ import print_function
|
2018-01-19 16:13:05 +00:00
|
|
|
|
|
|
|
import random
|
|
|
|
import sys
|
|
|
|
|
|
|
|
# Adds testrunner to the path hence it has to be imported at the beggining.
|
|
|
|
import base_runner
|
|
|
|
|
|
|
|
from testrunner.local import utils
|
|
|
|
|
|
|
|
from testrunner.testproc import fuzzer
|
|
|
|
from testrunner.testproc.base import TestProcProducer
|
2018-01-25 15:18:29 +00:00
|
|
|
from testrunner.testproc.combiner import CombinerProc
|
2018-01-19 16:13:05 +00:00
|
|
|
from testrunner.testproc.execution import ExecutionProc
|
2018-02-08 17:10:50 +00:00
|
|
|
from testrunner.testproc.expectation import ForgiveTimeoutProc
|
2018-01-19 16:13:05 +00:00
|
|
|
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
|
|
|
|
from testrunner.testproc.loader import LoadProc
|
2019-01-21 07:00:32 +00:00
|
|
|
from testrunner.testproc.progress import ResultsTracker
|
2018-01-31 09:18:13 +00:00
|
|
|
from testrunner.utils import random_utils
|
2018-01-19 16:13:05 +00:00
|
|
|
|
|
|
|
|
|
|
|
DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]
|
|
|
|
|
|
|
|
|
|
|
|
class NumFuzzer(base_runner.BaseTestRunner):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
super(NumFuzzer, self).__init__(*args, **kwargs)
|
|
|
|
|
2019-04-11 08:58:47 +00:00
|
|
|
@property
|
|
|
|
def framework_name(self):
|
|
|
|
return 'num_fuzzer'
|
|
|
|
|
2018-01-19 16:13:05 +00:00
|
|
|
def _add_parser_options(self, parser):
|
|
|
|
parser.add_option("--fuzzer-random-seed", default=0,
|
|
|
|
help="Default seed for initializing fuzzer random "
|
|
|
|
"generator")
|
2018-01-25 15:18:29 +00:00
|
|
|
parser.add_option("--tests-count", default=5, type="int",
|
|
|
|
help="Number of tests to generate from each base test. "
|
|
|
|
"Can be combined with --total-timeout-sec with "
|
|
|
|
"value 0 to provide infinite number of subtests. "
|
|
|
|
"When --combine-tests is set it indicates how many "
|
|
|
|
"tests to create in total")
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2018-01-23 15:42:37 +00:00
|
|
|
# Stress gc
|
2018-01-19 16:13:05 +00:00
|
|
|
parser.add_option("--stress-marking", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --stress-marking "
|
|
|
|
"flag to the test")
|
|
|
|
parser.add_option("--stress-scavenge", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --stress-scavenge "
|
|
|
|
"flag to the test")
|
|
|
|
parser.add_option("--stress-compaction", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --stress-compaction "
|
|
|
|
"flag to the test")
|
|
|
|
parser.add_option("--stress-gc", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --random-gc-interval "
|
|
|
|
"flag to the test")
|
2018-10-10 09:09:14 +00:00
|
|
|
|
|
|
|
# Stress tasks
|
|
|
|
parser.add_option("--stress-delay-tasks", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --stress-delay-tasks "
|
|
|
|
"flag to the test")
|
2018-02-16 19:15:01 +00:00
|
|
|
parser.add_option("--stress-thread-pool-size", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --thread-pool-size "
|
|
|
|
"flag to the test")
|
2018-01-23 15:42:37 +00:00
|
|
|
|
|
|
|
# Stress deopt
|
2018-01-22 15:33:05 +00:00
|
|
|
parser.add_option("--stress-deopt", default=0, type="int",
|
|
|
|
help="probability [0-10] of adding --deopt-every-n-times "
|
|
|
|
"flag to the test")
|
|
|
|
parser.add_option("--stress-deopt-min", default=1, type="int",
|
|
|
|
help="extends --stress-deopt to have minimum interval "
|
|
|
|
"between deopt points")
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2018-01-23 15:42:37 +00:00
|
|
|
# Combine multiple tests
|
|
|
|
parser.add_option("--combine-tests", default=False, action="store_true",
|
|
|
|
help="Combine multiple tests as one and run with "
|
|
|
|
"try-catch wrapper")
|
2018-01-25 15:18:29 +00:00
|
|
|
parser.add_option("--combine-max", default=100, type="int",
|
|
|
|
help="Maximum number of tests to combine")
|
|
|
|
parser.add_option("--combine-min", default=2, type="int",
|
|
|
|
help="Minimum number of tests to combine")
|
2018-01-23 15:42:37 +00:00
|
|
|
|
2018-04-18 06:29:10 +00:00
|
|
|
# Miscellaneous
|
|
|
|
parser.add_option("--variants", default='default',
|
|
|
|
help="Comma-separated list of testing variants")
|
|
|
|
|
2018-01-19 16:13:05 +00:00
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
|
def _process_options(self, options):
|
2018-01-31 09:18:13 +00:00
|
|
|
if not options.fuzzer_random_seed:
|
|
|
|
options.fuzzer_random_seed = random_utils.random_seed()
|
2018-01-24 19:50:26 +00:00
|
|
|
|
|
|
|
if options.total_timeout_sec:
|
|
|
|
options.tests_count = 0
|
|
|
|
|
2018-01-25 15:18:29 +00:00
|
|
|
if options.combine_tests:
|
|
|
|
if options.combine_min > options.combine_max:
|
2019-02-18 14:01:33 +00:00
|
|
|
print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
|
|
|
|
options.min_group_size, options.max_group_size))
|
2018-01-25 15:18:29 +00:00
|
|
|
raise base_runner.TestRunnerError()
|
|
|
|
|
2018-04-18 06:29:10 +00:00
|
|
|
if options.variants != 'default':
|
|
|
|
print ('Only default testing variant is supported with numfuzz')
|
2018-04-18 07:42:58 +00:00
|
|
|
raise base_runner.TestRunnerError()
|
2018-04-18 06:29:10 +00:00
|
|
|
|
2018-01-19 16:13:05 +00:00
|
|
|
return True
|
|
|
|
|
|
|
|
def _get_default_suite_names(self):
|
|
|
|
return DEFAULT_SUITES
|
|
|
|
|
2018-02-01 14:55:53 +00:00
|
|
|
def _get_statusfile_variables(self, options):
|
|
|
|
variables = (
|
|
|
|
super(NumFuzzer, self)._get_statusfile_variables(options))
|
|
|
|
variables.update({
|
|
|
|
'deopt_fuzzer': bool(options.stress_deopt),
|
2018-02-02 12:51:25 +00:00
|
|
|
'endurance_fuzzer': bool(options.combine_tests),
|
2018-02-01 14:55:53 +00:00
|
|
|
'gc_stress': bool(options.stress_gc),
|
|
|
|
'gc_fuzzer': bool(max([options.stress_marking,
|
|
|
|
options.stress_scavenge,
|
|
|
|
options.stress_compaction,
|
2018-02-16 19:15:01 +00:00
|
|
|
options.stress_gc,
|
2018-10-10 09:09:14 +00:00
|
|
|
options.stress_delay_tasks,
|
2018-02-16 19:15:01 +00:00
|
|
|
options.stress_thread_pool_size])),
|
2018-02-01 14:55:53 +00:00
|
|
|
})
|
|
|
|
return variables
|
2018-01-31 12:01:49 +00:00
|
|
|
|
2018-02-02 22:46:27 +00:00
|
|
|
def _do_execute(self, tests, args, options):
|
2019-02-01 09:40:59 +00:00
|
|
|
loader = LoadProc(tests)
|
2018-01-19 16:13:05 +00:00
|
|
|
fuzzer_rng = random.Random(options.fuzzer_random_seed)
|
|
|
|
|
2018-01-25 15:18:29 +00:00
|
|
|
combiner = self._create_combiner(fuzzer_rng, options)
|
2018-10-02 13:31:32 +00:00
|
|
|
results = self._create_result_tracker(options)
|
2018-02-01 10:01:21 +00:00
|
|
|
execproc = ExecutionProc(options.j)
|
2018-02-20 17:23:37 +00:00
|
|
|
sigproc = self._create_signal_proc()
|
2019-02-12 15:30:16 +00:00
|
|
|
indicators = self._create_progress_indicators(
|
|
|
|
tests.test_count_estimate, options)
|
2018-01-19 16:13:05 +00:00
|
|
|
procs = [
|
|
|
|
loader,
|
|
|
|
NameFilterProc(args) if args else None,
|
|
|
|
StatusFileFilterProc(None, None),
|
2018-01-25 15:18:29 +00:00
|
|
|
# TODO(majeski): Improve sharding when combiner is present. Maybe select
|
|
|
|
# different random seeds for shards instead of splitting tests.
|
2018-01-19 16:13:05 +00:00
|
|
|
self._create_shard_proc(options),
|
2018-02-08 17:10:50 +00:00
|
|
|
ForgiveTimeoutProc(),
|
2018-01-25 15:18:29 +00:00
|
|
|
combiner,
|
2018-01-31 10:21:44 +00:00
|
|
|
self._create_fuzzer(fuzzer_rng, options),
|
2018-02-20 17:23:37 +00:00
|
|
|
sigproc,
|
2018-01-22 11:51:56 +00:00
|
|
|
] + indicators + [
|
2018-01-19 16:13:05 +00:00
|
|
|
results,
|
2018-01-24 18:25:45 +00:00
|
|
|
self._create_timeout_proc(options),
|
2018-01-22 11:51:56 +00:00
|
|
|
self._create_rerun_proc(options),
|
2018-01-19 16:13:05 +00:00
|
|
|
execproc,
|
|
|
|
]
|
|
|
|
self._prepare_procs(procs)
|
2019-02-01 09:54:34 +00:00
|
|
|
loader.load_initial_tests(initial_batch_size=float('inf'))
|
2018-01-25 15:18:29 +00:00
|
|
|
|
|
|
|
# TODO(majeski): maybe some notification from loader would be better?
|
|
|
|
if combiner:
|
|
|
|
combiner.generate_initial_tests(options.j * 4)
|
2018-02-20 17:23:37 +00:00
|
|
|
|
|
|
|
# This starts up worker processes and blocks until all tests are
|
|
|
|
# processed.
|
|
|
|
execproc.run()
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2018-01-22 11:51:56 +00:00
|
|
|
for indicator in indicators:
|
|
|
|
indicator.finished()
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2019-02-18 14:01:33 +00:00
|
|
|
print('>>> %d tests ran' % results.total)
|
2018-01-19 16:13:05 +00:00
|
|
|
if results.failed:
|
2018-02-20 17:23:37 +00:00
|
|
|
return utils.EXIT_CODE_FAILURES
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2018-02-20 17:23:37 +00:00
|
|
|
# Indicate if a SIGINT or SIGTERM happened.
|
|
|
|
return sigproc.exit_code
|
2018-01-19 16:13:05 +00:00
|
|
|
|
2019-01-11 11:29:15 +00:00
|
|
|
def _is_testsuite_supported(self, suite, options):
|
|
|
|
return not options.combine_tests or suite.test_combiner_available()
|
2018-01-29 15:51:03 +00:00
|
|
|
|
2018-01-25 15:18:29 +00:00
|
|
|
def _create_combiner(self, rng, options):
|
|
|
|
if not options.combine_tests:
|
|
|
|
return None
|
|
|
|
return CombinerProc(rng, options.combine_min, options.combine_max,
|
|
|
|
options.tests_count)
|
|
|
|
|
|
|
|
def _create_fuzzer(self, rng, options):
|
|
|
|
return fuzzer.FuzzerProc(
|
|
|
|
rng,
|
2018-01-29 15:51:03 +00:00
|
|
|
self._tests_count(options),
|
2018-01-25 15:18:29 +00:00
|
|
|
self._create_fuzzer_configs(options),
|
2018-01-29 15:51:03 +00:00
|
|
|
self._disable_analysis(options),
|
2018-01-25 15:18:29 +00:00
|
|
|
)
|
|
|
|
|
2018-01-29 15:51:03 +00:00
|
|
|
def _tests_count(self, options):
|
|
|
|
if options.combine_tests:
|
|
|
|
return 1
|
|
|
|
return options.tests_count
|
|
|
|
|
|
|
|
def _disable_analysis(self, options):
|
|
|
|
"""Disable analysis phase when options are used that don't support it."""
|
2019-01-10 08:28:36 +00:00
|
|
|
return options.combine_tests
|
2018-01-29 15:51:03 +00:00
|
|
|
|
2018-01-19 16:13:05 +00:00
|
|
|
def _create_fuzzer_configs(self, options):
|
|
|
|
fuzzers = []
|
2018-01-23 15:42:37 +00:00
|
|
|
def add(name, prob, *args):
|
|
|
|
if prob:
|
|
|
|
fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))
|
|
|
|
|
|
|
|
add('compaction', options.stress_compaction)
|
|
|
|
add('marking', options.stress_marking)
|
|
|
|
add('scavenge', options.stress_scavenge)
|
|
|
|
add('gc_interval', options.stress_gc)
|
2018-02-16 19:15:01 +00:00
|
|
|
add('threads', options.stress_thread_pool_size)
|
2018-10-10 09:09:14 +00:00
|
|
|
add('delay', options.stress_delay_tasks)
|
2018-01-23 15:42:37 +00:00
|
|
|
add('deopt', options.stress_deopt, options.stress_deopt_min)
|
2018-01-19 16:13:05 +00:00
|
|
|
return fuzzers
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
sys.exit(NumFuzzer().execute())
|