AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[foozzie] Initial correctness fuzzer harness.

Browse Source 
Initial version of the correctness fuzzer harness for manual testing
and experiments.

For automated usage, some outstanding TODOs are left in the code. E.g.
- Hash source file names in error case
- Bundle script in out directory with executables
- Some suppressions are tied to already fixed bugs. We'll keep it like that for now to test
removing those suppressions in production later.

BUG=chromium:673246
NOTRY=true

Review-Url: https://codereview.chromium.org/2578503003
Cr-Commit-Position: refs/heads/master@{#41789}
This commit is contained in:
machenbach 2016-12-19 02:13:48 -08:00 committed by Commit bot
parent 815f91c0ed
commit e669816e1f
6 changed files with 761 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
tools/foozzie/v8_commands.py Normal file
View File

@ -0,0 +1,64 @@
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Fork from commands.py and output.py in v8 test driver.
import signal
import subprocess
import sys
from threading import Event, Timer
class Output(object):
def __init__(self, exit_code, timed_out, stdout, pid):
self.exit_code = exit_code
self.timed_out = timed_out
self.stdout = stdout
self.pid = pid
def HasCrashed(self):
# Timed out tests will have exit_code -signal.SIGTERM.
if self.timed_out:
return False
return (self.exit_code < 0 and
self.exit_code != -signal.SIGABRT)
def HasTimedOut(self):
return self.timed_out
def Execute(args, cwd, timeout=None):
popen_args = [c for c in args if c != ""]
try:
process = subprocess.Popen(
args=popen_args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd
)
except Exception as e:
sys.stderr.write("Error executing: %s\n" % popen_args)
raise e
timeout_event = Event()
def kill_process():
timeout_event.set()
try:
process.kill()
except OSError:
sys.stderr.write('Error: Process %s already ended.\n' % process.pid)
timer = Timer(timeout, kill_process)
timer.start()
stdout, _ = process.communicate()
timer.cancel()
return Output(
process.returncode,
timeout_event.is_set(),
stdout.decode('utf-8', 'replace').encode('utf-8'),
process.pid,
)

257
tools/foozzie/v8_foozzie.py Executable file
View File

@ -0,0 +1,257 @@
#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
V8 correctness fuzzer launcher script.
"""
import argparse
import itertools
import os
import re
import sys
import traceback
import v8_commands
import v8_suppressions
CONFIGS = dict(
default=[],
validate_asm=['--validate-asm'], # Maybe add , '--disable-asm-warnings'
fullcode=['--nocrankshaft', '--turbo-filter=~'],
noturbo=['--turbo-filter=~', '--noturbo-asm'],
noturbo_opt=['--always-opt', '--turbo-filter=~', '--noturbo-asm'],
ignition_staging=['--ignition-staging'],
ignition_turbo=['--ignition-staging', '--turbo'],
ignition_turbo_opt=['--ignition-staging', '--turbo', '--always-opt'],
)
# Timeout in seconds for one d8 run.
TIMEOUT = 3
# Return codes.
RETURN_PASS = 0
RETURN_FAILURE = 2
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
PREAMBLE = [
os.path.join(BASE_PATH, 'v8_mock.js'),
os.path.join(BASE_PATH, 'v8_suppressions.js'),
]
FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
'--invoke-weak-callbacks', '--omit-quit', '--es-staging']
SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
# Output for suppressed failure case.
FAILURE_HEADER_TEMPLATE = """
#
# V8 correctness failure
# V8 correctness configs: %(configs)s
# V8 correctness sources: %(sources)s
# V8 correctness suppression: %(suppression)s""".strip()
# Extended output for failure case. The 'CHECK' is for the minimizer.
FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """
#
# CHECK
#
# Compared %(first_config_label)s with %(second_config_label)s
#
# Flags of %(first_config_label)s:
%(first_config_flags)s
# Flags of %(second_config_label)s:
%(second_config_flags)s
#
# Difference:
%(difference)s
#
### Start of configuration %(first_config_label)s:
%(first_config_output)s
### End of configuration %(first_config_label)s
#
### Start of configuration %(second_config_label)s:
%(second_config_output)s
### End of configuration %(second_config_label)s
""".strip()
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'--random-seed', type=int, required=True,
help='random seed passed to both runs')
parser.add_argument(
'--first-arch', help='first architecture', default='x64')
parser.add_argument(
'--second-arch', help='second architecture', default='x64')
parser.add_argument(
'--first-config', help='first configuration', default='fullcode')
parser.add_argument(
'--second-config', help='second configuration', default='fullcode')
parser.add_argument(
'--first-d8', default='d8',
help='optional path to first d8 executable, '
'default: bundled in the same directory as this script')
parser.add_argument(
'--second-d8',
help='optional path to second d8 executable, default: same as first')
parser.add_argument('testcase', help='path to test case')
options = parser.parse_args()
# Ensure we make a sane comparison.
assert (options.first_arch != options.second_arch or
options.first_config != options.second_config) , (
'Need either arch or config difference.')
assert options.first_arch in SUPPORTED_ARCHS
assert options.second_arch in SUPPORTED_ARCHS
assert options.first_config in CONFIGS
assert options.second_config in CONFIGS
# Ensure we have a test case.
assert (os.path.exists(options.testcase) and
os.path.isfile(options.testcase)), (
'Test case %s doesn\'t exist' % options.testcase)
# Use first d8 as default for second d8.
options.second_d8 = options.second_d8 or options.first_d8
# Ensure absolute paths.
options.first_d8 = os.path.abspath(options.first_d8)
options.second_d8 = os.path.abspath(options.second_d8)
# Ensure executables exist.
assert os.path.exists(options.first_d8)
assert os.path.exists(options.second_d8)
# Ensure we use different executables when we claim we compare
# different architectures.
# TODO(machenbach): Infer arch from gn's build output.
if options.first_arch != options.second_arch:
assert options.first_d8 != options.second_d8
return options
def test_pattern_bailout(testcase, ignore_fun):
"""Print failure state and return if ignore_fun matches testcase."""
with open(testcase) as f:
bug = (ignore_fun(f.read()) or '').strip()
if bug:
print FAILURE_HEADER_TEMPLATE % dict(
configs='', sources='', suppression=bug)
return True
return False
def pass_bailout(output, step_number):
"""Print info and return if in timeout or crash pass states."""
if output.HasTimedOut():
# Dashed output, so that no other clusterfuzz tools can match the
# words timeout or crash.
print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
return True
if output.HasCrashed():
print '# V8 correctness - C-R-A-S-H %d' % step_number
return True
return False
def fail_bailout(output, ignore_by_output_fun):
"""Print failure state and return if ignore_by_output_fun matches output."""
bug = (ignore_by_output_fun(output.stdout) or '').strip()
if bug:
print FAILURE_HEADER_TEMPLATE % dict(
configs='', sources='', suppression=bug)
return True
return False
def main():
options = parse_args()
# Suppressions are architecture and configuration specific.
suppress = v8_suppressions.get_suppression(
options.first_arch, options.first_config,
options.second_arch, options.second_config,
)
if test_pattern_bailout(options.testcase, suppress.ignore):
return RETURN_FAILURE
common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
first_config_flags = common_flags + CONFIGS[options.first_config]
second_config_flags = common_flags + CONFIGS[options.second_config]
def run_d8(d8, config_flags):
return v8_commands.Execute(
[d8] + config_flags + PREAMBLE + [options.testcase],
cwd=os.path.dirname(options.testcase),
timeout=TIMEOUT,
)
first_config_output = run_d8(options.first_d8, first_config_flags)
# Early bailout based on first run's output.
if pass_bailout(first_config_output, 1):
return RETURN_PASS
if fail_bailout(first_config_output, suppress.ignore_by_output1):
return RETURN_FAILURE
second_config_output = run_d8(options.second_d8, second_config_flags)
# Bailout based on second run's output.
if pass_bailout(second_config_output, 2):
return RETURN_PASS
if fail_bailout(second_config_output, suppress.ignore_by_output2):
return RETURN_FAILURE
difference = suppress.diff(
first_config_output.stdout, second_config_output.stdout)
if difference:
# The first three entries will be parsed by clusterfuzz. Format changes
# will require changes on the clusterfuzz side.
first_config_label = '%s,%s' % (options.first_arch, options.first_config)
second_config_label = '%s,%s' % (options.second_arch, options.second_config)
print FAILURE_TEMPLATE % dict(
configs='%s:%s' % (first_config_label, second_config_label),
sources='', # TODO
suppression='', # We can't tie bugs to differences.
first_config_label=first_config_label,
second_config_label=second_config_label,
first_config_flags=' '.join(first_config_flags),
second_config_flags=' '.join(second_config_flags),
first_config_output=first_config_output.stdout,
second_config_output=second_config_output.stdout,
difference=difference,
)
return RETURN_FAILURE
# TODO(machenbach): Figure out if we could also return a bug in case there's
# no difference, but one of the line suppressions has matched - and without
# the match there would be a difference.
print '# V8 correctness - pass'
return RETURN_PASS
if __name__ == "__main__":
try:
result = main()
except SystemExit:
# Make sure clusterfuzz reports internal errors and wrong usage.
# Use one label for all internal and usage errors.
print FAILURE_HEADER_TEMPLATE % dict(
configs='', sources='', suppression='wrong_usage')
result = RETURN_FAILURE
except Exception as e:
print FAILURE_HEADER_TEMPLATE % dict(
configs='', sources='', suppression='internal_error')
print '# Internal error: %s' % e
traceback.print_exc(file=sys.stdout)
result = RETURN_FAILURE
sys.exit(result)

77
tools/foozzie/v8_foozzie_test.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import unittest
import v8_suppressions
class FuzzerTest(unittest.TestCase):
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression(
'x64', 'fullcode', 'x64', 'default')
one = ''
two = ''
diff = None
self.assertEquals(diff, suppress.diff(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None
self.assertEquals(diff, suppress.diff(one, two))
# Ignore line before caret, caret position, stack trace char numbers
# error message and validator output.
one = """
undefined
weird stuff
^
Validation of asm.js module failed: foo bar
somefile.js: TypeError: undefined is not a function
stack line :15: foo
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
stack line :2: foo
Validation of asm.js module failed: baz
undefined
"""
diff = None
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line'
self.assertEquals(diff, suppress.diff(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line'
self.assertEquals(diff, suppress.diff(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor"""
self.assertEquals(diff, suppress.diff(one, two))

76
tools/foozzie/v8_mock.js Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This is intended for permanent JS behavior changes for mocking out
// non-deterministic behavior. For temporary suppressions, please refer to
// v8_suppressions.js.
// This file is loaded before each correctness test cases and won't get
// minimized.
// This will be overridden in the test cases. The override can be minimized.
var __PrettyPrint = function __PrettyPrint(msg) { print(msg); };
// All calls to f.arguments are replaced by f.mock_arguments by an external
// script.
Object.prototype.mock_arguments = ['x', 'y']
// Mock Math.random.
var __magic_index_for_mocked_random = 0
Math.random = function(){
__magic_index_for_mocked_random = (__magic_index_for_mocked_random + 1) % 10
return __magic_index_for_mocked_random / 10.0;
}
// Mock Date.
var __magic_index_for_mocked_date = 0
var __magic_mocked_date = 1477662728696
__magic_mocked_date_now = function(){
__magic_index_for_mocked_date = (__magic_index_for_mocked_date + 1) % 10
__magic_mocked_date = __magic_mocked_date + __magic_index_for_mocked_date + 1
return __magic_mocked_date
}
var __original_date = Date;
__magic_mock_date_handler = {
construct: function(target, args, newTarget) {
if (args.length > 0) {
return new (Function.prototype.bind.apply(__original_date, [null].concat(args)));
} else {
return new __original_date(__magic_mocked_date_now());
}
},
get: function(target, property, receiver) {
if (property == "now") {
return __magic_mocked_date_now;
}
},
}
Date = new Proxy(Date, __magic_mock_date_handler);
// Mock Worker.
var __magic_index_for_mocked_worker = 0
// TODO(machenbach): Randomize this for each test case, but keep stable during
// comparison. Also data and random above.
var __magic_mocked_worker_messages = [
undefined, 0, -1, "", "foo", 42, [], {}, [0], {"x": 0}
]
Worker = function(code){
try {
__PrettyPrint(eval(code));
} catch(e) {
__PrettyPrint(e);
}
this.getMessage = function(){
__magic_index_for_mocked_worker = (__magic_index_for_mocked_worker + 1) % 10
return __magic_mocked_worker_messages[__magic_index_for_mocked_worker];
}
this.postMessage = function(msg){
__PrettyPrint(msg);
}
}

21
tools/foozzie/v8_suppressions.js Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This file is loaded before each correctness test case and after v8_mock.js.
// You can temporarily change JS behavior here to silence known problems.
// Please refer to a bug in a comment and remove the suppression once the
// problem is fixed.
// Suppress http://crbug.com/662429
var __real_Math_pow = Math.pow
Math.pow = function(a, b){
if (b < 0) {
return 0.000017;
} else {
return __real_Math_pow(a, b);
}
}
// Suppress http://crbug.com/663750
Object.freeze = function(o){ print (__PrettyPrint(o)); }

266
tools/foozzie/v8_suppressions.py Normal file
View File

@ -0,0 +1,266 @@
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Suppressions for V8 correctness fuzzer failures.
We support three types of suppressions:
1. Ignore test case by pattern.
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches a JS test case.
Subsequent matches will be recoreded under the first failure.
2. Ignore test run by output pattern:
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches the output of a particular run.
Subsequent matches will be recoreded under the first failure.
3. Relax line-to-line comparisons with expressions of lines to ignore and
lines to be normalized (i.e. ignore only portions of lines).
These are not tied to bugs, be careful to not silently switch off this tool!
Alternatively, think about adding a behavior change to v8_suppressions.js
to silence a particular class of problems.
"""
import itertools
import re
# Max line length for regular experessions checking for lines to ignore.
MAX_LINE_LENGTH = 512
# For ignoring lines before carets and to ignore caret positions.
CARET_RE = re.compile(r'^\s*\^\s*$')
# Ignore by test case pattern. Map from bug->regexp.
# Regular expressions are assumed to be compiled. We use regexp.match.
IGNORE_TEST_CASES = {
'crbug.com/662907':
re.compile(r'.*new Array.*\[\d+\] =.*'
r'((Array)|(Object)).prototype.__defineSetter__.*', re.S),
'crbug.com/663340':
re.compile(r'.*\.shift\(\).*', re.S),
'crbug.com/666308':
re.compile(r'.*End stripped down and modified version.*'
r'\.prototype.*instanceof.*.*', re.S),
}
# Ignore by output pattern. Map from config->bug->regexp. Config '' is used
# to match all configurations. Otherwise use either a compiler configuration,
# e.g. fullcode or validate_asm or an architecture, e.g. x64 or ia32 or a
# comma-separated combination, e.g. x64,fullcode, for more specific
# suppressions.
# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
# label.
# Regular expressions are assumed to be compiled. We use regexp.search.
IGNORE_OUTPUT = {
'': {
'crbug.com/664068':
re.compile(r'RangeError', re.S),
'crbug.com/669017':
re.compile(r'SyntaxError', re.S),
},
'validate_asm': {
'validate_asm':
re.compile(r'TypeError'),
},
}
# Lines matching any of the following regular expressions will be ignored
# if appearing on both sides. The capturing groups need to match exactly.
# Use uncompiled regular expressions - they'll be compiled later.
ALLOWED_LINE_DIFFS = [
# Ignore caret position in stack traces.
r'^\s*\^\s*$',
# Ignore some stack trace headers as messages might not match.
r'^(.*)TypeError: .* is not a function$',
r'^(.*)TypeError: .* is not a constructor$',
r'^(.*)TypeError: (.*) is not .*$',
r'^(.*)ReferenceError: .* is not defined$',
r'^(.*):\d+: ReferenceError: .* is not defined$',
# These are rarely needed. It includes some cases above.
r'^\w*Error: .* is not .*$',
r'^(.*) \w*Error: .* is not .*$',
r'^(.*):\d+: \w*Error: .* is not .*$',
# Some test cases just print the message.
r'^.* is not a function(.*)$',
r'^(.*) is not a .*$',
# crbug.com/669017
r'^(.*)SyntaxError: .*$',
# Ignore lines of stack traces as character positions might not match.
r'^ at (?:new )?([^:]*):\d+:\d+(.*)$',
r'^(.*):\d+:(.*)$',
# crbug.com/662840
r"^.*(?:Trying to access ')?(\w*)(?:(?:' through proxy)|"
r"(?: is not defined))$",
]
# Lines matching any of the following regular expressions will be ignored.
# Use uncompiled regular expressions - they'll be compiled later.
IGNORE_LINES = [
r'^Validation of asm\.js module failed: .+$',
r'^.*:\d+: Invalid asm.js: .*$',
r'^Warning: unknown flag .*$',
r'^Warning: .+ is deprecated.*$',
r'^Try --help for options$',
]
###############################################################################
# Implementation - you should not need to change anything below this point.
# Compile regular expressions.
ALLOWED_LINE_DIFFS = [re.compile(exp) for exp in ALLOWED_LINE_DIFFS]
IGNORE_LINES = [re.compile(exp) for exp in IGNORE_LINES]
def line_pairs(lines):
return itertools.izip_longest(
lines, itertools.islice(lines, 1, None), fillvalue=None)
def caret_match(line1, line2):
if (not line1 or
not line2 or
len(line1) > MAX_LINE_LENGTH or
len(line2) > MAX_LINE_LENGTH):
return False
return bool(CARET_RE.match(line1) and CARET_RE.match(line2))
def short_line_output(line):
if len(line) <= MAX_LINE_LENGTH:
# Avoid copying.
return line
return line[0:MAX_LINE_LENGTH] + '...'
def ignore_by_regexp(line1, line2, allowed):
if len(line1) > MAX_LINE_LENGTH or len(line2) > MAX_LINE_LENGTH:
return False
for exp in allowed:
match1 = exp.match(line1)
match2 = exp.match(line2)
if match1 and match2:
# If there are groups in the regexp, ensure the groups matched the same
# things.
if match1.groups() == match2.groups(): # tuple comparison
return True
return False
def diff_output(output1, output2, allowed, ignore1, ignore2):
def useful_line(ignore):
def fun(line):
return all(not e.match(line) for e in ignore)
return fun
lines1 = filter(useful_line(ignore1), output1)
lines2 = filter(useful_line(ignore2), output2)
for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest(
line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):
# Only one of the two iterators should run out.
assert not (line1 is None and line2 is None)
# One iterator ends earlier.
if line1 is None:
return '+ %s' % short_line_output(line2)
if line2 is None:
return '- %s' % short_line_output(line1)
# If lines are equal, no further checks are necessary.
if line1 == line2:
continue
# Look ahead. If next line is a caret, ignore this line.
if caret_match(lookahead1, lookahead2):
continue
# Check if a regexp allows these lines to be different.
if ignore_by_regexp(line1, line2, allowed):
continue
# Lines are different.
return '- %s\n+ %s' % (short_line_output(line1), short_line_output(line2))
# No difference found.
return None
def get_suppression(arch1, config1, arch2, config2):
return V8Suppression(arch1, config1, arch2, config2)
class Suppression(object):
def diff(self, output1, output2):
return None
def ignore(self, testcase):
return False
def ignore_by_output1(self, output):
return False
def ignore_by_output2(self, output):
return False
class V8Suppression(Suppression):
def __init__(self, arch1, config1, arch2, config2):
self.arch1 = arch1
self.config1 = config1
self.arch2 = arch2
self.config2 = config2
def diff(self, output1, output2):
return diff_output(
output1.splitlines(),
output2.splitlines(),
ALLOWED_LINE_DIFFS,
IGNORE_LINES,
IGNORE_LINES,
)
def ignore(self, testcase):
for bug, exp in IGNORE_TEST_CASES.iteritems():
if exp.match(testcase):
return bug
return False
def ignore_by_output1(self, output):
return self.ignore_by_output(output, self.arch1, self.config1)
def ignore_by_output2(self, output):
return self.ignore_by_output(output, self.arch2, self.config2)
def ignore_by_output(self, output, arch, config):
def check(mapping):
for bug, exp in mapping.iteritems():
if exp.search(output):
return bug
return None
bug = check(IGNORE_OUTPUT.get('', {}))
if bug:
return bug
bug = check(IGNORE_OUTPUT.get(arch, {}))
if bug:
return bug
bug = check(IGNORE_OUTPUT.get(config, {}))
if bug:
return bug
bug = check(IGNORE_OUTPUT.get('%s,%s' % (arch, config), {}))
if bug:
return bug
return None