v8/tools/clusterfuzz/v8_foozzie_test.py
Michael Achenbach 9fbb56f544 [foozzie] Mock out WebAssembly when comparing with jitless
No-Try: true
Bug: chromium:1048620
Change-Id: I399144a9d8075efe40125dfcbe1dbbd0aabe0fe9
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2036080
Reviewed-by: Mathias Bynens <mathias@chromium.org>
Reviewed-by: Tamer Tas <tmrts@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#66112}
2020-02-04 15:18:39 +00:00

239 lines
7.9 KiB
Python
Executable File

#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import random
import subprocess
import sys
import unittest
import v8_commands
import v8_foozzie
import v8_fuzz_config
import v8_suppressions
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
TEST_DATA = os.path.join(BASE_DIR, 'testdata')
class ConfigTest(unittest.TestCase):
def testExperiments(self):
"""Test that probabilities add up to 100 and that all config names exist.
"""
EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
CONFIGS = v8_foozzie.CONFIGS
assert sum(x[0] for x in EXPERIMENTS) == 100
assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
assert all(map(lambda x: x[3].endswith('d8'), EXPERIMENTS))
def testConfig(self):
"""Smoke test how to choose experiments.
When experiment distribution changes this test might change, too.
"""
self.assertEqual(
[
'--first-config=ignition',
'--second-config=ignition_turbo_opt',
'--second-d8=d8',
'--second-config-extra-flags=--stress-scavenge=100',
'--second-config-extra-flags=--no-regexp-tier-up',
'--second-config-extra-flags=--no-enable-ssse3',
'--second-config-extra-flags=--no-enable-bmi2',
'--second-config-extra-flags=--no-enable-lzcnt',
],
v8_fuzz_config.Config('foo', random.Random(42)).choose_foozzie_flags(),
)
class UnitTest(unittest.TestCase):
def testDiff(self):
# TODO(machenbach): Mock out suppression configuration.
suppress = v8_suppressions.get_suppression(
'x64', 'ignition', 'x64', 'ignition_turbo')
def diff_fun(one, two):
return suppress.diff_lines(one.splitlines(), two.splitlines())
one = ''
two = ''
diff = None, None
self.assertEquals(diff, diff_fun(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEquals(diff, diff_fun(one, two))
# Ignore line before caret, caret position and error message.
one = """
undefined
weird stuff
^
somefile.js: TypeError: undefined is not a function
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: baz is not a function
undefined
"""
diff = None, None
self.assertEquals(diff, diff_fun(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEquals(diff, diff_fun(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEquals(diff, diff_fun(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEquals(diff, diff_fun(one, two))
def testOutputCapping(self):
def output(stdout, is_crash):
exit_code = -1 if is_crash else 0
return v8_commands.Output(
exit_code=exit_code, timed_out=False, stdout=stdout, pid=0)
def check(stdout1, stdout2, is_crash1, is_crash2, capped_lines1,
capped_lines2):
output1 = output(stdout1, is_crash1)
output2 = output(stdout2, is_crash2)
self.assertEquals(
(capped_lines1.splitlines(), capped_lines2.splitlines()),
v8_suppressions.get_lines_capped(output1, output2))
# No capping, already equal.
check('1\n2', '1\n2', True, True, '1\n2', '1\n2')
# No crash, no capping.
check('1\n2', '1\n2\n3', False, False, '1\n2', '1\n2\n3')
check('1\n2\n3', '1\n2', False, False, '1\n2\n3', '1\n2')
# Cap smallest if all runs crash.
check('1\n2', '1\n2\n3', True, True, '1\n2', '1\n2')
check('1\n2\n3', '1\n2', True, True, '1\n2', '1\n2')
# Cap the non-crashy run.
check('1\n2\n3', '1\n2', False, True, '1\n2', '1\n2')
check('1\n2', '1\n2\n3', True, False, '1\n2', '1\n2')
# The crashy run has more output.
check('1\n2\n3', '1\n2', True, False, '1\n2\n3', '1\n2')
check('1\n2', '1\n2\n3', False, True, '1\n2', '1\n2\n3')
# Keep output difference when capping.
check('1\n2', '3\n4\n5', True, True, '1\n2', '3\n4')
check('1\n2\n3', '4\n5', True, True, '1\n2', '4\n5')
def cut_verbose_output(stdout):
# This removes first lines containing d8 commands.
return '\n'.join(stdout.split('\n')[4:])
def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
second_config = 'ignition_turbo'
if 'second_config' in kwargs:
second_config = 'jitless'
return subprocess.check_output([
sys.executable, FOOZZIE,
'--random-seed', '12345',
'--first-d8', os.path.join(TEST_DATA, 'baseline', 'd8.py'),
'--second-d8', os.path.join(TEST_DATA, second_d8_dir, 'd8.py'),
'--first-config', 'ignition',
'--second-config', second_config,
os.path.join(TEST_DATA, 'fuzz-123.js'),
] + list(extra_flags))
class SystemTest(unittest.TestCase):
"""This tests the whole correctness-fuzzing harness with fake build
artifacts.
Overview of fakes:
baseline: Example foozzie output including a syntax error.
build1: Difference to baseline is a stack trace differece expected to
be suppressed.
build2: Difference to baseline is a non-suppressed output difference
causing the script to fail.
build3: As build1 but with an architecture difference as well.
"""
def testSyntaxErrorDiffPass(self):
stdout = run_foozzie('build1', '--skip-sanity-checks')
self.assertEquals('# V8 correctness - pass\n', cut_verbose_output(stdout))
# Default comparison doesn't include any specific mock files.
self.assertNotIn('v8_mock_archs.js', stdout)
self.assertNotIn('v8_mock_webassembly.js', stdout)
def testDifferentOutputFail(self):
with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
expected_output = f.read()
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build2', '--skip-sanity-checks',
'--first-config-extra-flags=--flag1',
'--first-config-extra-flags=--flag2=0',
'--second-config-extra-flags=--flag3')
e = ctx.exception
self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEquals(expected_output, cut_verbose_output(e.output))
def testSanityCheck(self):
with open(os.path.join(TEST_DATA, 'sanity_check_output.txt')) as f:
expected_output = f.read()
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build2')
e = ctx.exception
self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEquals(expected_output, e.output)
def testDifferentArch(self):
"""Test that the architecture-specific mocks are passed to both runs when
we use executables with different architectures.
"""
# Build 3 simulates x86, while the baseline is x64.
stdout = run_foozzie('build3', '--skip-sanity-checks')
lines = stdout.split('\n')
# TODO(machenbach): Don't depend on the command-lines being printed in
# particular lines.
self.assertIn('v8_mock_archs.js', lines[1])
self.assertIn('v8_mock_archs.js', lines[3])
def testJitless(self):
"""Test that webassembly is mocked out when comparing with jitless."""
stdout = run_foozzie(
'build1', '--skip-sanity-checks', second_config='jitless')
lines = stdout.split('\n')
# TODO(machenbach): Don't depend on the command-lines being printed in
# particular lines.
self.assertIn('v8_mock_webassembly.js', lines[1])
self.assertIn('v8_mock_webassembly.js', lines[3])
if __name__ == '__main__':
unittest.main()