AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity
7bcac055b5
v8/tools/clusterfuzz/foozzie/v8_foozzie_test.py
Michael Achenbach 5d50024ed4 [foozzie] Filter some contradictory flags 
Add logic to drop cyclic contradictory flags from
correctness-fuzzing command lines. Add the currently known
biggest offenders.

Without this, the correctness fuzzing harness runs into a CHECK
failure during smoke testing, when attempting to pass cyclic flags
to d8. It fails fast, but uselessly burns fuzzing time.

This change drops one of the known cyclic flags instead to make the
test run still useful. The precedence is right to left like in the
V8 test framework.

Additionally on Clusterfuzz, all crashes during smoke testing are
deduped as one crash report. We don't know if there are other
problems before this one is fixed/hidden.

No-Try: true
Bug: chromium:1330303
Change-Id: I06cbb4655cd3cf467f5cce6f84dba653834ca72e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3865562
Reviewed-by: Clemens Backes <clemensb@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Alexander Schulze <alexschulze@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82939}
2022-09-02 13:29:23 +00:00

395 lines
13 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import random
import subprocess
import sys
import unittest
import unittest.mock
import v8_commands
import v8_foozzie
import v8_fuzz_config
import v8_suppressions
try:
basestring
except NameError:
basestring = str
PYTHON3 = sys.version_info >= (3, 0)
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
TEST_DATA = os.path.join(BASE_DIR, 'testdata')
KNOWN_BUILDS = [
'd8',
'clang_x86/d8',
'clang_x86_v8_arm/d8',
'clang_x64_v8_arm64/d8',
'clang_x64_pointer_compression/d8',
]
class ConfigTest(unittest.TestCase):
def testExperiments(self):
"""Test integrity of probabilities and configs."""
CONFIGS = v8_foozzie.CONFIGS
EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
FLAGS = v8_fuzz_config.ADDITIONAL_FLAGS
# Probabilities add up to 100%.
first_is_int = lambda x: type(x[0]) == int
assert all(map(first_is_int, EXPERIMENTS))
assert sum(x[0] for x in EXPERIMENTS) == 100
# Configs used in experiments are defined.
assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
# The last config item points to a known build configuration.
assert all(map(lambda x: x[3] in KNOWN_BUILDS, EXPERIMENTS))
# All flags have a probability.
first_is_float = lambda x: type(x[0]) == float
assert all(map(first_is_float, FLAGS))
first_between_0_and_1 = lambda x: x[0] > 0 and x[0] < 1
assert all(map(first_between_0_and_1, FLAGS))
# Test consistent flags.
second_is_string = lambda x: isinstance(x[1], basestring)
assert all(map(second_is_string, FLAGS))
# We allow spaces to separate more flags. We don't allow spaces in the flag
# value.
is_flag = lambda x: x.startswith('--')
all_parts_are_flags = lambda x: all(map(is_flag, x[1].split()))
assert all(map(all_parts_are_flags, FLAGS))
def testConfig(self):
"""Smoke test how to choose experiments."""
config = v8_fuzz_config.Config('foo', random.Random(42))
experiments = [
[25, 'ignition', 'jitless', 'd8'],
[75, 'ignition', 'ignition', 'clang_x86/d8'],
]
flags = [
[0.1, '--flag'],
[0.3, '--baz'],
[0.3, '--foo --bar'],
]
self.assertEqual(
[
'--first-config=ignition',
'--second-config=jitless',
'--second-d8=d8',
'--second-config-extra-flags=--baz',
'--second-config-extra-flags=--foo',
'--second-config-extra-flags=--bar',
],
config.choose_foozzie_flags(experiments, flags),
)
self.assertEqual(
[
'--first-config=ignition',
'--second-config=jitless',
'--second-d8=d8',
],
config.choose_foozzie_flags(experiments, flags),
)
class UnitTest(unittest.TestCase):
def testCluster(self):
crash_test_example_path = 'CrashTests/path/to/file.js'
self.assertEqual(
v8_foozzie.ORIGINAL_SOURCE_DEFAULT,
v8_foozzie.cluster_failures(''))
self.assertEqual(
v8_foozzie.ORIGINAL_SOURCE_CRASHTESTS,
v8_foozzie.cluster_failures(crash_test_example_path))
self.assertEqual(
'_o_O_',
v8_foozzie.cluster_failures(
crash_test_example_path,
known_failures={crash_test_example_path: '_o_O_'}))
self.assertEqual(
'980',
v8_foozzie.cluster_failures('v8/test/mjsunit/apply.js'))
def testDiff(self):
def diff_fun(one, two, skip=False):
suppress = v8_suppressions.get_suppression(skip)
return suppress.diff_lines(one.splitlines(), two.splitlines())
one = ''
two = ''
diff = None, None
self.assertEqual(diff, diff_fun(one, two))
one = 'a \n b\nc();'
two = 'a \n b\nc();'
diff = None, None
self.assertEqual(diff, diff_fun(one, two))
# Ignore line before caret and caret position.
one = """
undefined
weird stuff
^
somefile.js: TypeError: suppressed message
undefined
"""
two = """
undefined
other weird stuff
^
somefile.js: TypeError: suppressed message
undefined
"""
diff = None, None
self.assertEqual(diff, diff_fun(one, two))
one = """
Still equal
Extra line
"""
two = """
Still equal
"""
diff = '- Extra line', None
self.assertEqual(diff, diff_fun(one, two))
one = """
Still equal
"""
two = """
Still equal
Extra line
"""
diff = '+ Extra line', None
self.assertEqual(diff, diff_fun(one, two))
one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
self.assertEqual(diff, diff_fun(one, two))
# Test that skipping suppressions works.
one = """
v8-foozzie source: foo
weird stuff
^
"""
two = """
v8-foozzie source: foo
other weird stuff
^
"""
self.assertEqual((None, 'foo'), diff_fun(one, two))
diff = ('- ^\n+ ^', 'foo')
self.assertEqual(diff, diff_fun(one, two, skip=True))
def testOutputCapping(self):
def output(stdout, is_crash):
exit_code = -1 if is_crash else 0
return v8_commands.Output(exit_code=exit_code, stdout=stdout, pid=0)
def check(stdout1, stdout2, is_crash1, is_crash2, capped_lines1,
capped_lines2):
output1 = output(stdout1, is_crash1)
output2 = output(stdout2, is_crash2)
self.assertEqual(
(capped_lines1, capped_lines2),
v8_suppressions.get_output_capped(output1, output2))
# No capping, already equal.
check('1\n2', '1\n2', True, True, '1\n2', '1\n2')
# No crash, no capping.
check('1\n2', '1\n2\n3', False, False, '1\n2', '1\n2\n3')
check('1\n2\n3', '1\n2', False, False, '1\n2\n3', '1\n2')
# Cap smallest if all runs crash.
check('1\n2', '1\n2\n3', True, True, '1\n2', '1\n2')
check('1\n2\n3', '1\n2', True, True, '1\n2', '1\n2')
check('1\n2', '1\n23', True, True, '1\n2', '1\n2')
check('1\n23', '1\n2', True, True, '1\n2', '1\n2')
# Cap the non-crashy run.
check('1\n2\n3', '1\n2', False, True, '1\n2', '1\n2')
check('1\n2', '1\n2\n3', True, False, '1\n2', '1\n2')
check('1\n23', '1\n2', False, True, '1\n2', '1\n2')
check('1\n2', '1\n23', True, False, '1\n2', '1\n2')
# The crashy run has more output.
check('1\n2\n3', '1\n2', True, False, '1\n2\n3', '1\n2')
check('1\n2', '1\n2\n3', False, True, '1\n2', '1\n2\n3')
check('1\n23', '1\n2', True, False, '1\n23', '1\n2')
check('1\n2', '1\n23', False, True, '1\n2', '1\n23')
# Keep output difference when capping.
check('1\n2', '3\n4\n5', True, True, '1\n2', '3\n4')
check('1\n2\n3', '4\n5', True, True, '1\n2', '4\n5')
check('12', '345', True, True, '12', '34')
check('123', '45', True, True, '12', '45')
@unittest.mock.patch('v8_foozzie.DISALLOWED_FLAGS', ['A'])
@unittest.mock.patch('v8_foozzie.CONTRADICTORY_FLAGS',
[('B', 'C'), ('B', 'D')])
def testFilterFlags(self):
def check(input_flags, expected):
self.assertEqual(expected, v8_foozzie.filter_flags(input_flags))
check([], [])
check(['A'], [])
check(['D', 'A'], ['D'])
check(['A', 'D'], ['D'])
check(['C', 'D'], ['C', 'D'])
check(['E', 'C', 'D', 'F'], ['E', 'C', 'D', 'F'])
check(['B', 'D'], ['D'])
check(['D', 'B'], ['B'])
check(['C', 'B', 'D'], ['C', 'D'])
check(['E', 'C', 'A', 'F', 'B', 'G', 'D'], ['E', 'C', 'F', 'G', 'D'])
def cut_verbose_output(stdout, n_comp):
# This removes the first lines containing d8 commands of `n_comp` comparison
# runs.
return '\n'.join(stdout.split('\n')[n_comp * 2:])
def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
second_config = 'ignition_turbo'
if 'second_config' in kwargs:
second_config = 'jitless'
kwargs = {}
if PYTHON3:
kwargs['text'] = True
return subprocess.check_output([
sys.executable, FOOZZIE,
'--random-seed', '12345',
'--first-d8', os.path.join(TEST_DATA, 'baseline', 'd8.py'),
'--second-d8', os.path.join(TEST_DATA, second_d8_dir, 'd8.py'),
'--first-config', 'ignition',
'--second-config', second_config,
os.path.join(TEST_DATA, 'fuzz-123.js'),
] + list(extra_flags), **kwargs)
class SystemTest(unittest.TestCase):
"""This tests the whole correctness-fuzzing harness with fake build
artifacts.
Overview of fakes:
baseline: Example foozzie output including a syntax error.
build1: Difference to baseline is a stack trace difference expected to
be suppressed.
build2: Difference to baseline is a non-suppressed output difference
causing the script to fail.
build3: As build1 but with an architecture difference as well.
"""
def testSyntaxErrorDiffPass(self):
stdout = run_foozzie('build1', '--skip-smoke-tests')
self.assertEqual('# V8 correctness - pass\n',
cut_verbose_output(stdout, 3))
# Default comparison includes suppressions.
self.assertIn('v8_suppressions.js', stdout)
# Default comparison doesn't include any specific mock files.
self.assertNotIn('v8_mock_archs.js', stdout)
self.assertNotIn('v8_mock_webassembly.js', stdout)
def testDifferentOutputFail(self):
with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
expected_output = f.read()
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build2', '--skip-smoke-tests',
'--first-config-extra-flags=--flag1',
'--first-config-extra-flags=--flag2=0',
'--second-config-extra-flags=--flag3')
e = ctx.exception
self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEqual(expected_output, cut_verbose_output(e.output, 2))
def testSmokeTest(self):
with open(os.path.join(TEST_DATA, 'smoke_test_output.txt')) as f:
expected_output = f.read()
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build2')
e = ctx.exception
self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEqual(expected_output, e.output)
def testDifferentArch(self):
"""Test that the architecture-specific mocks are passed to both runs when
we use executables with different architectures.
"""
# Build 3 simulates x86, while the baseline is x64.
stdout = run_foozzie('build3', '--skip-smoke-tests')
lines = stdout.split('\n')
# TODO(machenbach): Don't depend on the command-lines being printed in
# particular lines.
self.assertIn('v8_mock_archs.js', lines[1])
self.assertIn('v8_mock_archs.js', lines[3])
def testDifferentArchFailFirst(self):
"""Test that we re-test against x64. This tests the path that also fails
on x64 and then reports the error as x64.
"""
with open(os.path.join(TEST_DATA, 'failure_output_arch.txt')) as f:
expected_output = f.read()
# Build 3 simulates x86 and produces a difference on --bad-flag, but
# the baseline build shows the same difference when --bad-flag is passed.
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build3', '--skip-smoke-tests',
'--second-config-extra-flags=--bad-flag')
e = ctx.exception
self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEqual(expected_output, cut_verbose_output(e.output, 3))
def testDifferentArchFailSecond(self):
"""As above, but we test the path that only fails in the second (ia32)
run and not with x64 and then reports the error as ia32.
"""
with open(os.path.join(TEST_DATA, 'failure_output_second.txt')) as f:
expected_output = f.read()
# Build 3 simulates x86 and produces a difference on --very-bad-flag,
# which the baseline build doesn't.
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie('build3', '--skip-smoke-tests',
'--second-config-extra-flags=--very-bad-flag')
e = ctx.exception
self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertEqual(expected_output, cut_verbose_output(e.output, 3))
def testJitless(self):
"""Test that webassembly is mocked out when comparing with jitless."""
stdout = run_foozzie(
'build1', '--skip-smoke-tests', second_config='jitless')
lines = stdout.split('\n')
# TODO(machenbach): Don't depend on the command-lines being printed in
# particular lines.
self.assertIn('v8_mock_webassembly.js', lines[1])
self.assertIn('v8_mock_webassembly.js', lines[3])
def testSkipSuppressions(self):
"""Test that the suppressions file is not passed when skipping
suppressions.
"""
# Compare baseline with baseline. This passes as there is no difference.
stdout = run_foozzie(
'baseline', '--skip-smoke-tests', '--skip-suppressions')
self.assertNotIn('v8_suppressions.js', stdout)
# Compare with a build that usually suppresses a difference. Now we fail
# since we skip suppressions.
with self.assertRaises(subprocess.CalledProcessError) as ctx:
run_foozzie(
'build1', '--skip-smoke-tests', '--skip-suppressions')
e = ctx.exception
self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
self.assertNotIn('v8_suppressions.js', e.output)
if __name__ == '__main__':
unittest.main()
Reference in New Issue View Git Blame Copy Permalink