v8/tools/clusterfuzz/v8_foozzie_test.py

#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import random
import subprocess
import sys
import unittest

import v8_commands
import v8_foozzie
import v8_fuzz_config
import v8_suppressions

try:
  basestring
except NameError:
  basestring = str

PYTHON3 = sys.version_info >= (3, 0)

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
TEST_DATA = os.path.join(BASE_DIR, 'testdata')

KNOWN_BUILDS = [
  'd8',
  'clang_x86/d8',
  'clang_x86_v8_arm/d8',
  'clang_x64_v8_arm64/d8',
  'clang_x64_pointer_compression/d8',
]


class ConfigTest(unittest.TestCase):
  def testExperiments(self):
    """Test integrity of probabilities and configs."""
    CONFIGS = v8_foozzie.CONFIGS
    EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
    FLAGS = v8_fuzz_config.ADDITIONAL_FLAGS
    # Probabilities add up to 100%.
    first_is_int = lambda x: type(x[0]) == int
    assert all(map(first_is_int, EXPERIMENTS))
    assert sum(x[0] for x in EXPERIMENTS) == 100
    # Configs used in experiments are defined.
    assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
    assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
    # The last config item points to a known build configuration.
    assert all(map(lambda x: x[3] in KNOWN_BUILDS, EXPERIMENTS))
    # All flags have a probability.
    first_is_float = lambda x: type(x[0]) == float
    assert all(map(first_is_float, FLAGS))
    first_between_0_and_1 = lambda x: x[0] > 0 and x[0] < 1
    assert all(map(first_between_0_and_1, FLAGS))
    # Test consistent flags.
    second_is_string = lambda x: isinstance(x[1], basestring)
    assert all(map(second_is_string, FLAGS))
    # We allow spaces to separate more flags. We don't allow spaces in the flag
    # value.
    is_flag = lambda x: x.startswith('--')
    all_parts_are_flags = lambda x: all(map(is_flag, x[1].split()))
    assert all(map(all_parts_are_flags, FLAGS))

  def testConfig(self):
    """Smoke test how to choose experiments."""
    config = v8_fuzz_config.Config('foo', random.Random(42))
    experiments = [
      [25, 'ignition', 'jitless', 'd8'],
      [75, 'ignition', 'ignition', 'clang_x86/d8'],
    ]
    flags = [
      [0.1, '--flag'],
      [0.3, '--baz'],
      [0.3, '--foo --bar'],
    ]
    self.assertEqual(
        [
          '--first-config=ignition',
          '--second-config=jitless',
          '--second-d8=d8',
          '--second-config-extra-flags=--baz',
          '--second-config-extra-flags=--foo',
          '--second-config-extra-flags=--bar',
        ],
        config.choose_foozzie_flags(experiments, flags),
    )
    self.assertEqual(
        [
          '--first-config=ignition',
          '--second-config=jitless',
          '--second-d8=d8',
        ],
        config.choose_foozzie_flags(experiments, flags),
    )


class UnitTest(unittest.TestCase):
  def testCluster(self):
    crash_test_example_path = 'CrashTests/path/to/file.js'
    self.assertEqual(
        v8_foozzie.ORIGINAL_SOURCE_DEFAULT,
        v8_foozzie.cluster_failures(''))
    self.assertEqual(
        v8_foozzie.ORIGINAL_SOURCE_CRASHTESTS,
        v8_foozzie.cluster_failures(crash_test_example_path))
    self.assertEqual(
        '_o_O_',
        v8_foozzie.cluster_failures(
            crash_test_example_path,
            known_failures={crash_test_example_path: '_o_O_'}))
    self.assertEqual(
        '980',
        v8_foozzie.cluster_failures('v8/test/mjsunit/apply.js'))

  def testDiff(self):
    def diff_fun(one, two, skip=False):
      suppress = v8_suppressions.get_suppression(skip)
      return suppress.diff_lines(one.splitlines(), two.splitlines())

    one = ''
    two = ''
    diff = None, None
    self.assertEqual(diff, diff_fun(one, two))

    one = 'a \n  b\nc();'
    two = 'a \n  b\nc();'
    diff = None, None
    self.assertEqual(diff, diff_fun(one, two))

    # Ignore line before caret and caret position.
    one = """
undefined
weird stuff
      ^
somefile.js: TypeError: suppressed message
  undefined
"""
    two = """
undefined
other weird stuff
            ^
somefile.js: TypeError: suppressed message
  undefined
"""
    diff = None, None
    self.assertEqual(diff, diff_fun(one, two))

    one = """
Still equal
Extra line
"""
    two = """
Still equal
"""
    diff = '- Extra line', None
    self.assertEqual(diff, diff_fun(one, two))

    one = """
Still equal
"""
    two = """
Still equal
Extra line
"""
    diff = '+ Extra line', None
    self.assertEqual(diff, diff_fun(one, two))

    one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
    two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
    diff = """- somefile.js: TypeError: undefined is not a constructor
+ otherfile.js: TypeError: undefined is not a constructor""", None
    self.assertEqual(diff, diff_fun(one, two))

    # Test that skipping suppressions works.
    one = """
v8-foozzie source: foo
weird stuff
      ^
"""
    two = """
v8-foozzie source: foo
other weird stuff
            ^
"""
    self.assertEqual((None, 'foo'), diff_fun(one, two))
    diff = ('-       ^\n+             ^', 'foo')
    self.assertEqual(diff, diff_fun(one, two, skip=True))

  def testOutputCapping(self):
    def output(stdout, is_crash):
      exit_code = -1 if is_crash else 0
      return v8_commands.Output(exit_code=exit_code, stdout=stdout, pid=0)

    def check(stdout1, stdout2, is_crash1, is_crash2, capped_lines1,
              capped_lines2):
      output1 = output(stdout1, is_crash1)
      output2 = output(stdout2, is_crash2)
      self.assertEqual(
          (capped_lines1, capped_lines2),
          v8_suppressions.get_output_capped(output1, output2))

    # No capping, already equal.
    check('1\n2', '1\n2', True, True, '1\n2', '1\n2')
    # No crash, no capping.
    check('1\n2', '1\n2\n3', False, False, '1\n2', '1\n2\n3')
    check('1\n2\n3', '1\n2', False, False, '1\n2\n3', '1\n2')
    # Cap smallest if all runs crash.
    check('1\n2', '1\n2\n3', True, True, '1\n2', '1\n2')
    check('1\n2\n3', '1\n2', True, True, '1\n2', '1\n2')
    check('1\n2', '1\n23', True, True, '1\n2', '1\n2')
    check('1\n23', '1\n2', True, True, '1\n2', '1\n2')
    # Cap the non-crashy run.
    check('1\n2\n3', '1\n2', False, True, '1\n2', '1\n2')
    check('1\n2', '1\n2\n3', True, False, '1\n2', '1\n2')
    check('1\n23', '1\n2', False, True, '1\n2', '1\n2')
    check('1\n2', '1\n23', True, False, '1\n2', '1\n2')
    # The crashy run has more output.
    check('1\n2\n3', '1\n2', True, False, '1\n2\n3', '1\n2')
    check('1\n2', '1\n2\n3', False, True, '1\n2', '1\n2\n3')
    check('1\n23', '1\n2', True, False, '1\n23', '1\n2')
    check('1\n2', '1\n23', False, True, '1\n2', '1\n23')
    # Keep output difference when capping.
    check('1\n2', '3\n4\n5', True, True, '1\n2', '3\n4')
    check('1\n2\n3', '4\n5', True, True, '1\n2', '4\n5')
    check('12', '345', True, True, '12', '34')
    check('123', '45', True, True, '12', '45')


def cut_verbose_output(stdout, n_comp):
  # This removes the first lines containing d8 commands of `n_comp` comparison
  # runs.
  return '\n'.join(stdout.split('\n')[n_comp * 2:])


def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
  second_config = 'ignition_turbo'
  if 'second_config' in kwargs:
    second_config = 'jitless'
  kwargs = {}
  if PYTHON3:
    kwargs['text'] = True
  return subprocess.check_output([
    sys.executable, FOOZZIE,
    '--random-seed', '12345',
    '--first-d8', os.path.join(TEST_DATA, 'baseline', 'd8.py'),
    '--second-d8', os.path.join(TEST_DATA, second_d8_dir, 'd8.py'),
    '--first-config', 'ignition',
    '--second-config', second_config,
    os.path.join(TEST_DATA, 'fuzz-123.js'),
  ] + list(extra_flags), **kwargs)

class SystemTest(unittest.TestCase):
  """This tests the whole correctness-fuzzing harness with fake build
  artifacts.

  Overview of fakes:
    baseline: Example foozzie output including a syntax error.
    build1: Difference to baseline is a stack trace difference expected to
            be suppressed.
    build2: Difference to baseline is a non-suppressed output difference
            causing the script to fail.
    build3: As build1 but with an architecture difference as well.
  """
  def testSyntaxErrorDiffPass(self):
    stdout = run_foozzie('build1', '--skip-smoke-tests')
    self.assertEqual('# V8 correctness - pass\n',
                     cut_verbose_output(stdout, 3))
    # Default comparison includes suppressions.
    self.assertIn('v8_suppressions.js', stdout)
    # Default comparison doesn't include any specific mock files.
    self.assertNotIn('v8_mock_archs.js', stdout)
    self.assertNotIn('v8_mock_webassembly.js', stdout)

  def testDifferentOutputFail(self):
    with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
      expected_output = f.read()
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie('build2', '--skip-smoke-tests',
                  '--first-config-extra-flags=--flag1',
                  '--first-config-extra-flags=--flag2=0',
                  '--second-config-extra-flags=--flag3')
    e = ctx.exception
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertEqual(expected_output, cut_verbose_output(e.output, 2))

  def testSmokeTest(self):
    with open(os.path.join(TEST_DATA, 'smoke_test_output.txt')) as f:
      expected_output = f.read()
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie('build2')
    e = ctx.exception
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertEqual(expected_output, e.output)

  def testDifferentArch(self):
    """Test that the architecture-specific mocks are passed to both runs when
    we use executables with different architectures.
    """
    # Build 3 simulates x86, while the baseline is x64.
    stdout = run_foozzie('build3', '--skip-smoke-tests')
    lines = stdout.split('\n')
    # TODO(machenbach): Don't depend on the command-lines being printed in
    # particular lines.
    self.assertIn('v8_mock_archs.js', lines[1])
    self.assertIn('v8_mock_archs.js', lines[3])

  def testDifferentArchFailFirst(self):
    """Test that we re-test against x64. This tests the path that also fails
    on x64 and then reports the error as x64.
    """
    with open(os.path.join(TEST_DATA, 'failure_output_arch.txt')) as f:
      expected_output = f.read()
    # Build 3 simulates x86 and produces a difference on --bad-flag, but
    # the baseline build shows the same difference when --bad-flag is passed.
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie('build3', '--skip-smoke-tests',
                  '--second-config-extra-flags=--bad-flag')
    e = ctx.exception
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertEqual(expected_output, cut_verbose_output(e.output, 3))

  def testDifferentArchFailSecond(self):
    """As above, but we test the path that only fails in the second (ia32)
    run and not with x64 and then reports the error as ia32.
    """
    with open(os.path.join(TEST_DATA, 'failure_output_second.txt')) as f:
      expected_output = f.read()
    # Build 3 simulates x86 and produces a difference on --very-bad-flag,
    # which the baseline build doesn't.
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie('build3', '--skip-smoke-tests',
                  '--second-config-extra-flags=--very-bad-flag')
    e = ctx.exception
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertEqual(expected_output, cut_verbose_output(e.output, 3))

  def testJitless(self):
    """Test that webassembly is mocked out when comparing with jitless."""
    stdout = run_foozzie(
        'build1', '--skip-smoke-tests', second_config='jitless')
    lines = stdout.split('\n')
    # TODO(machenbach): Don't depend on the command-lines being printed in
    # particular lines.
    self.assertIn('v8_mock_webassembly.js', lines[1])
    self.assertIn('v8_mock_webassembly.js', lines[3])

  def testSkipSuppressions(self):
    """Test that the suppressions file is not passed when skipping
    suppressions.
    """
    # Compare baseline with baseline. This passes as there is no difference.
    stdout = run_foozzie(
        'baseline', '--skip-smoke-tests', '--skip-suppressions')
    self.assertNotIn('v8_suppressions.js', stdout)

    # Compare with a build that usually suppresses a difference. Now we fail
    # since we skip suppressions.
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie(
          'build1', '--skip-smoke-tests', '--skip-suppressions')
    e = ctx.exception
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertNotIn('v8_suppressions.js', e.output)


if __name__ == '__main__':
  unittest.main()