[foozzie] Initial correctness fuzzer harness.

Initial version of the correctness fuzzer harness for manual testing
and experiments.

For automated usage, some outstanding TODOs are left in the code. E.g.
- Hash source file names in error case
- Bundle script in out directory with executables
- Some suppressions are tied to already fixed bugs. We'll keep it like that for now to test
removing those suppressions in production later.

BUG=chromium:673246
NOTRY=true

Review-Url: https://codereview.chromium.org/2578503003
Cr-Commit-Position: refs/heads/master@{#41789}

tools/foozzie/v8_commands.py

@@ -0,0 +1,64 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Fork from commands.py and output.py in v8 test driver.
+
+import signal
+import subprocess
+import sys
+from threading import Event, Timer
+
+
+class Output(object):
+  def __init__(self, exit_code, timed_out, stdout, pid):
+    self.exit_code = exit_code
+    self.timed_out = timed_out
+    self.stdout = stdout
+    self.pid = pid
+
+  def HasCrashed(self):
+    # Timed out tests will have exit_code -signal.SIGTERM.
+    if self.timed_out:
+      return False
+    return (self.exit_code < 0 and
+            self.exit_code != -signal.SIGABRT)
+
+  def HasTimedOut(self):
+    return self.timed_out
+
+
+def Execute(args, cwd, timeout=None):
+  popen_args = [c for c in args if c != ""]
+  try:
+    process = subprocess.Popen(
+      args=popen_args,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.STDOUT,
+      cwd=cwd
+    )
+  except Exception as e:
+    sys.stderr.write("Error executing: %s\n" % popen_args)
+    raise e
+
+  timeout_event = Event()
+
+  def kill_process():
+    timeout_event.set()
+    try:
+      process.kill()
+    except OSError:
+      sys.stderr.write('Error: Process %s already ended.\n' % process.pid)
+
+
+  timer = Timer(timeout, kill_process)
+  timer.start()
+  stdout, _ = process.communicate()
+  timer.cancel()
+
+  return Output(
+      process.returncode,
+      timeout_event.is_set(),
+      stdout.decode('utf-8', 'replace').encode('utf-8'),
+      process.pid,
+  )

tools/foozzie/v8_foozzie.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+V8 correctness fuzzer launcher script.
+"""
+
+import argparse
+import itertools
+import os
+import re
+import sys
+import traceback
+
+import v8_commands
+import v8_suppressions
+
+CONFIGS = dict(
+  default=[],
+  validate_asm=['--validate-asm'], # Maybe add , '--disable-asm-warnings'
+  fullcode=['--nocrankshaft', '--turbo-filter=~'],
+  noturbo=['--turbo-filter=~', '--noturbo-asm'],
+  noturbo_opt=['--always-opt', '--turbo-filter=~', '--noturbo-asm'],
+  ignition_staging=['--ignition-staging'],
+  ignition_turbo=['--ignition-staging', '--turbo'],
+  ignition_turbo_opt=['--ignition-staging', '--turbo', '--always-opt'],
+)
+
+# Timeout in seconds for one d8 run.
+TIMEOUT = 3
+
+# Return codes.
+RETURN_PASS = 0
+RETURN_FAILURE = 2
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+PREAMBLE = [
+  os.path.join(BASE_PATH, 'v8_mock.js'),
+  os.path.join(BASE_PATH, 'v8_suppressions.js'),
+]
+
+FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
+         '--invoke-weak-callbacks', '--omit-quit', '--es-staging']
+
+SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
+
+# Output for suppressed failure case.
+FAILURE_HEADER_TEMPLATE = """
+#
+# V8 correctness failure
+# V8 correctness configs: %(configs)s
+# V8 correctness sources: %(sources)s
+# V8 correctness suppression: %(suppression)s""".strip()
+
+# Extended output for failure case. The 'CHECK' is for the minimizer.
+FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """
+#
+# CHECK
+#
+# Compared %(first_config_label)s with %(second_config_label)s
+#
+# Flags of %(first_config_label)s:
+%(first_config_flags)s
+# Flags of %(second_config_label)s:
+%(second_config_flags)s
+#
+# Difference:
+%(difference)s
+#
+### Start of configuration %(first_config_label)s:
+%(first_config_output)s
+### End of configuration %(first_config_label)s
+#
+### Start of configuration %(second_config_label)s:
+%(second_config_output)s
+### End of configuration %(second_config_label)s
+""".strip()
+
+
+def parse_args():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    '--random-seed', type=int, required=True,
+    help='random seed passed to both runs')
+  parser.add_argument(
+      '--first-arch', help='first architecture', default='x64')
+  parser.add_argument(
+      '--second-arch', help='second architecture', default='x64')
+  parser.add_argument(
+      '--first-config', help='first configuration', default='fullcode')
+  parser.add_argument(
+      '--second-config', help='second configuration', default='fullcode')
+  parser.add_argument(
+      '--first-d8', default='d8',
+      help='optional path to first d8 executable, '
+           'default: bundled in the same directory as this script')
+  parser.add_argument(
+      '--second-d8',
+      help='optional path to second d8 executable, default: same as first')
+  parser.add_argument('testcase', help='path to test case')
+  options = parser.parse_args()
+
+  # Ensure we make a sane comparison.
+  assert (options.first_arch != options.second_arch or
+          options.first_config != options.second_config) , (
+      'Need either arch or config difference.')
+  assert options.first_arch in SUPPORTED_ARCHS
+  assert options.second_arch in SUPPORTED_ARCHS
+  assert options.first_config in CONFIGS
+  assert options.second_config in CONFIGS
+
+  # Ensure we have a test case.
+  assert (os.path.exists(options.testcase) and
+          os.path.isfile(options.testcase)), (
+      'Test case %s doesn\'t exist' % options.testcase)
+
+  # Use first d8 as default for second d8.
+  options.second_d8 = options.second_d8 or options.first_d8
+
+  # Ensure absolute paths.
+  options.first_d8 = os.path.abspath(options.first_d8)
+  options.second_d8 = os.path.abspath(options.second_d8)
+
+  # Ensure executables exist.
+  assert os.path.exists(options.first_d8)
+  assert os.path.exists(options.second_d8)
+
+  # Ensure we use different executables when we claim we compare
+  # different architectures.
+  # TODO(machenbach): Infer arch from gn's build output.
+  if options.first_arch != options.second_arch:
+    assert options.first_d8 != options.second_d8
+
+  return options
+
+
+def test_pattern_bailout(testcase, ignore_fun):
+  """Print failure state and return if ignore_fun matches testcase."""
+  with open(testcase) as f:
+    bug = (ignore_fun(f.read()) or '').strip()
+    if bug:
+      print FAILURE_HEADER_TEMPLATE % dict(
+          configs='', sources='', suppression=bug)
+      return True
+  return False
+
+
+def pass_bailout(output, step_number):
+  """Print info and return if in timeout or crash pass states."""
+  if output.HasTimedOut():
+    # Dashed output, so that no other clusterfuzz tools can match the
+    # words timeout or crash.
+    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
+    return True
+  if output.HasCrashed():
+    print '# V8 correctness - C-R-A-S-H %d' % step_number
+    return True
+  return False
+
+
+def fail_bailout(output, ignore_by_output_fun):
+  """Print failure state and return if ignore_by_output_fun matches output."""
+  bug = (ignore_by_output_fun(output.stdout) or '').strip()
+  if bug:
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression=bug)
+    return True
+  return False
+
+
+def main():
+  options = parse_args()
+
+  # Suppressions are architecture and configuration specific.
+  suppress = v8_suppressions.get_suppression(
+      options.first_arch, options.first_config,
+      options.second_arch, options.second_config,
+  )
+
+  if test_pattern_bailout(options.testcase, suppress.ignore):
+    return RETURN_FAILURE
+
+  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
+  first_config_flags = common_flags + CONFIGS[options.first_config]
+  second_config_flags = common_flags + CONFIGS[options.second_config]
+
+  def run_d8(d8, config_flags):
+    return v8_commands.Execute(
+        [d8] + config_flags + PREAMBLE + [options.testcase],
+        cwd=os.path.dirname(options.testcase),
+        timeout=TIMEOUT,
+    )
+
+  first_config_output = run_d8(options.first_d8, first_config_flags)
+
+  # Early bailout based on first run's output.
+  if pass_bailout(first_config_output, 1):
+    return RETURN_PASS
+  if fail_bailout(first_config_output, suppress.ignore_by_output1):
+    return RETURN_FAILURE
+
+  second_config_output = run_d8(options.second_d8, second_config_flags)
+
+  # Bailout based on second run's output.
+  if pass_bailout(second_config_output, 2):
+    return RETURN_PASS
+  if fail_bailout(second_config_output, suppress.ignore_by_output2):
+    return RETURN_FAILURE
+
+  difference = suppress.diff(
+      first_config_output.stdout, second_config_output.stdout)
+  if difference:
+    # The first three entries will be parsed by clusterfuzz. Format changes
+    # will require changes on the clusterfuzz side.
+    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
+    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
+    print FAILURE_TEMPLATE % dict(
+        configs='%s:%s' % (first_config_label, second_config_label),
+        sources='', # TODO
+        suppression='', # We can't tie bugs to differences.
+        first_config_label=first_config_label,
+        second_config_label=second_config_label,
+        first_config_flags=' '.join(first_config_flags),
+        second_config_flags=' '.join(second_config_flags),
+        first_config_output=first_config_output.stdout,
+        second_config_output=second_config_output.stdout,
+        difference=difference,
+    )
+    return RETURN_FAILURE
+
+  # TODO(machenbach): Figure out if we could also return a bug in case there's
+  # no difference, but one of the line suppressions has matched - and without
+  # the match there would be a difference.
+
+  print '# V8 correctness - pass'
+  return RETURN_PASS
+
+
+if __name__ == "__main__":
+  try:
+    result = main()
+  except SystemExit:
+    # Make sure clusterfuzz reports internal errors and wrong usage.
+    # Use one label for all internal and usage errors.
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression='wrong_usage')
+    result = RETURN_FAILURE
+  except Exception as e:
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression='internal_error')
+    print '# Internal error: %s' % e
+    traceback.print_exc(file=sys.stdout)
+    result = RETURN_FAILURE
+
+  sys.exit(result)

tools/foozzie/v8_foozzie_test.py

@@ -0,0 +1,77 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+import v8_suppressions
+
+class FuzzerTest(unittest.TestCase):
+  def testDiff(self):
+    # TODO(machenbach): Mock out suppression configuration.
+    suppress = v8_suppressions.get_suppression(
+        'x64', 'fullcode', 'x64', 'default')
+    one = ''
+    two = ''
+    diff = None
+    self.assertEquals(diff, suppress.diff(one, two))
+
+    one = 'a \n  b\nc();'
+    two = 'a \n  b\nc();'
+    diff = None
+    self.assertEquals(diff, suppress.diff(one, two))
+
+    # Ignore line before caret, caret position, stack trace char numbers
+    # error message and validator output.
+    one = """
+undefined
+weird stuff
+      ^
+Validation of asm.js module failed: foo bar
+somefile.js: TypeError: undefined is not a function
+stack line :15: foo
+  undefined
+"""
+    two = """
+undefined
+other weird stuff
+            ^
+somefile.js: TypeError: baz is not a function
+stack line :2: foo
+Validation of asm.js module failed: baz
+  undefined
+"""
+    diff = None
+    self.assertEquals(diff, suppress.diff(one, two))
+
+    one = """
+Still equal
+Extra line
+"""
+    two = """
+Still equal
+"""
+    diff = '- Extra line'
+    self.assertEquals(diff, suppress.diff(one, two))
+
+    one = """
+Still equal
+"""
+    two = """
+Still equal
+Extra line
+"""
+    diff = '+ Extra line'
+    self.assertEquals(diff, suppress.diff(one, two))
+
+    one = """
+undefined
+somefile.js: TypeError: undefined is not a constructor
+"""
+    two = """
+undefined
+otherfile.js: TypeError: undefined is not a constructor
+"""
+    diff = """- somefile.js: TypeError: undefined is not a constructor
++ otherfile.js: TypeError: undefined is not a constructor"""
+    self.assertEquals(diff, suppress.diff(one, two))

tools/foozzie/v8_mock.js

@@ -0,0 +1,76 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This is intended for permanent JS behavior changes for mocking out
+// non-deterministic behavior. For temporary suppressions, please refer to
+// v8_suppressions.js.
+// This file is loaded before each correctness test cases and won't get
+// minimized.
+
+
+// This will be overridden in the test cases. The override can be minimized.
+var __PrettyPrint = function __PrettyPrint(msg) { print(msg); };
+
+
+// All calls to f.arguments are replaced by f.mock_arguments by an external
+// script.
+Object.prototype.mock_arguments = ['x', 'y']
+
+
+// Mock Math.random.
+var __magic_index_for_mocked_random = 0
+Math.random = function(){
+  __magic_index_for_mocked_random = (__magic_index_for_mocked_random + 1) % 10
+  return __magic_index_for_mocked_random / 10.0;
+}
+
+
+// Mock Date.
+var __magic_index_for_mocked_date = 0
+var __magic_mocked_date = 1477662728696
+__magic_mocked_date_now = function(){
+  __magic_index_for_mocked_date = (__magic_index_for_mocked_date + 1) % 10
+  __magic_mocked_date = __magic_mocked_date + __magic_index_for_mocked_date + 1
+  return __magic_mocked_date
+}
+
+var __original_date = Date;
+__magic_mock_date_handler = {
+  construct: function(target, args, newTarget) {
+    if (args.length > 0) {
+      return new (Function.prototype.bind.apply(__original_date, [null].concat(args)));
+    } else {
+      return new __original_date(__magic_mocked_date_now());
+    }
+  },
+  get: function(target, property, receiver) {
+    if (property == "now") {
+      return __magic_mocked_date_now;
+    }
+  },
+}
+
+Date = new Proxy(Date, __magic_mock_date_handler);
+
+// Mock Worker.
+var __magic_index_for_mocked_worker = 0
+// TODO(machenbach): Randomize this for each test case, but keep stable during
+// comparison. Also data and random above.
+var __magic_mocked_worker_messages = [
+  undefined, 0, -1, "", "foo", 42, [], {}, [0], {"x": 0}
+]
+Worker = function(code){
+  try {
+    __PrettyPrint(eval(code));
+  } catch(e) {
+    __PrettyPrint(e);
+  }
+  this.getMessage = function(){
+    __magic_index_for_mocked_worker = (__magic_index_for_mocked_worker + 1) % 10
+    return __magic_mocked_worker_messages[__magic_index_for_mocked_worker];
+  }
+  this.postMessage = function(msg){
+    __PrettyPrint(msg);
+  }
+}

tools/foozzie/v8_suppressions.js

@@ -0,0 +1,21 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file is loaded before each correctness test case and after v8_mock.js.
+// You can temporarily change JS behavior here to silence known problems.
+// Please refer to a bug in a comment and remove the suppression once the
+// problem is fixed.
+
+// Suppress http://crbug.com/662429
+var __real_Math_pow = Math.pow
+Math.pow = function(a, b){
+  if (b < 0) {
+    return 0.000017;
+  } else {
+    return __real_Math_pow(a, b);
+  }
+}
+
+// Suppress http://crbug.com/663750
+Object.freeze = function(o){ print (__PrettyPrint(o)); }

tools/foozzie/v8_suppressions.py

@@ -0,0 +1,266 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Suppressions for V8 correctness fuzzer failures.
+
+We support three types of suppressions:
+1. Ignore test case by pattern.
+Map a regular expression to a bug entry. A new failure will be reported
+when the pattern matches a JS test case.
+Subsequent matches will be recoreded under the first failure.
+
+2. Ignore test run by output pattern:
+Map a regular expression to a bug entry. A new failure will be reported
+when the pattern matches the output of a particular run.
+Subsequent matches will be recoreded under the first failure.
+
+3. Relax line-to-line comparisons with expressions of lines to ignore and
+lines to be normalized (i.e. ignore only portions of lines).
+These are not tied to bugs, be careful to not silently switch off this tool!
+
+Alternatively, think about adding a behavior change to v8_suppressions.js
+to silence a particular class of problems.
+"""
+
+import itertools
+import re
+
+# Max line length for regular experessions checking for lines to ignore.
+MAX_LINE_LENGTH = 512
+
+# For ignoring lines before carets and to ignore caret positions.
+CARET_RE = re.compile(r'^\s*\^\s*$')
+
+# Ignore by test case pattern. Map from bug->regexp.
+# Regular expressions are assumed to be compiled. We use regexp.match.
+IGNORE_TEST_CASES = {
+  'crbug.com/662907':
+      re.compile(r'.*new Array.*\[\d+\] =.*'
+                 r'((Array)|(Object)).prototype.__defineSetter__.*', re.S),
+
+  'crbug.com/663340':
+      re.compile(r'.*\.shift\(\).*', re.S),
+
+  'crbug.com/666308':
+      re.compile(r'.*End stripped down and modified version.*'
+                 r'\.prototype.*instanceof.*.*', re.S),
+}
+
+# Ignore by output pattern. Map from config->bug->regexp. Config '' is used
+# to match all configurations. Otherwise use either a compiler configuration,
+# e.g. fullcode or validate_asm or an architecture, e.g. x64 or ia32 or a
+# comma-separated combination, e.g. x64,fullcode, for more specific
+# suppressions.
+# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
+# label.
+# Regular expressions are assumed to be compiled. We use regexp.search.
+IGNORE_OUTPUT = {
+  '': {
+    'crbug.com/664068':
+        re.compile(r'RangeError', re.S),
+
+    'crbug.com/669017':
+        re.compile(r'SyntaxError', re.S),
+  },
+  'validate_asm': {
+    'validate_asm':
+        re.compile(r'TypeError'),
+  },
+}
+
+# Lines matching any of the following regular expressions will be ignored
+# if appearing on both sides. The capturing groups need to match exactly.
+# Use uncompiled regular expressions - they'll be compiled later.
+ALLOWED_LINE_DIFFS = [
+  # Ignore caret position in stack traces.
+  r'^\s*\^\s*$',
+
+  # Ignore some stack trace headers as messages might not match.
+  r'^(.*)TypeError: .* is not a function$',
+  r'^(.*)TypeError: .* is not a constructor$',
+  r'^(.*)TypeError: (.*) is not .*$',
+  r'^(.*)ReferenceError: .* is not defined$',
+  r'^(.*):\d+: ReferenceError: .* is not defined$',
+
+  # These are rarely needed. It includes some cases above.
+  r'^\w*Error: .* is not .*$',
+  r'^(.*) \w*Error: .* is not .*$',
+  r'^(.*):\d+: \w*Error: .* is not .*$',
+
+  # Some test cases just print the message.
+  r'^.* is not a function(.*)$',
+  r'^(.*) is not a .*$',
+
+  # crbug.com/669017
+  r'^(.*)SyntaxError: .*$',
+
+  # Ignore lines of stack traces as character positions might not match.
+  r'^    at (?:new )?([^:]*):\d+:\d+(.*)$',
+  r'^(.*):\d+:(.*)$',
+
+  # crbug.com/662840
+  r"^.*(?:Trying to access ')?(\w*)(?:(?:' through proxy)|"
+  r"(?: is not defined))$",
+]
+
+# Lines matching any of the following regular expressions will be ignored.
+# Use uncompiled regular expressions - they'll be compiled later.
+IGNORE_LINES = [
+  r'^Validation of asm\.js module failed: .+$',
+  r'^.*:\d+: Invalid asm.js: .*$',
+  r'^Warning: unknown flag .*$',
+  r'^Warning: .+ is deprecated.*$',
+  r'^Try --help for options$',
+]
+
+
+###############################################################################
+# Implementation - you should not need to change anything below this point.
+
+# Compile regular expressions.
+ALLOWED_LINE_DIFFS = [re.compile(exp) for exp in ALLOWED_LINE_DIFFS]
+IGNORE_LINES = [re.compile(exp) for exp in IGNORE_LINES]
+
+
+def line_pairs(lines):
+  return itertools.izip_longest(
+      lines, itertools.islice(lines, 1, None), fillvalue=None)
+
+
+def caret_match(line1, line2):
+  if (not line1 or
+      not line2 or
+      len(line1) > MAX_LINE_LENGTH or
+      len(line2) > MAX_LINE_LENGTH):
+    return False
+  return bool(CARET_RE.match(line1) and CARET_RE.match(line2))
+
+
+def short_line_output(line):
+  if len(line) <= MAX_LINE_LENGTH:
+    # Avoid copying.
+    return line
+  return line[0:MAX_LINE_LENGTH] + '...'
+
+
+def ignore_by_regexp(line1, line2, allowed):
+  if len(line1) > MAX_LINE_LENGTH or len(line2) > MAX_LINE_LENGTH:
+    return False
+  for exp in allowed:
+    match1 = exp.match(line1)
+    match2 = exp.match(line2)
+    if match1 and match2:
+      # If there are groups in the regexp, ensure the groups matched the same
+      # things.
+      if match1.groups() == match2.groups():  # tuple comparison
+        return True
+  return False
+
+
+def diff_output(output1, output2, allowed, ignore1, ignore2):
+  def useful_line(ignore):
+    def fun(line):
+      return all(not e.match(line) for e in ignore)
+    return fun
+
+  lines1 = filter(useful_line(ignore1), output1)
+  lines2 = filter(useful_line(ignore2), output2)
+
+  for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest(
+      line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):
+
+    # Only one of the two iterators should run out.
+    assert not (line1 is None and line2 is None)
+
+    # One iterator ends earlier.
+    if line1 is None:
+      return '+ %s' % short_line_output(line2)
+    if line2 is None:
+      return '- %s' % short_line_output(line1)
+
+    # If lines are equal, no further checks are necessary.
+    if line1 == line2:
+      continue
+
+    # Look ahead. If next line is a caret, ignore this line.
+    if caret_match(lookahead1, lookahead2):
+      continue
+
+    # Check if a regexp allows these lines to be different.
+    if ignore_by_regexp(line1, line2, allowed):
+      continue
+
+    # Lines are different.
+    return '- %s\n+ %s' % (short_line_output(line1), short_line_output(line2))
+
+  # No difference found.
+  return None
+
+
+def get_suppression(arch1, config1, arch2, config2):
+  return V8Suppression(arch1, config1, arch2, config2)
+
+
+class Suppression(object):
+  def diff(self, output1, output2):
+    return None
+
+  def ignore(self, testcase):
+    return False
+
+  def ignore_by_output1(self, output):
+    return False
+
+  def ignore_by_output2(self, output):
+    return False
+
+
+class V8Suppression(Suppression):
+  def __init__(self, arch1, config1, arch2, config2):
+    self.arch1 = arch1
+    self.config1 = config1
+    self.arch2 = arch2
+    self.config2 = config2
+
+  def diff(self, output1, output2):
+    return diff_output(
+        output1.splitlines(),
+        output2.splitlines(),
+        ALLOWED_LINE_DIFFS,
+        IGNORE_LINES,
+        IGNORE_LINES,
+    )
+
+  def ignore(self, testcase):
+    for bug, exp in IGNORE_TEST_CASES.iteritems():
+      if exp.match(testcase):
+        return bug
+    return False
+
+  def ignore_by_output1(self, output):
+    return self.ignore_by_output(output, self.arch1, self.config1)
+
+  def ignore_by_output2(self, output):
+    return self.ignore_by_output(output, self.arch2, self.config2)
+
+  def ignore_by_output(self, output, arch, config):
+    def check(mapping):
+      for bug, exp in mapping.iteritems():
+        if exp.search(output):
+          return bug
+      return None
+    bug = check(IGNORE_OUTPUT.get('', {}))
+    if bug:
+      return bug
+    bug = check(IGNORE_OUTPUT.get(arch, {}))
+    if bug:
+      return bug
+    bug = check(IGNORE_OUTPUT.get(config, {}))
+    if bug:
+      return bug
+    bug = check(IGNORE_OUTPUT.get('%s,%s' % (arch, config), {}))
+    if bug:
+      return bug
+    return None