[foozzie] Add sanity checks to avoid bug flooding

This lets foozzie call d8 with sanity output before doing the actual
correctness comparisons. This will make clusterfuzz dedupe cases on
the difference found in the sanity checks.

Also adding missing OWNERS file.

NOTRY=true

Bug: chromium:933076
Change-Id: I4229183726064cc0ad76da8fe432e1dbb601a7ba
Reviewed-on: https://chromium-review.googlesource.com/c/1491221
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Belozorov <sergiyb@chromium.org>
Reviewed-by: Tamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59938}

tools/clusterfuzz/OWNERS

@@ -0,0 +1,5 @@
+set noparent
+
+machenbach@chromium.org
+sergiyb@chromium.org
+tmrts@chromium.org

tools/clusterfuzz/testdata/sanity_check_output.txt

@@ -0,0 +1,47 @@
+#
+# V8 correctness failure
+# V8 correctness configs: x64,ignition:x64,ignition_turbo
+# V8 correctness sources: sanity check failed
+# V8 correctness suppression: 
+#
+# CHECK
+#
+# Compared x64,ignition with x64,ignition_turbo
+#
+# Flags of x64,ignition:
+--abort_on_stack_or_string_length_overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --wasm-num-compilation-tasks=0 --suppress-asm-messages --random-seed 12345 --turbo-filter=~ --noopt --liftoff --no-wasm-tier-up
+# Flags of x64,ignition_turbo:
+--abort_on_stack_or_string_length_overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --wasm-num-compilation-tasks=0 --suppress-asm-messages --random-seed 12345 --stress-scavenge=100
+#
+# Difference:
+- unknown
++ not unknown
+#
+### Start of configuration x64,ignition:
+
+1
+v8-foozzie source: name/to/a/file.js
+2
+v8-foozzie source: name/to/file.js
+  weird error
+        ^
+3
+unknown
+
+
+### End of configuration x64,ignition
+#
+### Start of configuration x64,ignition_turbo:
+
+1
+v8-foozzie source: name/to/a/file.js
+2
+v8-foozzie source: name/to/file.js
+  weird other error
+^
+3
+not unknown
+
+
+### End of configuration x64,ignition_turbo
+

tools/clusterfuzz/v8_foozzie.py

@@ -101,6 +101,7 @@ PREAMBLE = [
   os.path.join(BASE_PATH, 'v8_suppressions.js'),
 ]
 ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
+SANITY_CHECKS = os.path.join(BASE_PATH, 'v8_sanity_checks.js')
 
 FLAGS = ['--abort_on_stack_or_string_length_overflow', '--expose-gc',
          '--allow-natives-syntax', '--invoke-weak-callbacks', '--omit-quit',
@@ -129,10 +130,7 @@ FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
 %(second_config_flags)s
 #
 # Difference:
-%(difference)s
-#
-# Source file:
-%(source)s
+%(difference)s%(source_file_text)s
 #
 ### Start of configuration %(first_config_label)s:
 %(first_config_output)s
@@ -143,6 +141,12 @@ FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
 ### End of configuration %(second_config_label)s
 """
 
+SOURCE_FILE_TEMPLATE = """
+#
+# Source file:
+%s"""
+
+
 FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)')
 SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);')
 
@@ -179,6 +183,9 @@ def parse_args():
   parser.add_argument(
       '--second-d8',
       help='optional path to second d8 executable, default: same as first')
+  parser.add_argument(
+      '--skip-sanity-checks', default=False, action='store_true',
+      help='skip sanity checks for testing purposes')
   parser.add_argument('testcase', help='path to test case')
   options = parser.parse_args()
 
@@ -259,6 +266,32 @@ def fail_bailout(output, ignore_by_output_fun):
   return False
 
 
+def print_difference(
+    options, source_key, first_config_flags, second_config_flags,
+    first_config_output, second_config_output, difference, source=None):
+  # The first three entries will be parsed by clusterfuzz. Format changes
+  # will require changes on the clusterfuzz side.
+  first_config_label = '%s,%s' % (options.first_arch, options.first_config)
+  second_config_label = '%s,%s' % (options.second_arch, options.second_config)
+  source_file_text = SOURCE_FILE_TEMPLATE % source if source else ''
+  print((FAILURE_TEMPLATE % dict(
+      configs='%s:%s' % (first_config_label, second_config_label),
+      source_file_text=source_file_text,
+      source_key=source_key,
+      suppression='', # We can't tie bugs to differences.
+      first_config_label=first_config_label,
+      second_config_label=second_config_label,
+      first_config_flags=' '.join(first_config_flags),
+      second_config_flags=' '.join(second_config_flags),
+      first_config_output=
+          first_config_output.stdout.decode('utf-8', 'replace'),
+      second_config_output=
+          second_config_output.stdout.decode('utf-8', 'replace'),
+      source=source,
+      difference=difference.decode('utf-8', 'replace'),
+  )).encode('utf-8', 'replace'))
+
+
 def main():
   options = parse_args()
   rng = random.Random(options.random_seed)
@@ -287,28 +320,49 @@ def main():
     if rng.random() < p:
       second_config_flags.append(flag)
 
-  def run_d8(d8, config_flags):
+  def run_d8(d8, config_flags, config_label=None, testcase=options.testcase):
     preamble = PREAMBLE[:]
     if options.first_arch != options.second_arch:
       preamble.append(ARCH_MOCKS)
-    args = [d8] + config_flags + preamble + [options.testcase]
-    print(" ".join(args))
+    args = [d8] + config_flags + preamble + [testcase]
+    if config_label:
+      print('# Command line for %s comparison:' % config_label)
+      print(' '.join(args))
     if d8.endswith('.py'):
       # Wrap with python in tests.
       args = [sys.executable] + args
     return v8_commands.Execute(
         args,
-        cwd=os.path.dirname(os.path.abspath(options.testcase)),
+        cwd=os.path.dirname(os.path.abspath(testcase)),
         timeout=TIMEOUT,
     )
 
-  first_config_output = run_d8(options.first_d8, first_config_flags)
+  # Sanity checks. Run both configurations with the sanity-checks file only and
+  # bail out early if different.
+  if not options.skip_sanity_checks:
+    first_config_output = run_d8(
+        options.first_d8, first_config_flags, testcase=SANITY_CHECKS)
+    second_config_output = run_d8(
+        options.second_d8, second_config_flags, testcase=SANITY_CHECKS)
+    difference, _ = suppress.diff(
+        first_config_output.stdout, second_config_output.stdout)
+    if difference:
+      # Special source key for sanity checks so that clusterfuzz dedupes all
+      # cases on this in case it's hit.
+      source_key = 'sanity check failed'
+      print_difference(
+          options, source_key, first_config_flags, second_config_flags,
+          first_config_output, second_config_output, difference)
+      return RETURN_FAIL
+
+  first_config_output = run_d8(options.first_d8, first_config_flags, 'first')
 
   # Early bailout based on first run's output.
   if pass_bailout(first_config_output, 1):
     return RETURN_PASS
 
-  second_config_output = run_d8(options.second_d8, second_config_flags)
+  second_config_output = run_d8(
+      options.second_d8, second_config_flags, 'second')
 
   # Bailout based on second run's output.
   if pass_bailout(second_config_output, 2):
@@ -320,7 +374,6 @@ def main():
   if source:
     source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
   else:
-    source = ORIGINAL_SOURCE_DEFAULT
     source_key = ORIGINAL_SOURCE_DEFAULT
 
   if difference:
@@ -332,25 +385,9 @@ def main():
     if fail_bailout(second_config_output, suppress.ignore_by_output2):
       return RETURN_FAIL
 
-    # The first three entries will be parsed by clusterfuzz. Format changes
-    # will require changes on the clusterfuzz side.
-    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
-    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
-    print((FAILURE_TEMPLATE % dict(
-        configs='%s:%s' % (first_config_label, second_config_label),
-        source_key=source_key,
-        suppression='', # We can't tie bugs to differences.
-        first_config_label=first_config_label,
-        second_config_label=second_config_label,
-        first_config_flags=' '.join(first_config_flags),
-        second_config_flags=' '.join(second_config_flags),
-        first_config_output=
-            first_config_output.stdout.decode('utf-8', 'replace'),
-        second_config_output=
-            second_config_output.stdout.decode('utf-8', 'replace'),
-        source=source,
-        difference=difference.decode('utf-8', 'replace'),
-    )).encode('utf-8', 'replace'))
+    print_difference(
+        options, source_key, first_config_flags, second_config_flags,
+        first_config_output, second_config_output, difference, source)
     return RETURN_FAIL
 
   # TODO(machenbach): Figure out if we could also return a bug in case there's

tools/clusterfuzz/v8_foozzie_test.py

@@ -113,10 +113,11 @@ otherfile.js: TypeError: undefined is not a constructor
 
 
 def cut_verbose_output(stdout):
-  return '\n'.join(stdout.split('\n')[2:])
+  # This removes first lines containing d8 commands.
+  return '\n'.join(stdout.split('\n')[4:])
 
 
-def run_foozzie(first_d8, second_d8):
+def run_foozzie(first_d8, second_d8, *extra_flags):
   return subprocess.check_output([
     sys.executable, FOOZZIE,
     '--random-seed', '12345',
@@ -125,23 +126,31 @@ def run_foozzie(first_d8, second_d8):
     '--first-config', 'ignition',
     '--second-config', 'ignition_turbo',
     os.path.join(TEST_DATA, 'fuzz-123.js'),
-  ])
+  ] + list(extra_flags))
 
 
 class SystemTest(unittest.TestCase):
   def testSyntaxErrorDiffPass(self):
-    stdout = run_foozzie('test_d8_1.py', 'test_d8_2.py')
+    stdout = run_foozzie('test_d8_1.py', 'test_d8_2.py', '--skip-sanity-checks')
     self.assertEquals('# V8 correctness - pass\n', cut_verbose_output(stdout))
 
   def testDifferentOutputFail(self):
     with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
       expected_output = f.read()
     with self.assertRaises(subprocess.CalledProcessError) as ctx:
-      run_foozzie('test_d8_1.py', 'test_d8_3.py')
+      run_foozzie('test_d8_1.py', 'test_d8_3.py', '--skip-sanity-checks')
     e = ctx.exception
     self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
     self.assertEquals(expected_output, cut_verbose_output(e.output))
 
+  def testSanityCheck(self):
+    with open(os.path.join(TEST_DATA, 'sanity_check_output.txt')) as f:
+      expected_output = f.read()
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      run_foozzie('test_d8_1.py', 'test_d8_3.py')
+    e = ctx.exception
+    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertEquals(expected_output, e.output)
 
 if __name__ == '__main__':
   unittest.main()

tools/clusterfuzz/v8_sanity_checks.js

@@ -0,0 +1,22 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file is executed separately before the correctness test case. Add here
+// checking of global properties that should never differ in any configuration.
+// A difference found in the prints below will prevent any further correctness
+// comparison for the selected configurations to avoid flooding bugs.
+
+print("https://crbug.com/932656");
+print(Object.getOwnPropertyNames(this));
+
+print("https://crbug.com/935800");
+(function () {
+  function foo() {
+    "use asm";
+    function baz() {}
+    return {bar: baz};
+  }
+  // TODO(mstarzinger): Uncomment once https://crbug.com/935800 is resolved.
+  // print(Object.getOwnPropertyNames(foo().bar));
+})();

tools/v8_presubmit.py

@@ -655,6 +655,7 @@ def CheckDeps(workspace):
 def PyTests(workspace):
   result = True
   for script in [
+      join(workspace, 'tools', 'clusterfuzz', 'v8_foozzie_test.py'),
       join(workspace, 'tools', 'release', 'test_scripts.py'),
       join(workspace, 'tools', 'unittests', 'run_tests_test.py'),
       join(workspace, 'tools', 'testrunner', 'testproc', 'variant_unittest.py'),