[foozzie] Add some system tests.

BUG=chromium:673246
NOTRY=true

Review-Url: https://codereview.chromium.org/2585193003
Cr-Commit-Position: refs/heads/master@{#41836}

tools/foozzie/testdata/failure_output.txt

@@ -0,0 +1,43 @@
+#
+# V8 correctness failure
+# V8 correctness configs: x64,fullcode:x64,ignition_staging
+# V8 correctness sources: 
+# V8 correctness suppression: 
+#
+# CHECK
+#
+# Compared x64,fullcode with x64,ignition_staging
+#
+# Flags of x64,fullcode:
+--abort_on_stack_overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --random-seed 12345 --nocrankshaft --turbo-filter=~
+# Flags of x64,ignition_staging:
+--abort_on_stack_overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --random-seed 12345 --ignition-staging
+#
+# Difference:
+- unknown
++ not unknown
+#
+### Start of configuration x64,fullcode:
+
+1
+2
+  weird error
+        ^
+3
+unknown
+
+
+### End of configuration x64,fullcode
+#
+### Start of configuration x64,ignition_staging:
+
+1
+2
+  weird other error
+^
+3
+not unknown
+
+
+### End of configuration x64,ignition_staging
+

tools/foozzie/testdata/fuzz-test1.js

@@ -0,0 +1,5 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Empty test dummy.

tools/foozzie/testdata/test_d8_1.py

@@ -0,0 +1,12 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+print """
+1
+2
+  weird error
+        ^
+3
+unknown
+"""

tools/foozzie/testdata/test_d8_2.py

@@ -0,0 +1,12 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+print """
+1
+2
+  weird other error
+^
+3
+unknown
+"""

tools/foozzie/testdata/test_d8_3.py

@@ -0,0 +1,12 @@
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+print """
+1
+2
+  weird other error
+^
+3
+not unknown
+"""

tools/foozzie/v8_foozzie.py

@@ -33,7 +33,7 @@ TIMEOUT = 3
 
 # Return codes.
 RETURN_PASS = 0
-RETURN_FAILURE = 2
+RETURN_FAIL = 2
 
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 PREAMBLE = [
@@ -47,16 +47,15 @@ FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
 SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
 
 # Output for suppressed failure case.
-FAILURE_HEADER_TEMPLATE = """
-#
+FAILURE_HEADER_TEMPLATE = """#
 # V8 correctness failure
 # V8 correctness configs: %(configs)s
 # V8 correctness sources: %(sources)s
-# V8 correctness suppression: %(suppression)s""".strip()
+# V8 correctness suppression: %(suppression)s
+"""
 
 # Extended output for failure case. The 'CHECK' is for the minimizer.
-FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """
-#
+FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
 # CHECK
 #
 # Compared %(first_config_label)s with %(second_config_label)s
@@ -76,7 +75,7 @@ FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """
 ### Start of configuration %(second_config_label)s:
 %(second_config_output)s
 ### End of configuration %(second_config_label)s
-""".strip()
+"""
 
 
 def parse_args():
@@ -180,15 +179,19 @@ def main():
   )
 
   if test_pattern_bailout(options.testcase, suppress.ignore):
-    return RETURN_FAILURE
+    return RETURN_FAIL
 
   common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
   first_config_flags = common_flags + CONFIGS[options.first_config]
   second_config_flags = common_flags + CONFIGS[options.second_config]
 
   def run_d8(d8, config_flags):
+    args = [d8] + config_flags + PREAMBLE + [options.testcase]
+    if d8.endswith('.py'):
+      # Wrap with python in tests.
+      args = [sys.executable] + args
     return v8_commands.Execute(
-        [d8] + config_flags + PREAMBLE + [options.testcase],
+        args,
         cwd=os.path.dirname(options.testcase),
         timeout=TIMEOUT,
     )
@@ -199,7 +202,7 @@ def main():
   if pass_bailout(first_config_output, 1):
     return RETURN_PASS
   if fail_bailout(first_config_output, suppress.ignore_by_output1):
-    return RETURN_FAILURE
+    return RETURN_FAIL
 
   second_config_output = run_d8(options.second_d8, second_config_flags)
 
@@ -207,7 +210,7 @@ def main():
   if pass_bailout(second_config_output, 2):
     return RETURN_PASS
   if fail_bailout(second_config_output, suppress.ignore_by_output2):
-    return RETURN_FAILURE
+    return RETURN_FAIL
 
   difference = suppress.diff(
       first_config_output.stdout, second_config_output.stdout)
@@ -228,7 +231,7 @@ def main():
         second_config_output=second_config_output.stdout,
         difference=difference,
     )
-    return RETURN_FAILURE
+    return RETURN_FAIL
 
   # TODO(machenbach): Figure out if we could also return a bug in case there's
   # no difference, but one of the line suppressions has matched - and without
@@ -246,12 +249,12 @@ if __name__ == "__main__":
     # Use one label for all internal and usage errors.
     print FAILURE_HEADER_TEMPLATE % dict(
         configs='', sources='', suppression='wrong_usage')
-    result = RETURN_FAILURE
+    result = RETURN_FAIL
   except Exception as e:
     print FAILURE_HEADER_TEMPLATE % dict(
         configs='', sources='', suppression='internal_error')
     print '# Internal error: %s' % e
     traceback.print_exc(file=sys.stdout)
-    result = RETURN_FAILURE
+    result = RETURN_FAIL
 
   sys.exit(result)

tools/foozzie/v8_foozzie_test.py

@@ -2,11 +2,19 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+import subprocess
+import sys
 import unittest
 
+import v8_foozzie
 import v8_suppressions
 
-class FuzzerTest(unittest.TestCase):
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
+TEST_DATA = os.path.join(BASE_DIR, 'testdata')
+
+class UnitTest(unittest.TestCase):
   def testDiff(self):
     # TODO(machenbach): Mock out suppression configuration.
     suppress = v8_suppressions.get_suppression(
@@ -75,3 +83,29 @@ otherfile.js: TypeError: undefined is not a constructor
     diff = """- somefile.js: TypeError: undefined is not a constructor
 + otherfile.js: TypeError: undefined is not a constructor"""
     self.assertEquals(diff, suppress.diff(one, two))
+
+
+def run_foozzie(first_d8, second_d8):
+  return subprocess.check_output([
+    sys.executable, FOOZZIE,
+    '--random-seed', '12345',
+    '--first-d8', os.path.join(TEST_DATA, first_d8),
+    '--second-d8', os.path.join(TEST_DATA, second_d8),
+    '--second-config', 'ignition_staging',
+    os.path.join(TEST_DATA, 'fuzz-test1.js'),
+  ])
+
+
+class SystemTest(unittest.TestCase):
+  def testSyntaxErrorDiffPass(self):
+    stdout = run_foozzie('test_d8_1.py', 'test_d8_2.py')
+    self.assertEquals('# V8 correctness - pass\n', stdout)
+
+  def testDifferentOutputFail(self):
+    with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
+      expected_output = f.read()
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      run_foozzie('test_d8_1.py', 'test_d8_3.py')
+    e = ctx.exception
+    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertEquals(expected_output, e.output)