From 1ed3bd5304fd04734ccffcc59fbf6449aece9cd0 Mon Sep 17 00:00:00 2001
From: Michal Majewski <majeski@google.com>
Date: Wed, 20 Dec 2017 13:35:08 +0100
Subject: [PATCH] [test] Output processors for mozilla and test262.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I8920ed24699ab5e6e4ed82f38bd7c8d8548fddfe
Reviewed-on: https://chromium-review.googlesource.com/834131
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50235}
---
 test/mozilla/testcfg.py | 26 +++++++++++++------
 test/test262/testcfg.py | 55 ++++++++++++++++++++++++-----------------
 2 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/test/mozilla/testcfg.py b/test/mozilla/testcfg.py
index 3af4492ca2..f55c366ab7 100644
--- a/test/mozilla/testcfg.py
+++ b/test/mozilla/testcfg.py
@@ -29,6 +29,7 @@
 import os
 
 from testrunner.local import testsuite
+from testrunner.objects import outproc
 from testrunner.objects import testcase
 
 EXCLUDED = ["CVS", ".svn"]
@@ -83,14 +84,6 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def IsNegativeTest(self, test):
-    return test.path.endswith("-n")
-
-  def IsFailureOutput(self, test, output):
-    if output.exit_code != 0:
-      return True
-    return "FAILED!" in output.stdout
-
 
 class TestCase(testcase.TestCase):
   def _get_files_params(self, ctx):
@@ -113,6 +106,23 @@ class TestCase(testcase.TestCase):
   def _get_source_path(self):
     return os.path.join(self.suite.testroot, self.path + self._get_suffix())
 
+  def _output_proc_class(self):
+    return OutProc
+
+
+class OutProc(outproc.OutProc):
+  def __init__(self, test):
+    self._negative = test.path.endswith('-n')
+
+  def _is_failure_output(self, output):
+    return (
+      output.exit_code != 0 or
+      'FAILED!' in output.stdout
+    )
+
+  def _is_negative(self):
+    return self._negative
+
 
 def GetSuite(name, root):
   return TestSuite(name, root)
diff --git a/test/test262/testcfg.py b/test/test262/testcfg.py
index c6e9df0446..4cb56dd76a 100644
--- a/test/test262/testcfg.py
+++ b/test/test262/testcfg.py
@@ -37,6 +37,7 @@ import tarfile
 from testrunner.local import statusfile
 from testrunner.local import testsuite
 from testrunner.local import utils
+from testrunner.objects import outproc
 from testrunner.objects import testcase
 
 # TODO(littledan): move the flag mapping into the status file
@@ -189,28 +190,6 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def IsFailureOutput(self, test, output):
-    test_record = test.test_record
-    if output.exit_code != 0:
-      return True
-    if ("negative" in test_record and
-        "type" in test_record["negative"] and
-        self._ParseException(output.stdout, test) !=
-            test_record["negative"]["type"]):
-        return True
-    return "FAILED!" in output.stdout
-
-  def _ParseException(self, string, test):
-    # somefile:somelinenumber: someerror[: sometext]
-    # somefile might include an optional drive letter on windows e.g. "e:".
-    match = re.search(
-        '^(?:\w:)?[^:]*:[0-9]+: ([^: ]+?)($|: )', string, re.MULTILINE)
-    if match:
-      return match.group(1).strip()
-    else:
-      print "Error parsing exception for %s" % test
-      return None
-
   def _VariantGeneratorFactory(self):
     return VariantGenerator
 
@@ -259,6 +238,38 @@ class TestCase(testcase.TestCase):
       return path
     return os.path.join(self.suite.testroot, filename)
 
+  def _output_proc_class(self):
+    return OutProc
+
+
+class OutProc(outproc.OutProc):
+  def __init__(self, test):
+    self._expected_exception = (
+      test.test_record
+      .get('negative', {})
+      .get('type', None))
+
+  def _is_failure_output(self, output):
+    if output.exit_code != 0:
+      return True
+    if (self._expected_exception and
+        self._expected_exception != self._parse_exception(output.stdout)):
+      return True
+    return 'FAILED!' in output.stdout
+
+  def _parse_exception(self, string):
+    # somefile:somelinenumber: someerror[: sometext]
+    # somefile might include an optional drive letter on windows e.g. "e:".
+    match = re.search(
+        '^(?:\w:)?[^:]*:[0-9]+: ([^: ]+?)($|: )', string, re.MULTILINE)
+    if match:
+      return match.group(1).strip()
+    else:
+      return None
+
+  def _is_negative(self):
+    return False
+
 
 def GetSuite(name, root):
   return TestSuite(name, root)