[tools] Add FAIL_PHASE_ONLY status file modifier for test262 tests

R=machenbach@chromium.org, mathias@chromium.org

Bug: v8:8467
Change-Id: I7369fa993d0762cf9c3c6f55361d074d067a0eb5
Reviewed-on: https://chromium-review.googlesource.com/c/1338347
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Sergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#57734}

test/test262/BUILD.gn

@@ -15,9 +15,10 @@ group("v8_test262") {
     "detachArrayBuffer.js",
     "harness/",
     "harness-adapt.js",
+    "harness-adapt-donotevaluate.js",
     "harness-agent.js",
     "test262.status",
     "testcfg.py",
     "local-tests/",
   ]
-}
+}

test/test262/harness-adapt-donotevaluate.js

@@ -0,0 +1,12 @@
+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// V8 has several long-standing bugs where "early errors", i.e. errors that are
+// supposed to be thrown at parse time, end up being thrown at runtime instead.
+// This file is used to implement the FAIL_PHASE_ONLY outcome as used in
+// test/test262/test262.status. Tests marked with this outcome are run in a
+// special mode that verifies that a) V8 throws an exception at all, and b) that
+// the exception has the correct type, but ignores the fact that they are thrown
+// after parsing is done. See crbug.com/v8/8467 for details.
+$DONOTEVALUATE = () => {};

test/test262/test262.status

@@ -178,7 +178,7 @@
   'language/expressions/assignment/destructuring/keyed-destructuring-property-reference-target-evaluation-order': [FAIL],
 
   # https://bugs.chromium.org/p/v8/issues/detail?id=896
-  'built-ins/RegExp/property-escapes/binary-property-with-value-ASCII_-_F': [FAIL],
+  'built-ins/RegExp/property-escapes/binary-property-with-value-ASCII_-_F': [FAIL_PHASE_ONLY],
   'built-ins/RegExp/property-escapes/binary-property-with-value-ASCII_-_F-negated': [FAIL],
   'built-ins/RegExp/property-escapes/binary-property-with-value-ASCII_-_Invalid': [FAIL],
   'built-ins/RegExp/property-escapes/binary-property-with-value-ASCII_-_Invalid-negated': [FAIL],

test/test262/testcfg.py

@@ -173,6 +173,8 @@ class TestCase(testcase.D8TestCase):
         list(self.suite.harness) +
         ([os.path.join(self.suite.root, "harness-agent.js")]
          if self.path.startswith('built-ins/Atomics') else []) +
+        ([os.path.join(self.suite.root, "harness-adapt-donotevaluate.js")]
+         if self.fail_phase_only else []) +
         self._get_includes() +
         (["--module"] if "module" in self.test_record else []) +
         [self._get_source_path()]

tools/testrunner/local/statusfile.py

@@ -45,12 +45,13 @@ FAIL_SLOPPY = "FAIL_SLOPPY"
 SKIP = "SKIP"
 SLOW = "SLOW"
 NO_VARIANTS = "NO_VARIANTS"
+FAIL_PHASE_ONLY = "FAIL_PHASE_ONLY"
 
 ALWAYS = "ALWAYS"
 
 KEYWORDS = {}
 for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, NO_VARIANTS, FAIL_SLOPPY,
-            ALWAYS]:
+            ALWAYS, FAIL_PHASE_ONLY]:
   KEYWORDS[key] = key
 
 # Support arches, modes to be written as keywords instead of strings.

tools/testrunner/objects/testcase.py

@@ -112,6 +112,10 @@ class TestCase(object):
       self._parse_status_file_outcomes(self._statusfile_outcomes))
 
   def _parse_status_file_outcomes(self, outcomes):
+    # This flag does not affect the test execution or outcome parsing by
+    # default, but subclasses can implement custom logic when it is set.
+    self.fail_phase_only = statusfile.FAIL_PHASE_ONLY in outcomes
+
     if (statusfile.FAIL_SLOPPY in outcomes and
         '--use-strict' not in self.variant_flags):
       return outproc.OUTCOMES_FAIL