[wasm] Add proposal js-api spec tests to the testrunner

At the moment we only run the js-api spec tests of the core API on our
try bots. With the new staging process we want to introduce for
WebAssembly language features, see
https://docs.google.com/document/d/1hB8mpWmzmtaxZ8PuJEkAWLwFqXTjrw7mJ3Ge9W1dB4E,
we also want to run the js-api spec tests of proposals for which we
already staged the implementation. With this CL I do the following
changes:

1) The tools/wasm/update-wasm-spec-tests.sh now copies the js-api spec
   tests of the main spec and of the proposals to test/wasm-js/tests,
   and then uploads this directory to google cloud storage. The main
   spec tests are in test/wasm-js/tests, the proposal tests are in
   test/wasm-js/tests/proposals/PROPOSAL_NAME/.
2) Adjust the test-runner in test/wasm-js to run tests in
   tests/* instead of data/test/js-api/*. Thereby it also runs the
   proposal tests in test/wasm-js/tests/proposals/PROPOSAL_NAME/.
   For the proposal tests, the test runner now also adds d8 flags.
3) Remove the dependency to https://github.com/WebAssembly/spec from
   DEPS.
4) Cleanup .gitignore and wasm-js.status
5) Disable spec tests we don't pass with the new proposal.

R=tmrts@chromium.org

Bug: v8:9653
Change-Id: Ib3420871f17cb146d6cc7868f5613942a7f79d84
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1768372
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Tamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#63419}

.gitignore

@@ -53,6 +53,8 @@
 /test/test262/data
 /test/test262/harness
 /test/wasm-js/data
+/test/wasm-js/tests
+/test/wasm-js/tests.tar.gz
 /test/wasm-spec-tests/tests
 /test/wasm-spec-tests/tests.tar.gz
 /third_party/*

DEPS

@@ -245,8 +245,6 @@ deps = {
     'condition': 'checkout_mac',
     'dep_type': 'cipd',
   },
-  'v8/test/wasm-js/data':
-    Var('chromium_url') + '/external/github.com/WebAssembly/spec.git' + '@' + 'd22a7659e5f2f09be9532e2c72c04407134cc08e',
   'v8/third_party/perfetto':
     Var('android_url') + '/platform/external/perfetto.git' + '@' + '3150f3961532dccfe18a6c87c9494f754d3b6a51',
   'v8/third_party/protobuf':
@@ -361,6 +359,17 @@ hooks = [
                 '-s', 'v8/test/wasm-spec-tests/tests.tar.gz.sha1',
     ],
   },
+  {
+    'name': 'wasm_js',
+    'pattern': '.',
+    'action': [ 'download_from_google_storage',
+                '--no_resume',
+                '--no_auth',
+                '-u',
+                '--bucket', 'v8-wasm-spec-tests',
+                '-s', 'v8/test/wasm-js/tests.tar.gz.sha1',
+    ],
+  },
   {
     'name': 'sysroot_arm',
     'pattern': '.',

test/wasm-js/testcfg.py

@@ -13,6 +13,16 @@ WPT_ROOT = "/wasm/jsapi/"
 META_SCRIPT_REGEXP = re.compile(r"META:\s*script=(.*)")
 META_TIMEOUT_REGEXP = re.compile(r"META:\s*timeout=(.*)")
 
+proposal_flags = [{
+                    'name': 'reference-types',
+                    'flags': ['--experimental-wasm-anyref',
+                               '--no-experimental-wasm-bulk-memory']
+                  },
+                  {
+                    'name': 'bulk-memory-operations',
+                    'flags': ['--experimental-wasm-bulk-memory']
+                  }]
+
 
 class TestLoader(testsuite.JSTestLoader):
   @property
@@ -25,7 +35,7 @@ class TestSuite(testsuite.TestSuite):
     super(TestSuite, self).__init__(*args, **kwargs)
     self.mjsunit_js = os.path.join(os.path.dirname(self.root), "mjsunit",
                                    "mjsunit.js")
-    self.test_root = os.path.join(self.root, "data", "test", "js-api")
+    self.test_root = os.path.join(self.root, "tests")
     self._test_loader.test_root = self.test_root
 
   def _test_loader_class(self):
@@ -34,6 +44,8 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
+def get_proposal_path_identifier(proposal):
+  return os.sep.join(['proposals', proposal['name']])
 
 class TestCase(testcase.D8TestCase):
   def _get_timeout_param(self):
@@ -50,19 +62,27 @@ class TestCase(testcase.D8TestCase):
       return None
 
   def _get_files_params(self):
-    files = [os.path.join(self.suite.mjsunit_js),
+    files = [self.suite.mjsunit_js,
              os.path.join(self.suite.root, "testharness.js")]
 
     source = self.get_source()
+    current_dir = os.path.dirname(self._get_source_path())
     for script in META_SCRIPT_REGEXP.findall(source):
       if script.startswith(WPT_ROOT):
         # Matched an absolute path, strip the root and replace it with our
         # local root.
-        script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):])
+        found = False
+        for proposal in proposal_flags:
+          if get_proposal_path_identifier(proposal) in current_dir:
+            found = True
+            script = os.path.join(self.suite.test_root,
+                                  os.sep.join(['proposals', proposal['name']]),
+                                  script[len(WPT_ROOT):])
+        if not found:
+          script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):])
       elif not script.startswith("/"):
         # Matched a relative path, prepend this test's directory.
-        thisdir = os.path.dirname(self._get_source_path())
+        script = os.path.join(current_dir, script)
-        script = os.path.join(thisdir, script)
       else:
         raise Exception("Unexpected absolute path for script: \"%s\"" % script);
 
@@ -74,6 +94,12 @@ class TestCase(testcase.D8TestCase):
     ])
     return files
 
+  def _get_source_flags(self):
+    for proposal in proposal_flags:
+      if get_proposal_path_identifier(proposal) in self.path:
+        return proposal['flags']
+    return []
+
   def _get_source_path(self):
     # All tests are named `path/name.any.js`
     return os.path.join(self.suite.test_root, self.path + ANY_JS)

test/wasm-js/tests.tar.gz.sha1

@@ -0,0 +1 @@
+baf13bfd0e16aab9f386f59c06062885e18b6dbb

test/wasm-js/wasm-js.status

@@ -6,6 +6,7 @@
 [ALWAYS, {
   # https://bugs.chromium.org/p/v8/issues/detail?id=8633
   'limits': [SKIP],
+  'proposals/reference-types/limits': [SKIP],
 }], # ALWAYS
 
 ['arch == s390 or arch == s390x or system == aix', {

test/wasm-spec-tests/testcfg.py

@@ -21,16 +21,20 @@ class TestLoader(testsuite.JSTestLoader):
   pass
 
 class TestSuite(testsuite.TestSuite):
+  def __init__(self, *args, **kwargs):
+    super(TestSuite, self).__init__(*args, **kwargs)
+    self.test_root = os.path.join(self.root, "tests")
+    self._test_loader.test_root = self.test_root
+
   def _test_loader_class(self):
     return TestLoader
 
   def _test_class(self):
     return TestCase
 
-
 class TestCase(testcase.D8TestCase):
   def _get_files_params(self):
-    return [os.path.join(self.suite.root, self.path + self._get_suffix())]
+    return [os.path.join(self.suite.test_root, self.path + self._get_suffix())]
 
   def _get_source_flags(self):
     for proposal in proposal_flags:

test/wasm-spec-tests/tests.tar.gz.sha1

@@ -1 +1 @@
-b02f00e24b28ad76537a10a788a8be966c3577bd
+2ace97f753d5868bb4db1ddba8e91634f9af8343

test/wasm-spec-tests/wasm-spec-tests.status

@@ -5,52 +5,59 @@
 [
 [ALWAYS, {
   #TODO(ahaas): Add additional stack checks on mips.
-  'tests/skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]],
+  'skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]],
   # TODO(v8:9144): The MVP behavior when bounds-checking segments changed in
   # the bulk-memory proposal. Since we've enabled bulk-memory by default, we
   # need to update to use its testsuite.
-  'tests/linking': [FAIL],
+  'linking': [FAIL],
-  'tests/elem': [FAIL],
+  'elem': [FAIL],
-  'tests/data': [FAIL],
+  'data': [FAIL],
+  # TODO(v8:9658): The encoding of element segments changed in the bulk memory
+  # proposal
+  'proposals/bulk-memory-operations/bulk': [FAIL],
+  'proposals/bulk-memory-operations/table_init': [FAIL],
+  'proposals/bulk-memory-operations/table_copy': [FAIL],
+  'proposals/bulk-memory-operations/elem': [FAIL],
+  'proposals/bulk-memory-operations/binary': [FAIL],
 }],  # ALWAYS
 
 ['arch == mipsel or arch == mips64el or arch == mips or arch == mips64', {
   # These tests fail because mips does not support the correct NaN bit patterns.
-  'tests/float_misc': [SKIP],
+  'float_misc': [SKIP],
-  'tests/float_exprs': [SKIP],
+  'float_exprs': [SKIP],
-  'tests/f32': [SKIP],
+  'f32': [SKIP],
-  'tests/f64': [SKIP],
+  'f64': [SKIP],
-  'tests/f32_bitwise': [SKIP],
+  'f32_bitwise': [SKIP],
-  'tests/f64_bitwise': [SKIP],
+  'f64_bitwise': [SKIP],
-  'tests/proposals/reference-types/conversions':  [SKIP],
+  'proposals/reference-types/conversions':  [SKIP],
-  'tests/proposals/bulk-memory-operations/conversions': [SKIP],
+  'proposals/bulk-memory-operations/conversions': [SKIP],
 }],  # 'arch == mipsel or arch == mips64el or arch == mips or arch == mips64'
 
 ['(arch == mipsel or arch == mips64el or arch == mips or arch == mips64) and not simulator_run', {
   # This test fail because mips does not support the correct NaN bit patterns.
   # But it doesn't fail in simulator.
-  'tests/conversions': [SKIP],
+  'conversions': [SKIP],
 }],  # '(arch == mipsel or arch == mips64el or arch == mips or arch == mips64) and not simulator_run'
 
 ['arch == arm and not simulator_run', {
   # Too slow on chromebooks.
-  'tests/br_table': [SKIP],
+  'br_table': [SKIP],
 }],  # 'arch == arm and not simulator_run'
 
 ['arch == ppc or arch == ppc64', {
   # These tests fail because ppc float min and max doesn't convert sNaN to qNaN.
-  'tests/f32': [SKIP],
+  'f32': [SKIP],
-  'tests/f64': [SKIP],
+  'f64': [SKIP],
   # This test fails because ppc float to double doesn't convert sNaN to qNaN.
-  'tests/conversions': [SKIP],
+  'conversions': [SKIP],
-   'tests/proposals/reference-types/conversions':  [SKIP],
+   'proposals/reference-types/conversions':  [SKIP],
-   'tests/proposals/bulk-memory-operations/conversions': [SKIP],
+   'proposals/bulk-memory-operations/conversions': [SKIP],
 }],  # 'arch == ppc or arch == ppc64'
 
 ['arch == s390 or arch == s390x', {
   # These tests fail because s390 float min and max doesn't convert sNaN to qNaN.
-  'tests/f32': [SKIP],
+  'f32': [SKIP],
-  'tests/f64': [SKIP],
+  'f64': [SKIP],
 }],  # 'arch == s390 or arch == s390x'
 
 ##############################################################################

tools/wasm/update-wasm-spec-tests.sh

@@ -30,6 +30,8 @@ V8_DIR="${TOOLS_WASM_DIR}/../.."
 SPEC_TEST_DIR=${V8_DIR}/test/wasm-spec-tests
 TMP_DIR=${SPEC_TEST_DIR}/tmp
 
+JS_API_TEST_DIR=${V8_DIR}/test/wasm-js
+
 log_and_run cd ${V8_DIR}
 
 log_and_run rm -rf ${SPEC_TEST_DIR}/tests
@@ -40,21 +42,31 @@ log_and_run mkdir ${SPEC_TEST_DIR}/tests/proposals
 log_and_run rm -rf ${TMP_DIR}
 log_and_run mkdir ${TMP_DIR}
 
+log_and_run rm -rf ${JS_API_TEST_DIR}/tests
+log_and_run mkdir ${JS_API_TEST_DIR}/tests
+log_and_run mkdir ${JS_API_TEST_DIR}/tests/proposals
+
 ###############################################################################
 # Generate the spec tests.
 ###############################################################################
 
-log_and_run cd ${V8_DIR}/test/wasm-js/data/interpreter
+echo Process spec
+log_and_run cd ${TMP_DIR}
+log_and_run git clone https://github.com/WebAssembly/spec
+log_and_run cd spec/interpreter
+
 # The next step requires that ocaml is installed. See the README.md in
-# ${V8_DIR}/test/wasm-js/data/interpreter/.
+# https://github.com/WebAssembly/spec/tree/master/interpreter/.
 log_and_run make clean opt
 
-log_and_run cd ${V8_DIR}/test/wasm-js/data/test/core
+log_and_run cd ${TMP_DIR}/spec/test/core
 log_and_run cp *.wast ${SPEC_TEST_DIR}/tests/
 
-log_and_run ./run.py --wasm ${V8_DIR}/test/wasm-js/data/interpreter/wasm --out ${TMP_DIR}
+log_and_run ./run.py --wasm ${TMP_DIR}/spec/interpreter/wasm --out ${TMP_DIR}
 log_and_run cp ${TMP_DIR}/*.js ${SPEC_TEST_DIR}/tests/
 
+log_and_run cp -r ${TMP_DIR}/spec/test/js-api/* ${JS_API_TEST_DIR}/tests
+
 ###############################################################################
 # Generate the proposal tests.
 ###############################################################################
@@ -63,6 +75,7 @@ repos='bulk-memory-operations reference-types'
 
 for repo in ${repos}; do
   echo "Process ${repo}"
+  echo ">> Process core tests"
   log_and_run cd ${TMP_DIR}
   log_and_run git clone https://github.com/WebAssembly/${repo}
   # Compile the spec interpreter to generate the .js test cases later.
@@ -76,13 +89,27 @@ for repo in ${repos}; do
   for abs_filename in ${TMP_DIR}/${repo}/test/core/*.wast; do
     rel_filename="$(basename -- $abs_filename)"
     test_name=${rel_filename%.wast}
-    spec_filename=${V8_DIR}/test/wasm-js/data/test/core/${rel_filename}
+    spec_filename=${TMP_DIR}/spec/test/core/${rel_filename}
     if [ ! -f "$spec_filename" ] || ! cmp -s $abs_filename $spec_filename ; then
       log_and_run cp ${rel_filename} ${SPEC_TEST_DIR}/tests/proposals/${repo}/
       log_and_run ./run.py --wasm ../../interpreter/wasm ${rel_filename} --out _build 2> /dev/null
     fi
   done
   log_and_run cp _build/*.js ${SPEC_TEST_DIR}/tests/proposals/${repo}/
+
+  echo ">> Process js-api tests"
+  log_and_run mkdir ${JS_API_TEST_DIR}/tests/proposals/${repo}
+  log_and_run cp -r ${TMP_DIR}/${repo}/test/js-api/* ${JS_API_TEST_DIR}/tests/proposals/${repo}
+  # Delete duplicate tests
+  log_and_run cd ${JS_API_TEST_DIR}/tests
+  for spec_test_name in $(find ./ -name '*.any.js' -not -wholename '*/proposals/*'); do
+    proposal_test_name="proposals/${repo}/${spec_test_name}"
+    if [ -f "$proposal_test_name" ] && cmp -s $spec_test_name $proposal_test_name ; then
+      log_and_run rm $proposal_test_name
+    elif [ -f "$proposal_test_name" ]; then
+      echo "keep" $proposal_test_name
+    fi
+  done
 done
 
 ###############################################################################
@@ -95,6 +122,10 @@ echo "The following files will get uploaded:"
 ls -R tests
 echo
 
+cd ${JS_API_TEST_DIR}
+ls -R tests
+echo
+
 log_and_run rm -rf ${TMP_DIR}
 
 ###############################################################################
@@ -111,3 +142,6 @@ echo "* When the script asks you for your project-id, use 0."
 echo "****************************************************************************"
 log_and_run cd ${SPEC_TEST_DIR}
 log_and_run upload_to_google_storage.py -a -b v8-wasm-spec-tests tests
+
+log_and_run cd ${JS_API_TEST_DIR}
+log_and_run upload_to_google_storage.py -a -b v8-wasm-spec-tests tests