From 4f34d353d2fff1952fe083678b7a13851eadf5e5 Mon Sep 17 00:00:00 2001 From: Andreas Haas Date: Tue, 27 Aug 2019 16:14:34 +0200 Subject: [PATCH] [wasm] Add proposal js-api spec tests to the testrunner At the moment we only run the js-api spec tests of the core API on our try bots. With the new staging process we want to introduce for WebAssembly language features, see https://docs.google.com/document/d/1hB8mpWmzmtaxZ8PuJEkAWLwFqXTjrw7mJ3Ge9W1dB4E, we also want to run the js-api spec tests of proposals for which we already staged the implementation. With this CL I do the following changes: 1) The tools/wasm/update-wasm-spec-tests.sh now copies the js-api spec tests of the main spec and of the proposals to test/wasm-js/tests, and then uploads this directory to google cloud storage. The main spec tests are in test/wasm-js/tests, the proposal tests are in test/wasm-js/tests/proposals/PROPOSAL_NAME/. 2) Adjust the test-runner in test/wasm-js to run tests in tests/* instead of data/test/js-api/*. Thereby it also runs the proposal tests in test/wasm-js/tests/proposals/PROPOSAL_NAME/. For the proposal tests, the test runner now also adds d8 flags. 3) Remove the dependency to https://github.com/WebAssembly/spec from DEPS. 4) Cleanup .gitignore and wasm-js.status 5) Disable spec tests we don't pass with the new proposal. R=tmrts@chromium.org Bug: v8:9653 Change-Id: Ib3420871f17cb146d6cc7868f5613942a7f79d84 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1768372 Commit-Queue: Tamer Tas Reviewed-by: Tamer Tas Cr-Commit-Position: refs/heads/master@{#63419} --- .gitignore | 2 + DEPS | 13 +++++- test/wasm-js/testcfg.py | 36 ++++++++++++--- test/wasm-js/tests.tar.gz.sha1 | 1 + test/wasm-js/wasm-js.status | 1 + test/wasm-spec-tests/testcfg.py | 8 +++- test/wasm-spec-tests/tests.tar.gz.sha1 | 2 +- test/wasm-spec-tests/wasm-spec-tests.status | 49 ++++++++++++--------- tools/wasm/update-wasm-spec-tests.sh | 44 +++++++++++++++--- 9 files changed, 120 insertions(+), 36 deletions(-) create mode 100644 test/wasm-js/tests.tar.gz.sha1 diff --git a/.gitignore b/.gitignore index d0e1081a39..ba755f0f49 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,8 @@ /test/test262/data /test/test262/harness /test/wasm-js/data +/test/wasm-js/tests +/test/wasm-js/tests.tar.gz /test/wasm-spec-tests/tests /test/wasm-spec-tests/tests.tar.gz /third_party/* diff --git a/DEPS b/DEPS index a510be8a76..7379e6404f 100644 --- a/DEPS +++ b/DEPS @@ -245,8 +245,6 @@ deps = { 'condition': 'checkout_mac', 'dep_type': 'cipd', }, - 'v8/test/wasm-js/data': - Var('chromium_url') + '/external/github.com/WebAssembly/spec.git' + '@' + 'd22a7659e5f2f09be9532e2c72c04407134cc08e', 'v8/third_party/perfetto': Var('android_url') + '/platform/external/perfetto.git' + '@' + '3150f3961532dccfe18a6c87c9494f754d3b6a51', 'v8/third_party/protobuf': @@ -361,6 +359,17 @@ hooks = [ '-s', 'v8/test/wasm-spec-tests/tests.tar.gz.sha1', ], }, + { + 'name': 'wasm_js', + 'pattern': '.', + 'action': [ 'download_from_google_storage', + '--no_resume', + '--no_auth', + '-u', + '--bucket', 'v8-wasm-spec-tests', + '-s', 'v8/test/wasm-js/tests.tar.gz.sha1', + ], + }, { 'name': 'sysroot_arm', 'pattern': '.', diff --git a/test/wasm-js/testcfg.py b/test/wasm-js/testcfg.py index 3f3c67ac6a..40ff2ecad1 100644 --- a/test/wasm-js/testcfg.py +++ b/test/wasm-js/testcfg.py @@ -13,6 +13,16 @@ WPT_ROOT = "/wasm/jsapi/" META_SCRIPT_REGEXP = re.compile(r"META:\s*script=(.*)") META_TIMEOUT_REGEXP = re.compile(r"META:\s*timeout=(.*)") +proposal_flags = [{ + 'name': 'reference-types', + 'flags': ['--experimental-wasm-anyref', + '--no-experimental-wasm-bulk-memory'] + }, + { + 'name': 'bulk-memory-operations', + 'flags': ['--experimental-wasm-bulk-memory'] + }] + class TestLoader(testsuite.JSTestLoader): @property @@ -25,7 +35,7 @@ class TestSuite(testsuite.TestSuite): super(TestSuite, self).__init__(*args, **kwargs) self.mjsunit_js = os.path.join(os.path.dirname(self.root), "mjsunit", "mjsunit.js") - self.test_root = os.path.join(self.root, "data", "test", "js-api") + self.test_root = os.path.join(self.root, "tests") self._test_loader.test_root = self.test_root def _test_loader_class(self): @@ -34,6 +44,8 @@ class TestSuite(testsuite.TestSuite): def _test_class(self): return TestCase +def get_proposal_path_identifier(proposal): + return os.sep.join(['proposals', proposal['name']]) class TestCase(testcase.D8TestCase): def _get_timeout_param(self): @@ -50,19 +62,27 @@ class TestCase(testcase.D8TestCase): return None def _get_files_params(self): - files = [os.path.join(self.suite.mjsunit_js), + files = [self.suite.mjsunit_js, os.path.join(self.suite.root, "testharness.js")] source = self.get_source() + current_dir = os.path.dirname(self._get_source_path()) for script in META_SCRIPT_REGEXP.findall(source): if script.startswith(WPT_ROOT): # Matched an absolute path, strip the root and replace it with our # local root. - script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):]) + found = False + for proposal in proposal_flags: + if get_proposal_path_identifier(proposal) in current_dir: + found = True + script = os.path.join(self.suite.test_root, + os.sep.join(['proposals', proposal['name']]), + script[len(WPT_ROOT):]) + if not found: + script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):]) elif not script.startswith("/"): # Matched a relative path, prepend this test's directory. - thisdir = os.path.dirname(self._get_source_path()) - script = os.path.join(thisdir, script) + script = os.path.join(current_dir, script) else: raise Exception("Unexpected absolute path for script: \"%s\"" % script); @@ -74,6 +94,12 @@ class TestCase(testcase.D8TestCase): ]) return files + def _get_source_flags(self): + for proposal in proposal_flags: + if get_proposal_path_identifier(proposal) in self.path: + return proposal['flags'] + return [] + def _get_source_path(self): # All tests are named `path/name.any.js` return os.path.join(self.suite.test_root, self.path + ANY_JS) diff --git a/test/wasm-js/tests.tar.gz.sha1 b/test/wasm-js/tests.tar.gz.sha1 new file mode 100644 index 0000000000..56c3e45d00 --- /dev/null +++ b/test/wasm-js/tests.tar.gz.sha1 @@ -0,0 +1 @@ +baf13bfd0e16aab9f386f59c06062885e18b6dbb \ No newline at end of file diff --git a/test/wasm-js/wasm-js.status b/test/wasm-js/wasm-js.status index 51961fd46d..42ad2a4152 100644 --- a/test/wasm-js/wasm-js.status +++ b/test/wasm-js/wasm-js.status @@ -6,6 +6,7 @@ [ALWAYS, { # https://bugs.chromium.org/p/v8/issues/detail?id=8633 'limits': [SKIP], + 'proposals/reference-types/limits': [SKIP], }], # ALWAYS ['arch == s390 or arch == s390x or system == aix', { diff --git a/test/wasm-spec-tests/testcfg.py b/test/wasm-spec-tests/testcfg.py index 34230b07d0..9f957183f6 100644 --- a/test/wasm-spec-tests/testcfg.py +++ b/test/wasm-spec-tests/testcfg.py @@ -21,16 +21,20 @@ class TestLoader(testsuite.JSTestLoader): pass class TestSuite(testsuite.TestSuite): + def __init__(self, *args, **kwargs): + super(TestSuite, self).__init__(*args, **kwargs) + self.test_root = os.path.join(self.root, "tests") + self._test_loader.test_root = self.test_root + def _test_loader_class(self): return TestLoader def _test_class(self): return TestCase - class TestCase(testcase.D8TestCase): def _get_files_params(self): - return [os.path.join(self.suite.root, self.path + self._get_suffix())] + return [os.path.join(self.suite.test_root, self.path + self._get_suffix())] def _get_source_flags(self): for proposal in proposal_flags: diff --git a/test/wasm-spec-tests/tests.tar.gz.sha1 b/test/wasm-spec-tests/tests.tar.gz.sha1 index 0b068afe0a..2686b46944 100644 --- a/test/wasm-spec-tests/tests.tar.gz.sha1 +++ b/test/wasm-spec-tests/tests.tar.gz.sha1 @@ -1 +1 @@ -b02f00e24b28ad76537a10a788a8be966c3577bd \ No newline at end of file +2ace97f753d5868bb4db1ddba8e91634f9af8343 \ No newline at end of file diff --git a/test/wasm-spec-tests/wasm-spec-tests.status b/test/wasm-spec-tests/wasm-spec-tests.status index 2972217ea2..d853ab2265 100644 --- a/test/wasm-spec-tests/wasm-spec-tests.status +++ b/test/wasm-spec-tests/wasm-spec-tests.status @@ -5,52 +5,59 @@ [ [ALWAYS, { #TODO(ahaas): Add additional stack checks on mips. - 'tests/skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]], + 'skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]], # TODO(v8:9144): The MVP behavior when bounds-checking segments changed in # the bulk-memory proposal. Since we've enabled bulk-memory by default, we # need to update to use its testsuite. - 'tests/linking': [FAIL], - 'tests/elem': [FAIL], - 'tests/data': [FAIL], + 'linking': [FAIL], + 'elem': [FAIL], + 'data': [FAIL], + # TODO(v8:9658): The encoding of element segments changed in the bulk memory + # proposal + 'proposals/bulk-memory-operations/bulk': [FAIL], + 'proposals/bulk-memory-operations/table_init': [FAIL], + 'proposals/bulk-memory-operations/table_copy': [FAIL], + 'proposals/bulk-memory-operations/elem': [FAIL], + 'proposals/bulk-memory-operations/binary': [FAIL], }], # ALWAYS ['arch == mipsel or arch == mips64el or arch == mips or arch == mips64', { # These tests fail because mips does not support the correct NaN bit patterns. - 'tests/float_misc': [SKIP], - 'tests/float_exprs': [SKIP], - 'tests/f32': [SKIP], - 'tests/f64': [SKIP], - 'tests/f32_bitwise': [SKIP], - 'tests/f64_bitwise': [SKIP], - 'tests/proposals/reference-types/conversions': [SKIP], - 'tests/proposals/bulk-memory-operations/conversions': [SKIP], + 'float_misc': [SKIP], + 'float_exprs': [SKIP], + 'f32': [SKIP], + 'f64': [SKIP], + 'f32_bitwise': [SKIP], + 'f64_bitwise': [SKIP], + 'proposals/reference-types/conversions': [SKIP], + 'proposals/bulk-memory-operations/conversions': [SKIP], }], # 'arch == mipsel or arch == mips64el or arch == mips or arch == mips64' ['(arch == mipsel or arch == mips64el or arch == mips or arch == mips64) and not simulator_run', { # This test fail because mips does not support the correct NaN bit patterns. # But it doesn't fail in simulator. - 'tests/conversions': [SKIP], + 'conversions': [SKIP], }], # '(arch == mipsel or arch == mips64el or arch == mips or arch == mips64) and not simulator_run' ['arch == arm and not simulator_run', { # Too slow on chromebooks. - 'tests/br_table': [SKIP], + 'br_table': [SKIP], }], # 'arch == arm and not simulator_run' ['arch == ppc or arch == ppc64', { # These tests fail because ppc float min and max doesn't convert sNaN to qNaN. - 'tests/f32': [SKIP], - 'tests/f64': [SKIP], + 'f32': [SKIP], + 'f64': [SKIP], # This test fails because ppc float to double doesn't convert sNaN to qNaN. - 'tests/conversions': [SKIP], - 'tests/proposals/reference-types/conversions': [SKIP], - 'tests/proposals/bulk-memory-operations/conversions': [SKIP], + 'conversions': [SKIP], + 'proposals/reference-types/conversions': [SKIP], + 'proposals/bulk-memory-operations/conversions': [SKIP], }], # 'arch == ppc or arch == ppc64' ['arch == s390 or arch == s390x', { # These tests fail because s390 float min and max doesn't convert sNaN to qNaN. - 'tests/f32': [SKIP], - 'tests/f64': [SKIP], + 'f32': [SKIP], + 'f64': [SKIP], }], # 'arch == s390 or arch == s390x' ############################################################################## diff --git a/tools/wasm/update-wasm-spec-tests.sh b/tools/wasm/update-wasm-spec-tests.sh index d029ffe604..ef99bf8a37 100755 --- a/tools/wasm/update-wasm-spec-tests.sh +++ b/tools/wasm/update-wasm-spec-tests.sh @@ -30,6 +30,8 @@ V8_DIR="${TOOLS_WASM_DIR}/../.." SPEC_TEST_DIR=${V8_DIR}/test/wasm-spec-tests TMP_DIR=${SPEC_TEST_DIR}/tmp +JS_API_TEST_DIR=${V8_DIR}/test/wasm-js + log_and_run cd ${V8_DIR} log_and_run rm -rf ${SPEC_TEST_DIR}/tests @@ -40,21 +42,31 @@ log_and_run mkdir ${SPEC_TEST_DIR}/tests/proposals log_and_run rm -rf ${TMP_DIR} log_and_run mkdir ${TMP_DIR} +log_and_run rm -rf ${JS_API_TEST_DIR}/tests +log_and_run mkdir ${JS_API_TEST_DIR}/tests +log_and_run mkdir ${JS_API_TEST_DIR}/tests/proposals + ############################################################################### # Generate the spec tests. ############################################################################### -log_and_run cd ${V8_DIR}/test/wasm-js/data/interpreter +echo Process spec +log_and_run cd ${TMP_DIR} +log_and_run git clone https://github.com/WebAssembly/spec +log_and_run cd spec/interpreter + # The next step requires that ocaml is installed. See the README.md in -# ${V8_DIR}/test/wasm-js/data/interpreter/. +# https://github.com/WebAssembly/spec/tree/master/interpreter/. log_and_run make clean opt -log_and_run cd ${V8_DIR}/test/wasm-js/data/test/core +log_and_run cd ${TMP_DIR}/spec/test/core log_and_run cp *.wast ${SPEC_TEST_DIR}/tests/ -log_and_run ./run.py --wasm ${V8_DIR}/test/wasm-js/data/interpreter/wasm --out ${TMP_DIR} +log_and_run ./run.py --wasm ${TMP_DIR}/spec/interpreter/wasm --out ${TMP_DIR} log_and_run cp ${TMP_DIR}/*.js ${SPEC_TEST_DIR}/tests/ +log_and_run cp -r ${TMP_DIR}/spec/test/js-api/* ${JS_API_TEST_DIR}/tests + ############################################################################### # Generate the proposal tests. ############################################################################### @@ -63,6 +75,7 @@ repos='bulk-memory-operations reference-types' for repo in ${repos}; do echo "Process ${repo}" + echo ">> Process core tests" log_and_run cd ${TMP_DIR} log_and_run git clone https://github.com/WebAssembly/${repo} # Compile the spec interpreter to generate the .js test cases later. @@ -76,13 +89,27 @@ for repo in ${repos}; do for abs_filename in ${TMP_DIR}/${repo}/test/core/*.wast; do rel_filename="$(basename -- $abs_filename)" test_name=${rel_filename%.wast} - spec_filename=${V8_DIR}/test/wasm-js/data/test/core/${rel_filename} + spec_filename=${TMP_DIR}/spec/test/core/${rel_filename} if [ ! -f "$spec_filename" ] || ! cmp -s $abs_filename $spec_filename ; then log_and_run cp ${rel_filename} ${SPEC_TEST_DIR}/tests/proposals/${repo}/ log_and_run ./run.py --wasm ../../interpreter/wasm ${rel_filename} --out _build 2> /dev/null fi done log_and_run cp _build/*.js ${SPEC_TEST_DIR}/tests/proposals/${repo}/ + + echo ">> Process js-api tests" + log_and_run mkdir ${JS_API_TEST_DIR}/tests/proposals/${repo} + log_and_run cp -r ${TMP_DIR}/${repo}/test/js-api/* ${JS_API_TEST_DIR}/tests/proposals/${repo} + # Delete duplicate tests + log_and_run cd ${JS_API_TEST_DIR}/tests + for spec_test_name in $(find ./ -name '*.any.js' -not -wholename '*/proposals/*'); do + proposal_test_name="proposals/${repo}/${spec_test_name}" + if [ -f "$proposal_test_name" ] && cmp -s $spec_test_name $proposal_test_name ; then + log_and_run rm $proposal_test_name + elif [ -f "$proposal_test_name" ]; then + echo "keep" $proposal_test_name + fi + done done ############################################################################### @@ -95,6 +122,10 @@ echo "The following files will get uploaded:" ls -R tests echo +cd ${JS_API_TEST_DIR} +ls -R tests +echo + log_and_run rm -rf ${TMP_DIR} ############################################################################### @@ -111,3 +142,6 @@ echo "* When the script asks you for your project-id, use 0." echo "****************************************************************************" log_and_run cd ${SPEC_TEST_DIR} log_and_run upload_to_google_storage.py -a -b v8-wasm-spec-tests tests + +log_and_run cd ${JS_API_TEST_DIR} +log_and_run upload_to_google_storage.py -a -b v8-wasm-spec-tests tests