7aac6bc905
Moves the unicode predicate cache tables out of the unicode cache, and turns them into generic predicates in char-predicates.h which use static constexpr tables. This drops the per-isolate cost of unicode caches, and removes the need for accessing the unicode cache from most files. It does remove the mutability of the cache, which means that there may be regressions when parsing non-ASCII identifiers. Most likely the benefits to ASCII identifiers/keywords will outweigh any non-ASCII costs. Change-Id: I9a7a8b7c9b22d3e9ede824ab4e27f133ce20a399 Reviewed-on: https://chromium-review.googlesource.com/c/1335564 Reviewed-by: Yang Guo <yangguo@chromium.org> Reviewed-by: Toon Verwaest <verwaest@chromium.org> Reviewed-by: Michael Starzinger <mstarzinger@chromium.org> Commit-Queue: Leszek Swirski <leszeks@chromium.org> Cr-Commit-Position: refs/heads/master@{#57506}
108 lines
3.3 KiB
C++
108 lines
3.3 KiB
C++
// Copyright 2016 the V8 project authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
// Tests v8::internal::Scanner. Note that presently most unit tests for the
|
|
// Scanner are in cctest/test-parsing.cc, rather than here.
|
|
|
|
#include "src/handles-inl.h"
|
|
#include "src/objects-inl.h"
|
|
#include "src/parsing/scanner-character-streams.h"
|
|
#include "src/parsing/scanner.h"
|
|
#include "test/cctest/cctest.h"
|
|
|
|
namespace v8 {
|
|
namespace internal {
|
|
|
|
namespace {
|
|
|
|
const char src_simple[] = "function foo() { var x = 2 * a() + b; }";
|
|
|
|
struct ScannerTestHelper {
|
|
ScannerTestHelper() = default;
|
|
ScannerTestHelper(ScannerTestHelper&& other) V8_NOEXCEPT
|
|
: stream(std::move(other.stream)),
|
|
scanner(std::move(other.scanner)) {}
|
|
|
|
std::unique_ptr<Utf16CharacterStream> stream;
|
|
std::unique_ptr<Scanner> scanner;
|
|
|
|
Scanner* operator->() const { return scanner.get(); }
|
|
Scanner* get() const { return scanner.get(); }
|
|
};
|
|
|
|
ScannerTestHelper make_scanner(const char* src) {
|
|
ScannerTestHelper helper;
|
|
helper.stream = ScannerStream::ForTesting(src);
|
|
helper.scanner =
|
|
std::unique_ptr<Scanner>(new Scanner(helper.stream.get(), false));
|
|
helper.scanner->Initialize();
|
|
return helper;
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
// CHECK_TOK checks token equality, but by checking for equality of the token
|
|
// names. That should have the same result, but has much nicer error messaages.
|
|
#define CHECK_TOK(a, b) CHECK_EQ(Token::Name(a), Token::Name(b))
|
|
|
|
TEST(Bookmarks) {
|
|
// Scan through the given source and record the tokens for use as reference
|
|
// below.
|
|
std::vector<Token::Value> tokens;
|
|
{
|
|
auto scanner = make_scanner(src_simple);
|
|
do {
|
|
tokens.push_back(scanner->Next());
|
|
} while (scanner->current_token() != Token::EOS);
|
|
}
|
|
|
|
// For each position:
|
|
// - Scan through file,
|
|
// - set a bookmark once the position is reached,
|
|
// - scan a bit more,
|
|
// - reset to the bookmark, and
|
|
// - scan until the end.
|
|
// At each step, compare to the reference token sequence generated above.
|
|
for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) {
|
|
auto scanner = make_scanner(src_simple);
|
|
Scanner::BookmarkScope bookmark(scanner.get());
|
|
|
|
for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) {
|
|
if (i == bookmark_pos) {
|
|
bookmark.Set();
|
|
}
|
|
CHECK_TOK(tokens[i], scanner->Next());
|
|
}
|
|
|
|
bookmark.Apply();
|
|
for (size_t i = bookmark_pos; i < tokens.size(); i++) {
|
|
CHECK_TOK(tokens[i], scanner->Next());
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(AllThePushbacks) {
|
|
const struct {
|
|
const char* src;
|
|
const Token::Value tokens[5]; // Large enough for any of the test cases.
|
|
} test_cases[] = {
|
|
{"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
|
|
{"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}},
|
|
{"<!-x",
|
|
{Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
|
|
{"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}},
|
|
};
|
|
|
|
for (const auto& test_case : test_cases) {
|
|
auto scanner = make_scanner(test_case.src);
|
|
for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) {
|
|
CHECK_TOK(test_case.tokens[i], scanner->Next());
|
|
}
|
|
CHECK_TOK(Token::EOS, scanner->Next());
|
|
}
|
|
}
|
|
|
|
} // namespace internal
|
|
} // namespace v8
|