AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[parser] Remove use counter for U+2028 & U+2029

Browse Source 
The use counter was originally added in
d3c9812143
(https://chromium-review.googlesource.com/c/v8/v8/+/693155).

The CL that removes the plumbing in Chromium is here:
https://chromium-review.googlesource.com/c/chromium/src/+/819632

BUG=v8:6827

Cq-Include-Trybots: master.tryserver.chromium.linux:linux_chromium_rel_ng
Change-Id: Ie5f861fe2a64454e682d8cd0618c948642a32886
Reviewed-on: https://chromium-review.googlesource.com/819553
Commit-Queue: Mathias Bynens <mathias@chromium.org>
Reviewed-by: Adam Klein <adamk@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50009}
This commit is contained in:
Mathias Bynens 2017-12-11 20:13:03 +01:00 committed by Commit Bot
parent 8e32805f5d
commit 8b1e0c6480
5 changed files with 34 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/parsing/parser.cc
View File

@ -506,7 +506,7 @@ Parser::Parser(ParseInfo* info)
info->runtime_call_stats(), info->logger(),
info->script().is_null() ? -1 : info->script()->id(),
info->is_module(), true),
scanner_(info->unicode_cache(), use_counts_),
scanner_(info->unicode_cache()),
reusable_preparser_(nullptr),
mode_(PARSE_EAGERLY), // Lazy mode must be set explicitly.
source_range_map_(info->source_range_map()),

44
src/parsing/scanner.cc
View File

@ -174,31 +174,15 @@ bool Scanner::BookmarkScope::HasBeenApplied() {
return bookmark_ == kBookmarkWasApplied;
}
// LineTerminator: 'JS_Line_Terminator' in point.properties
// ES#sec-line-terminators lists exactly 4 code points:
// LF (U+000A), CR (U+000D), LS(U+2028), PS(U+2029)
bool Scanner::IsLineTerminator(uc32 c) {
if (c == 0x000A || c == 0x000D) {
return true;
}
if (c == 0x2028 || c == 0x2029) {
++use_counts_[v8::Isolate::UseCounterFeature::
kLineOrParagraphSeparatorAsLineTerminator];
return true;
}
return false;
}
// ----------------------------------------------------------------------------
// Scanner
Scanner::Scanner(UnicodeCache* unicode_cache, int* use_counts)
Scanner::Scanner(UnicodeCache* unicode_cache)
: unicode_cache_(unicode_cache),
octal_pos_(Location::invalid()),
octal_message_(MessageTemplate::kNone),
found_html_comment_(false),
allow_harmony_bigint_(false),
use_counts_(use_counts) {}
allow_harmony_bigint_(false) {}
void Scanner::Initialize(Utf16CharacterStream* source, bool is_module) {
DCHECK_NOT_NULL(source);
@ -458,7 +442,7 @@ Token::Value Scanner::SkipWhiteSpace() {
// Advance as long as character is a WhiteSpace or LineTerminator.
// Remember if the latter is the case.
if (IsLineTerminator(c0_)) {
if (unibrow::IsLineTerminator(c0_)) {
has_line_terminator_before_next_ = true;
} else if (!unicode_cache_->IsWhiteSpace(c0_)) {
break;
@ -515,7 +499,7 @@ Token::Value Scanner::SkipSingleLineComment() {
// separately by the lexical grammar and becomes part of the
// stream of input elements for the syntactic grammar (see
// ECMA-262, section 7.4).
while (c0_ != kEndOfInput && !IsLineTerminator(c0_)) {
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
Advance();
}
@ -525,7 +509,7 @@ Token::Value Scanner::SkipSingleLineComment() {
Token::Value Scanner::SkipSourceURLComment() {
TryToParseSourceURLComment();
while (c0_ != kEndOfInput && !IsLineTerminator(c0_)) {
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
Advance();
}
@ -561,7 +545,7 @@ void Scanner::TryToParseSourceURLComment() {
while (c0_ != kEndOfInput && unicode_cache_->IsWhiteSpace(c0_)) {
Advance();
}
while (c0_ != kEndOfInput && !IsLineTerminator(c0_)) {
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
// Disallowed characters.
if (c0_ == '"' || c0_ == '\'') {
value->Reset();
@ -574,7 +558,7 @@ void Scanner::TryToParseSourceURLComment() {
Advance();
}
// Allow whitespace at the end.
while (c0_ != kEndOfInput && !IsLineTerminator(c0_)) {
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
if (!unicode_cache_->IsWhiteSpace(c0_)) {
value->Reset();
break;
@ -591,7 +575,7 @@ Token::Value Scanner::SkipMultiLineComment() {
while (c0_ != kEndOfInput) {
uc32 ch = c0_;
Advance();
if (c0_ != kEndOfInput && IsLineTerminator(ch)) {
if (c0_ != kEndOfInput && unibrow::IsLineTerminator(ch)) {
// Following ECMA-262, section 7.4, a comment containing
// a newline will make the comment count as a line-terminator.
has_multiline_comment_before_next_ = true;
@ -988,7 +972,8 @@ bool Scanner::ScanEscape() {
Advance<capture_raw>();
// Skip escaped newlines.
if (!in_template_literal && c0_ != kEndOfInput && IsLineTerminator(c)) {
if (!in_template_literal && c0_ != kEndOfInput &&
unibrow::IsLineTerminator(c)) {
// Allow escaped CR+LF newlines in multiline string literals.
if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>();
return true;
@ -1081,7 +1066,8 @@ Token::Value Scanner::ScanString() {
AddLiteralChar(c);
}
while (c0_ != quote && c0_ != kEndOfInput && !IsLineTerminator(c0_)) {
while (c0_ != quote && c0_ != kEndOfInput &&
!unibrow::IsLineTerminator(c0_)) {
uc32 c = c0_;
Advance();
if (c == '\\') {
@ -1137,7 +1123,7 @@ Token::Value Scanner::ScanTemplateSpan() {
ReduceRawLiteralLength(2);
break;
} else if (c == '\\') {
if (c0_ != kEndOfInput && IsLineTerminator(c0_)) {
if (c0_ != kEndOfInput && unibrow::IsLineTerminator(c0_)) {
// The TV of LineContinuation :: \ LineTerminatorSequence is the empty
// code unit sequence.
uc32 lastChar = c0_;
@ -1698,12 +1684,12 @@ bool Scanner::ScanRegExpPattern() {
}
while (c0_ != '/' || in_character_class) {
if (c0_ == kEndOfInput || IsLineTerminator(c0_)) {
if (c0_ == kEndOfInput || unibrow::IsLineTerminator(c0_)) {
return false;
}
if (c0_ == '\\') { // Escape sequence.
AddLiteralCharAdvance();
if (c0_ == kEndOfInput || IsLineTerminator(c0_)) {
if (c0_ == kEndOfInput || unibrow::IsLineTerminator(c0_)) {
return false;
}
AddLiteralCharAdvance();

6
src/parsing/scanner.h
View File

@ -207,7 +207,7 @@ class Scanner {
static const int kNoOctalLocation = -1;
static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
explicit Scanner(UnicodeCache* scanner_contants, int* use_counts_);
explicit Scanner(UnicodeCache* scanner_contants);
void Initialize(Utf16CharacterStream* source, bool is_module);
@ -736,8 +736,6 @@ class Scanner {
bool is_module_;
bool IsLineTerminator(uc32 c);
Token::Value ScanTemplateSpan();
// Return the current source position.
@ -805,8 +803,6 @@ class Scanner {
// Whether to recognize BIGINT tokens.
bool allow_harmony_bigint_;
int* use_counts_;
MessageTemplate::Template scanner_error_;
Location scanner_error_location_;
};

8
test/cctest/parsing/test-scanner.cc
View File

@ -29,7 +29,6 @@ struct ScannerTestHelper {
std::unique_ptr<UnicodeCache> unicode_cache;
std::unique_ptr<Utf16CharacterStream> stream;
std::unique_ptr<Scanner> scanner;
int use_counts[v8::Isolate::kUseCounterFeatureCount];
Scanner* operator->() const { return scanner.get(); }
Scanner* get() const { return scanner.get(); }
@ -39,11 +38,8 @@ ScannerTestHelper make_scanner(const char* src) {
ScannerTestHelper helper;
helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache);
helper.stream = ScannerStream::ForTesting(src);
for (int i = 0; i < v8::Isolate::kUseCounterFeatureCount; i++) {
helper.use_counts[i] = 0;
}
helper.scanner = std::unique_ptr<Scanner>(
new Scanner(helper.unicode_cache.get(), helper.use_counts));
helper.scanner =
std::unique_ptr<Scanner>(new Scanner(helper.unicode_cache.get()));
helper.scanner->Initialize(helper.stream.get(), false);
return helper;
}

48
test/cctest/test-parsing.cc
View File

@ -93,7 +93,7 @@ TEST(ScanKeywords) {
CHECK(static_cast<int>(sizeof(buffer)) >= length);
{
auto stream = i::ScannerStream::ForTesting(keyword, length);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), false);
CHECK_EQ(key_token.token, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
@ -101,7 +101,7 @@ TEST(ScanKeywords) {
// Removing characters will make keyword matching fail.
{
auto stream = i::ScannerStream::ForTesting(keyword, length - 1);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), false);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
@ -112,7 +112,7 @@ TEST(ScanKeywords) {
i::MemMove(buffer, keyword, length);
buffer[length] = chars_to_append[j];
auto stream = i::ScannerStream::ForTesting(buffer, length + 1);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), false);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
@ -122,7 +122,7 @@ TEST(ScanKeywords) {
i::MemMove(buffer, keyword, length);
buffer[length - 1] = '_';
auto stream = i::ScannerStream::ForTesting(buffer, length);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), false);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
@ -189,7 +189,7 @@ TEST(ScanHTMLEndComments) {
for (int i = 0; tests[i]; i++) {
const char* source = tests[i];
auto stream = i::ScannerStream::ForTesting(source);
i::Scanner scanner(i_isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(i_isolate->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Zone zone(i_isolate->allocator(), ZONE_NAME);
i::AstValueFactory ast_value_factory(&zone,
@ -208,7 +208,7 @@ TEST(ScanHTMLEndComments) {
for (int i = 0; fail_tests[i]; i++) {
const char* source = fail_tests[i];
auto stream = i::ScannerStream::ForTesting(source);
i::Scanner scanner(i_isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(i_isolate->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Zone zone(i_isolate->allocator(), ZONE_NAME);
i::AstValueFactory ast_value_factory(&zone,
@ -233,7 +233,7 @@ TEST(ScanHtmlComments) {
// Disallow HTML comments.
{
auto stream = i::ScannerStream::ForTesting(src);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), true);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::ILLEGAL, scanner.Next());
@ -242,7 +242,7 @@ TEST(ScanHtmlComments) {
// Skip HTML comments:
{
auto stream = i::ScannerStream::ForTesting(src);
i::Scanner scanner(&unicode_cache, global_use_counts);
i::Scanner scanner(&unicode_cache);
scanner.Initialize(stream.get(), false);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
@ -400,7 +400,7 @@ TEST(StandAlonePreParser) {
uintptr_t stack_limit = i_isolate->stack_guard()->real_climit();
for (int i = 0; programs[i]; i++) {
auto stream = i::ScannerStream::ForTesting(programs[i]);
i::Scanner scanner(i_isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(i_isolate->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Zone zone(i_isolate->allocator(), ZONE_NAME);
@ -433,7 +433,7 @@ TEST(StandAlonePreParserNoNatives) {
uintptr_t stack_limit = isolate->stack_guard()->real_climit();
for (int i = 0; programs[i]; i++) {
auto stream = i::ScannerStream::ForTesting(programs[i]);
i::Scanner scanner(isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(isolate->unicode_cache());
scanner.Initialize(stream.get(), false);
// Preparser defaults to disallowing natives syntax.
@ -504,7 +504,7 @@ TEST(RegressChromium62639) {
// failed in debug mode, and sometimes crashed in release mode.
auto stream = i::ScannerStream::ForTesting(program);
i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), global_use_counts);
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
i::AstValueFactory ast_value_factory(
@ -579,7 +579,7 @@ TEST(PreParseOverflow) {
uintptr_t stack_limit = isolate->stack_guard()->real_climit();
auto stream = i::ScannerStream::ForTesting(program.get(), kProgramSize);
i::Scanner scanner(isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(isolate->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
@ -599,7 +599,7 @@ void TestStreamScanner(i::Utf16CharacterStream* stream,
i::Token::Value* expected_tokens,
int skip_pos = 0, // Zero means not skipping.
int skip_to = 0) {
i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), global_use_counts);
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
scanner.Initialize(stream, false);
int i = 0;
@ -677,7 +677,7 @@ TEST(StreamScanner) {
void TestScanRegExp(const char* re_source, const char* expected) {
auto stream = i::ScannerStream::ForTesting(re_source);
i::HandleScope scope(CcTest::i_isolate());
i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), global_use_counts);
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
scanner.Initialize(stream.get(), false);
i::Token::Value start = scanner.peek();
@ -1362,7 +1362,7 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
// Preparse the data.
i::PendingCompilationErrorHandler pending_error_handler;
if (test_preparser) {
i::Scanner scanner(isolate->unicode_cache(), global_use_counts);
i::Scanner scanner(isolate->unicode_cache());
std::unique_ptr<i::Utf16CharacterStream> stream(
i::ScannerStream::For(source));
i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
@ -3843,24 +3843,6 @@ TEST(LineOrParagraphSeparatorAsLineTerminator) {
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(LineOrParagraphSeparatorAsLineTerminatorUseCount) {
i::Isolate* isolate = CcTest::i_isolate();
i::HandleScope scope(isolate);
LocalContext env;
int use_counts[v8::Isolate::kUseCounterFeatureCount] = {};
global_use_counts = use_counts;
CcTest::isolate()->SetUseCounterCallback(MockUseCounterCallback);
CompileRun("");
CHECK_EQ(0, use_counts[v8::Isolate::UseCounterFeature::
kLineOrParagraphSeparatorAsLineTerminator]);
CompileRun("// Foo\xE2\x80\xA8"); // "// Foo<U+2028>"
CHECK_LT(0, use_counts[v8::Isolate::UseCounterFeature::
kLineOrParagraphSeparatorAsLineTerminator]);
CompileRun("// Foo\xE2\x80\xA9"); // "// Foo<U+2029>"
CHECK_LT(1, use_counts[v8::Isolate::UseCounterFeature::
kLineOrParagraphSeparatorAsLineTerminator]);
}
TEST(ErrorsArrowFormalParameters) {
const char* context_data[][2] = {
{ "()", "=>{}" },