From 750ab8834165cbc00b3df348f10f931d80d778b3 Mon Sep 17 00:00:00 2001 From: "dcarney@chromium.org" Date: Thu, 13 Mar 2014 08:29:31 +0000 Subject: [PATCH] move remaining uses of scanner literals into scanner R=marja@chromium.org BUG= Review URL: https://codereview.chromium.org/198713002 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19880 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- src/preparser.cc | 16 +---- src/preparser.h | 6 +- src/scanner.cc | 35 +++++++---- src/scanner.h | 121 +++++++++++++++++++----------------- test/cctest/test-parsing.cc | 9 ++- 5 files changed, 98 insertions(+), 89 deletions(-) diff --git a/src/preparser.cc b/src/preparser.cc index 8865c505bf..bd10db3d01 100644 --- a/src/preparser.cc +++ b/src/preparser.cc @@ -1164,14 +1164,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral( reserved_error_loc = scanner()->location(); } - int prev_value; - if (scanner()->is_literal_one_byte()) { - prev_value = duplicate_finder.AddAsciiSymbol( - scanner()->literal_one_byte_string(), 1); - } else { - prev_value = - duplicate_finder.AddUtf16Symbol(scanner()->literal_utf16_string(), 1); - } + int prev_value = scanner()->FindSymbol(&duplicate_finder, 1); if (!dupe_error_loc.IsValid() && prev_value != 0) { dupe_error_loc = scanner()->location(); @@ -1273,12 +1266,7 @@ PreParser::Expression PreParser::ParseV8Intrinsic(bool* ok) { void PreParser::LogSymbol() { - int identifier_pos = position(); - if (scanner()->is_literal_one_byte()) { - log_->LogAsciiSymbol(identifier_pos, scanner()->literal_one_byte_string()); - } else { - log_->LogUtf16Symbol(identifier_pos, scanner()->literal_utf16_string()); - } + scanner()->LogSymbol(log_, position()); } diff --git a/src/preparser.h b/src/preparser.h index 6a98fe94b4..06880d5274 100644 --- a/src/preparser.h +++ b/src/preparser.h @@ -1549,11 +1549,9 @@ void ParserBase::ObjectLiteralChecker::CheckProperty( bool* ok) { int old; if (property == Token::NUMBER) { - old = finder_.AddNumber(scanner()->literal_one_byte_string(), type); - } else if (scanner()->is_literal_one_byte()) { - old = finder_.AddAsciiSymbol(scanner()->literal_one_byte_string(), type); + old = scanner()->FindNumber(&finder_, type); } else { - old = finder_.AddUtf16Symbol(scanner()->literal_utf16_string(), type); + old = scanner()->FindSymbol(&finder_, type); } PropertyKind old_type = static_cast(old); if (HasConflict(old_type, type)) { diff --git a/src/scanner.cc b/src/scanner.cc index 45e0cae876..a48be2989a 100644 --- a/src/scanner.cc +++ b/src/scanner.cc @@ -36,6 +36,7 @@ #include "conversions-inl.h" #include "list-inl.h" #include "v8.h" +#include "parser.h" namespace v8 { namespace internal { @@ -1115,18 +1116,6 @@ bool Scanner::ScanRegExpFlags() { } -Handle Scanner::AllocateLiteralString(Isolate* isolate, - PretenureFlag tenured) { - if (is_literal_one_byte()) { - return isolate->factory()->NewStringFromOneByte( - Vector::cast(literal_one_byte_string()), tenured); - } else { - return isolate->factory()->NewStringFromTwoByte( - literal_utf16_string(), tenured); - } -} - - Handle Scanner::AllocateNextLiteralString(Isolate* isolate, PretenureFlag tenured) { if (is_next_literal_one_byte()) { @@ -1158,6 +1147,28 @@ double Scanner::DoubleValue() { } +int Scanner::FindNumber(DuplicateFinder* finder, int value) { + return finder->AddNumber(literal_one_byte_string(), value); +} + + +int Scanner::FindSymbol(DuplicateFinder* finder, int value) { + if (is_literal_one_byte()) { + return finder->AddAsciiSymbol(literal_one_byte_string(), value); + } + return finder->AddUtf16Symbol(literal_utf16_string(), value); +} + + +void Scanner::LogSymbol(ParserRecorder* log, int position) { + if (is_literal_one_byte()) { + log->LogAsciiSymbol(position, literal_one_byte_string()); + } else { + log->LogUtf16Symbol(position, literal_utf16_string()); + } +} + + int DuplicateFinder::AddAsciiSymbol(Vector key, int value) { return AddSymbol(Vector::cast(key), true, value); } diff --git a/src/scanner.h b/src/scanner.h index b6a5603ae6..367714c5b9 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -44,6 +44,9 @@ namespace v8 { namespace internal { +class ParserRecorder; + + // Returns the value (0 .. 15) of a hexadecimal character c. // If c is not a legal hexadecimal character, returns a value < 0. inline int HexValue(uc32 c) { @@ -370,32 +373,13 @@ class Scanner { // Returns the location information for the current token // (the token last returned by Next()). Location location() const { return current_.location; } - // Returns the literal string, if any, for the current token (the - // token last returned by Next()). The string is 0-terminated. - // Literal strings are collected for identifiers, strings, and - // numbers. - // These functions only give the correct result if the literal - // was scanned between calls to StartLiteral() and TerminateLiteral(). - Vector literal_one_byte_string() { - ASSERT_NOT_NULL(current_.literal_chars); - return current_.literal_chars->one_byte_literal(); - } - Vector literal_utf16_string() { - ASSERT_NOT_NULL(current_.literal_chars); - return current_.literal_chars->utf16_literal(); - } - bool is_literal_one_byte() { - ASSERT_NOT_NULL(current_.literal_chars); - return current_.literal_chars->is_one_byte(); - } - bool is_literal_contextual_keyword(Vector keyword) { - ASSERT_NOT_NULL(current_.literal_chars); - return current_.literal_chars->is_contextual_keyword(keyword); - } - int literal_length() const { - ASSERT_NOT_NULL(current_.literal_chars); - return current_.literal_chars->length(); - } + + // Similar functions for the upcoming token. + + // One token look-ahead (past the token returned by Next()). + Token::Value peek() const { return next_.token; } + + Location peek_location() const { return next_.location; } bool literal_contains_escapes() const { Location location = current_.location; @@ -406,38 +390,15 @@ class Scanner { } return current_.literal_chars->length() != source_length; } - - // Similar functions for the upcoming token. - - // One token look-ahead (past the token returned by Next()). - Token::Value peek() const { return next_.token; } - - Location peek_location() const { return next_.location; } - - // Returns the literal string for the next token (the token that - // would be returned if Next() were called). - Vector next_literal_one_byte_string() { - ASSERT_NOT_NULL(next_.literal_chars); - return next_.literal_chars->one_byte_literal(); - } - Vector next_literal_utf16_string() { - ASSERT_NOT_NULL(next_.literal_chars); - return next_.literal_chars->utf16_literal(); - } - bool is_next_literal_one_byte() { - ASSERT_NOT_NULL(next_.literal_chars); - return next_.literal_chars->is_one_byte(); + bool is_literal_contextual_keyword(Vector keyword) { + ASSERT_NOT_NULL(current_.literal_chars); + return current_.literal_chars->is_contextual_keyword(keyword); } bool is_next_contextual_keyword(Vector keyword) { ASSERT_NOT_NULL(next_.literal_chars); return next_.literal_chars->is_contextual_keyword(keyword); } - int next_literal_length() const { - ASSERT_NOT_NULL(next_.literal_chars); - return next_.literal_chars->length(); - } - Handle AllocateLiteralString(Isolate* isolate, PretenureFlag tenured); Handle AllocateNextLiteralString(Isolate* isolate, PretenureFlag tenured); Handle AllocateInternalizedString(Isolate* isolate); @@ -461,13 +422,13 @@ class Scanner { } } + int FindNumber(DuplicateFinder* finder, int value); + int FindSymbol(DuplicateFinder* finder, int value); + + void LogSymbol(ParserRecorder* log, int position); + UnicodeCache* unicode_cache() { return unicode_cache_; } - static const int kCharacterLookaheadBufferSize = 1; - - // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. - uc32 ScanOctalEscape(uc32 c, int length); - // Returns the location of the last seen octal literal. Location octal_position() const { return octal_pos_; } void clear_octal_position() { octal_pos_ = Location::invalid(); } @@ -519,6 +480,11 @@ class Scanner { LiteralBuffer* literal_chars; }; + static const int kCharacterLookaheadBufferSize = 1; + + // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. + uc32 ScanOctalEscape(uc32 c, int length); + // Call this after setting source_ to the input. void Init() { // Set c0_ (one character ahead) @@ -579,6 +545,47 @@ class Scanner { } } + // Returns the literal string, if any, for the current token (the + // token last returned by Next()). The string is 0-terminated. + // Literal strings are collected for identifiers, strings, and + // numbers. + // These functions only give the correct result if the literal + // was scanned between calls to StartLiteral() and TerminateLiteral(). + Vector literal_one_byte_string() { + ASSERT_NOT_NULL(current_.literal_chars); + return current_.literal_chars->one_byte_literal(); + } + Vector literal_utf16_string() { + ASSERT_NOT_NULL(current_.literal_chars); + return current_.literal_chars->utf16_literal(); + } + bool is_literal_one_byte() { + ASSERT_NOT_NULL(current_.literal_chars); + return current_.literal_chars->is_one_byte(); + } + int literal_length() const { + ASSERT_NOT_NULL(current_.literal_chars); + return current_.literal_chars->length(); + } + // Returns the literal string for the next token (the token that + // would be returned if Next() were called). + Vector next_literal_one_byte_string() { + ASSERT_NOT_NULL(next_.literal_chars); + return next_.literal_chars->one_byte_literal(); + } + Vector next_literal_utf16_string() { + ASSERT_NOT_NULL(next_.literal_chars); + return next_.literal_chars->utf16_literal(); + } + bool is_next_literal_one_byte() { + ASSERT_NOT_NULL(next_.literal_chars); + return next_.literal_chars->is_one_byte(); + } + int next_literal_length() const { + ASSERT_NOT_NULL(next_.literal_chars); + return next_.literal_chars->length(); + } + uc32 ScanHexNumber(int expected_length); // Scans a single JavaScript token. diff --git a/test/cctest/test-parsing.cc b/test/cctest/test-parsing.cc index 3a20595db8..673660a529 100644 --- a/test/cctest/test-parsing.cc +++ b/test/cctest/test-parsing.cc @@ -795,6 +795,7 @@ void TestScanRegExp(const char* re_source, const char* expected) { i::Utf8ToUtf16CharacterStream stream( reinterpret_cast(re_source), static_cast(strlen(re_source))); + i::HandleScope scope(CcTest::i_isolate()); i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); scanner.Initialize(&stream); @@ -802,8 +803,12 @@ void TestScanRegExp(const char* re_source, const char* expected) { CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV); CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV)); scanner.Next(); // Current token is now the regexp literal. - CHECK(scanner.is_literal_one_byte()); - i::Vector actual = scanner.literal_one_byte_string(); + i::Handle val = + scanner.AllocateInternalizedString(CcTest::i_isolate()); + i::DisallowHeapAllocation no_alloc; + i::String::FlatContent content = val->GetFlatContent(); + CHECK(content.IsAscii()); + i::Vector actual = content.ToOneByteVector(); for (int i = 0; i < actual.length(); i++) { CHECK_NE('\0', expected[i]); CHECK_EQ(expected[i], actual[i]);