move remaining uses of scanner literals into scanner

R=marja@chromium.org

BUG=

Review URL: https://codereview.chromium.org/198713002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19880 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
This commit is contained in:
dcarney@chromium.org 2014-03-13 08:29:31 +00:00
parent 4da15df008
commit 750ab88341
5 changed files with 98 additions and 89 deletions

View File

@ -1164,14 +1164,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
reserved_error_loc = scanner()->location();
}
int prev_value;
if (scanner()->is_literal_one_byte()) {
prev_value = duplicate_finder.AddAsciiSymbol(
scanner()->literal_one_byte_string(), 1);
} else {
prev_value =
duplicate_finder.AddUtf16Symbol(scanner()->literal_utf16_string(), 1);
}
int prev_value = scanner()->FindSymbol(&duplicate_finder, 1);
if (!dupe_error_loc.IsValid() && prev_value != 0) {
dupe_error_loc = scanner()->location();
@ -1273,12 +1266,7 @@ PreParser::Expression PreParser::ParseV8Intrinsic(bool* ok) {
void PreParser::LogSymbol() {
int identifier_pos = position();
if (scanner()->is_literal_one_byte()) {
log_->LogAsciiSymbol(identifier_pos, scanner()->literal_one_byte_string());
} else {
log_->LogUtf16Symbol(identifier_pos, scanner()->literal_utf16_string());
}
scanner()->LogSymbol(log_, position());
}

View File

@ -1549,11 +1549,9 @@ void ParserBase<Traits>::ObjectLiteralChecker::CheckProperty(
bool* ok) {
int old;
if (property == Token::NUMBER) {
old = finder_.AddNumber(scanner()->literal_one_byte_string(), type);
} else if (scanner()->is_literal_one_byte()) {
old = finder_.AddAsciiSymbol(scanner()->literal_one_byte_string(), type);
old = scanner()->FindNumber(&finder_, type);
} else {
old = finder_.AddUtf16Symbol(scanner()->literal_utf16_string(), type);
old = scanner()->FindSymbol(&finder_, type);
}
PropertyKind old_type = static_cast<PropertyKind>(old);
if (HasConflict(old_type, type)) {

View File

@ -36,6 +36,7 @@
#include "conversions-inl.h"
#include "list-inl.h"
#include "v8.h"
#include "parser.h"
namespace v8 {
namespace internal {
@ -1115,18 +1116,6 @@ bool Scanner::ScanRegExpFlags() {
}
Handle<String> Scanner::AllocateLiteralString(Isolate* isolate,
PretenureFlag tenured) {
if (is_literal_one_byte()) {
return isolate->factory()->NewStringFromOneByte(
Vector<const uint8_t>::cast(literal_one_byte_string()), tenured);
} else {
return isolate->factory()->NewStringFromTwoByte(
literal_utf16_string(), tenured);
}
}
Handle<String> Scanner::AllocateNextLiteralString(Isolate* isolate,
PretenureFlag tenured) {
if (is_next_literal_one_byte()) {
@ -1158,6 +1147,28 @@ double Scanner::DoubleValue() {
}
int Scanner::FindNumber(DuplicateFinder* finder, int value) {
return finder->AddNumber(literal_one_byte_string(), value);
}
int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
if (is_literal_one_byte()) {
return finder->AddAsciiSymbol(literal_one_byte_string(), value);
}
return finder->AddUtf16Symbol(literal_utf16_string(), value);
}
void Scanner::LogSymbol(ParserRecorder* log, int position) {
if (is_literal_one_byte()) {
log->LogAsciiSymbol(position, literal_one_byte_string());
} else {
log->LogUtf16Symbol(position, literal_utf16_string());
}
}
int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) {
return AddSymbol(Vector<const byte>::cast(key), true, value);
}

View File

@ -44,6 +44,9 @@ namespace v8 {
namespace internal {
class ParserRecorder;
// Returns the value (0 .. 15) of a hexadecimal character c.
// If c is not a legal hexadecimal character, returns a value < 0.
inline int HexValue(uc32 c) {
@ -370,32 +373,13 @@ class Scanner {
// Returns the location information for the current token
// (the token last returned by Next()).
Location location() const { return current_.location; }
// Returns the literal string, if any, for the current token (the
// token last returned by Next()). The string is 0-terminated.
// Literal strings are collected for identifiers, strings, and
// numbers.
// These functions only give the correct result if the literal
// was scanned between calls to StartLiteral() and TerminateLiteral().
Vector<const char> literal_one_byte_string() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->one_byte_literal();
}
Vector<const uc16> literal_utf16_string() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->utf16_literal();
}
bool is_literal_one_byte() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->is_one_byte();
}
bool is_literal_contextual_keyword(Vector<const char> keyword) {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->is_contextual_keyword(keyword);
}
int literal_length() const {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->length();
}
// Similar functions for the upcoming token.
// One token look-ahead (past the token returned by Next()).
Token::Value peek() const { return next_.token; }
Location peek_location() const { return next_.location; }
bool literal_contains_escapes() const {
Location location = current_.location;
@ -406,38 +390,15 @@ class Scanner {
}
return current_.literal_chars->length() != source_length;
}
// Similar functions for the upcoming token.
// One token look-ahead (past the token returned by Next()).
Token::Value peek() const { return next_.token; }
Location peek_location() const { return next_.location; }
// Returns the literal string for the next token (the token that
// would be returned if Next() were called).
Vector<const char> next_literal_one_byte_string() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->one_byte_literal();
}
Vector<const uc16> next_literal_utf16_string() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->utf16_literal();
}
bool is_next_literal_one_byte() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->is_one_byte();
bool is_literal_contextual_keyword(Vector<const char> keyword) {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->is_contextual_keyword(keyword);
}
bool is_next_contextual_keyword(Vector<const char> keyword) {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->is_contextual_keyword(keyword);
}
int next_literal_length() const {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->length();
}
Handle<String> AllocateLiteralString(Isolate* isolate, PretenureFlag tenured);
Handle<String> AllocateNextLiteralString(Isolate* isolate,
PretenureFlag tenured);
Handle<String> AllocateInternalizedString(Isolate* isolate);
@ -461,13 +422,13 @@ class Scanner {
}
}
int FindNumber(DuplicateFinder* finder, int value);
int FindSymbol(DuplicateFinder* finder, int value);
void LogSymbol(ParserRecorder* log, int position);
UnicodeCache* unicode_cache() { return unicode_cache_; }
static const int kCharacterLookaheadBufferSize = 1;
// Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
uc32 ScanOctalEscape(uc32 c, int length);
// Returns the location of the last seen octal literal.
Location octal_position() const { return octal_pos_; }
void clear_octal_position() { octal_pos_ = Location::invalid(); }
@ -519,6 +480,11 @@ class Scanner {
LiteralBuffer* literal_chars;
};
static const int kCharacterLookaheadBufferSize = 1;
// Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
uc32 ScanOctalEscape(uc32 c, int length);
// Call this after setting source_ to the input.
void Init() {
// Set c0_ (one character ahead)
@ -579,6 +545,47 @@ class Scanner {
}
}
// Returns the literal string, if any, for the current token (the
// token last returned by Next()). The string is 0-terminated.
// Literal strings are collected for identifiers, strings, and
// numbers.
// These functions only give the correct result if the literal
// was scanned between calls to StartLiteral() and TerminateLiteral().
Vector<const char> literal_one_byte_string() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->one_byte_literal();
}
Vector<const uc16> literal_utf16_string() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->utf16_literal();
}
bool is_literal_one_byte() {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->is_one_byte();
}
int literal_length() const {
ASSERT_NOT_NULL(current_.literal_chars);
return current_.literal_chars->length();
}
// Returns the literal string for the next token (the token that
// would be returned if Next() were called).
Vector<const char> next_literal_one_byte_string() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->one_byte_literal();
}
Vector<const uc16> next_literal_utf16_string() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->utf16_literal();
}
bool is_next_literal_one_byte() {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->is_one_byte();
}
int next_literal_length() const {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->length();
}
uc32 ScanHexNumber(int expected_length);
// Scans a single JavaScript token.

View File

@ -795,6 +795,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
i::Utf8ToUtf16CharacterStream stream(
reinterpret_cast<const i::byte*>(re_source),
static_cast<unsigned>(strlen(re_source)));
i::HandleScope scope(CcTest::i_isolate());
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
scanner.Initialize(&stream);
@ -802,8 +803,12 @@ void TestScanRegExp(const char* re_source, const char* expected) {
CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
scanner.Next(); // Current token is now the regexp literal.
CHECK(scanner.is_literal_one_byte());
i::Vector<const char> actual = scanner.literal_one_byte_string();
i::Handle<i::String> val =
scanner.AllocateInternalizedString(CcTest::i_isolate());
i::DisallowHeapAllocation no_alloc;
i::String::FlatContent content = val->GetFlatContent();
CHECK(content.IsAscii());
i::Vector<const uint8_t> actual = content.ToOneByteVector();
for (int i = 0; i < actual.length(); i++) {
CHECK_NE('\0', expected[i]);
CHECK_EQ(expected[i], actual[i]);