move remaining uses of scanner literals into scanner
R=marja@chromium.org BUG= Review URL: https://codereview.chromium.org/198713002 git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19880 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
This commit is contained in:
parent
4da15df008
commit
750ab88341
@ -1164,14 +1164,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
|
||||
reserved_error_loc = scanner()->location();
|
||||
}
|
||||
|
||||
int prev_value;
|
||||
if (scanner()->is_literal_one_byte()) {
|
||||
prev_value = duplicate_finder.AddAsciiSymbol(
|
||||
scanner()->literal_one_byte_string(), 1);
|
||||
} else {
|
||||
prev_value =
|
||||
duplicate_finder.AddUtf16Symbol(scanner()->literal_utf16_string(), 1);
|
||||
}
|
||||
int prev_value = scanner()->FindSymbol(&duplicate_finder, 1);
|
||||
|
||||
if (!dupe_error_loc.IsValid() && prev_value != 0) {
|
||||
dupe_error_loc = scanner()->location();
|
||||
@ -1273,12 +1266,7 @@ PreParser::Expression PreParser::ParseV8Intrinsic(bool* ok) {
|
||||
|
||||
|
||||
void PreParser::LogSymbol() {
|
||||
int identifier_pos = position();
|
||||
if (scanner()->is_literal_one_byte()) {
|
||||
log_->LogAsciiSymbol(identifier_pos, scanner()->literal_one_byte_string());
|
||||
} else {
|
||||
log_->LogUtf16Symbol(identifier_pos, scanner()->literal_utf16_string());
|
||||
}
|
||||
scanner()->LogSymbol(log_, position());
|
||||
}
|
||||
|
||||
|
||||
|
@ -1549,11 +1549,9 @@ void ParserBase<Traits>::ObjectLiteralChecker::CheckProperty(
|
||||
bool* ok) {
|
||||
int old;
|
||||
if (property == Token::NUMBER) {
|
||||
old = finder_.AddNumber(scanner()->literal_one_byte_string(), type);
|
||||
} else if (scanner()->is_literal_one_byte()) {
|
||||
old = finder_.AddAsciiSymbol(scanner()->literal_one_byte_string(), type);
|
||||
old = scanner()->FindNumber(&finder_, type);
|
||||
} else {
|
||||
old = finder_.AddUtf16Symbol(scanner()->literal_utf16_string(), type);
|
||||
old = scanner()->FindSymbol(&finder_, type);
|
||||
}
|
||||
PropertyKind old_type = static_cast<PropertyKind>(old);
|
||||
if (HasConflict(old_type, type)) {
|
||||
|
@ -36,6 +36,7 @@
|
||||
#include "conversions-inl.h"
|
||||
#include "list-inl.h"
|
||||
#include "v8.h"
|
||||
#include "parser.h"
|
||||
|
||||
namespace v8 {
|
||||
namespace internal {
|
||||
@ -1115,18 +1116,6 @@ bool Scanner::ScanRegExpFlags() {
|
||||
}
|
||||
|
||||
|
||||
Handle<String> Scanner::AllocateLiteralString(Isolate* isolate,
|
||||
PretenureFlag tenured) {
|
||||
if (is_literal_one_byte()) {
|
||||
return isolate->factory()->NewStringFromOneByte(
|
||||
Vector<const uint8_t>::cast(literal_one_byte_string()), tenured);
|
||||
} else {
|
||||
return isolate->factory()->NewStringFromTwoByte(
|
||||
literal_utf16_string(), tenured);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Handle<String> Scanner::AllocateNextLiteralString(Isolate* isolate,
|
||||
PretenureFlag tenured) {
|
||||
if (is_next_literal_one_byte()) {
|
||||
@ -1158,6 +1147,28 @@ double Scanner::DoubleValue() {
|
||||
}
|
||||
|
||||
|
||||
int Scanner::FindNumber(DuplicateFinder* finder, int value) {
|
||||
return finder->AddNumber(literal_one_byte_string(), value);
|
||||
}
|
||||
|
||||
|
||||
int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
|
||||
if (is_literal_one_byte()) {
|
||||
return finder->AddAsciiSymbol(literal_one_byte_string(), value);
|
||||
}
|
||||
return finder->AddUtf16Symbol(literal_utf16_string(), value);
|
||||
}
|
||||
|
||||
|
||||
void Scanner::LogSymbol(ParserRecorder* log, int position) {
|
||||
if (is_literal_one_byte()) {
|
||||
log->LogAsciiSymbol(position, literal_one_byte_string());
|
||||
} else {
|
||||
log->LogUtf16Symbol(position, literal_utf16_string());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) {
|
||||
return AddSymbol(Vector<const byte>::cast(key), true, value);
|
||||
}
|
||||
|
121
src/scanner.h
121
src/scanner.h
@ -44,6 +44,9 @@ namespace v8 {
|
||||
namespace internal {
|
||||
|
||||
|
||||
class ParserRecorder;
|
||||
|
||||
|
||||
// Returns the value (0 .. 15) of a hexadecimal character c.
|
||||
// If c is not a legal hexadecimal character, returns a value < 0.
|
||||
inline int HexValue(uc32 c) {
|
||||
@ -370,32 +373,13 @@ class Scanner {
|
||||
// Returns the location information for the current token
|
||||
// (the token last returned by Next()).
|
||||
Location location() const { return current_.location; }
|
||||
// Returns the literal string, if any, for the current token (the
|
||||
// token last returned by Next()). The string is 0-terminated.
|
||||
// Literal strings are collected for identifiers, strings, and
|
||||
// numbers.
|
||||
// These functions only give the correct result if the literal
|
||||
// was scanned between calls to StartLiteral() and TerminateLiteral().
|
||||
Vector<const char> literal_one_byte_string() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->one_byte_literal();
|
||||
}
|
||||
Vector<const uc16> literal_utf16_string() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->utf16_literal();
|
||||
}
|
||||
bool is_literal_one_byte() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->is_one_byte();
|
||||
}
|
||||
bool is_literal_contextual_keyword(Vector<const char> keyword) {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->is_contextual_keyword(keyword);
|
||||
}
|
||||
int literal_length() const {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->length();
|
||||
}
|
||||
|
||||
// Similar functions for the upcoming token.
|
||||
|
||||
// One token look-ahead (past the token returned by Next()).
|
||||
Token::Value peek() const { return next_.token; }
|
||||
|
||||
Location peek_location() const { return next_.location; }
|
||||
|
||||
bool literal_contains_escapes() const {
|
||||
Location location = current_.location;
|
||||
@ -406,38 +390,15 @@ class Scanner {
|
||||
}
|
||||
return current_.literal_chars->length() != source_length;
|
||||
}
|
||||
|
||||
// Similar functions for the upcoming token.
|
||||
|
||||
// One token look-ahead (past the token returned by Next()).
|
||||
Token::Value peek() const { return next_.token; }
|
||||
|
||||
Location peek_location() const { return next_.location; }
|
||||
|
||||
// Returns the literal string for the next token (the token that
|
||||
// would be returned if Next() were called).
|
||||
Vector<const char> next_literal_one_byte_string() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->one_byte_literal();
|
||||
}
|
||||
Vector<const uc16> next_literal_utf16_string() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->utf16_literal();
|
||||
}
|
||||
bool is_next_literal_one_byte() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->is_one_byte();
|
||||
bool is_literal_contextual_keyword(Vector<const char> keyword) {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->is_contextual_keyword(keyword);
|
||||
}
|
||||
bool is_next_contextual_keyword(Vector<const char> keyword) {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->is_contextual_keyword(keyword);
|
||||
}
|
||||
int next_literal_length() const {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->length();
|
||||
}
|
||||
|
||||
Handle<String> AllocateLiteralString(Isolate* isolate, PretenureFlag tenured);
|
||||
Handle<String> AllocateNextLiteralString(Isolate* isolate,
|
||||
PretenureFlag tenured);
|
||||
Handle<String> AllocateInternalizedString(Isolate* isolate);
|
||||
@ -461,13 +422,13 @@ class Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
int FindNumber(DuplicateFinder* finder, int value);
|
||||
int FindSymbol(DuplicateFinder* finder, int value);
|
||||
|
||||
void LogSymbol(ParserRecorder* log, int position);
|
||||
|
||||
UnicodeCache* unicode_cache() { return unicode_cache_; }
|
||||
|
||||
static const int kCharacterLookaheadBufferSize = 1;
|
||||
|
||||
// Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
|
||||
uc32 ScanOctalEscape(uc32 c, int length);
|
||||
|
||||
// Returns the location of the last seen octal literal.
|
||||
Location octal_position() const { return octal_pos_; }
|
||||
void clear_octal_position() { octal_pos_ = Location::invalid(); }
|
||||
@ -519,6 +480,11 @@ class Scanner {
|
||||
LiteralBuffer* literal_chars;
|
||||
};
|
||||
|
||||
static const int kCharacterLookaheadBufferSize = 1;
|
||||
|
||||
// Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
|
||||
uc32 ScanOctalEscape(uc32 c, int length);
|
||||
|
||||
// Call this after setting source_ to the input.
|
||||
void Init() {
|
||||
// Set c0_ (one character ahead)
|
||||
@ -579,6 +545,47 @@ class Scanner {
|
||||
}
|
||||
}
|
||||
|
||||
// Returns the literal string, if any, for the current token (the
|
||||
// token last returned by Next()). The string is 0-terminated.
|
||||
// Literal strings are collected for identifiers, strings, and
|
||||
// numbers.
|
||||
// These functions only give the correct result if the literal
|
||||
// was scanned between calls to StartLiteral() and TerminateLiteral().
|
||||
Vector<const char> literal_one_byte_string() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->one_byte_literal();
|
||||
}
|
||||
Vector<const uc16> literal_utf16_string() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->utf16_literal();
|
||||
}
|
||||
bool is_literal_one_byte() {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->is_one_byte();
|
||||
}
|
||||
int literal_length() const {
|
||||
ASSERT_NOT_NULL(current_.literal_chars);
|
||||
return current_.literal_chars->length();
|
||||
}
|
||||
// Returns the literal string for the next token (the token that
|
||||
// would be returned if Next() were called).
|
||||
Vector<const char> next_literal_one_byte_string() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->one_byte_literal();
|
||||
}
|
||||
Vector<const uc16> next_literal_utf16_string() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->utf16_literal();
|
||||
}
|
||||
bool is_next_literal_one_byte() {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->is_one_byte();
|
||||
}
|
||||
int next_literal_length() const {
|
||||
ASSERT_NOT_NULL(next_.literal_chars);
|
||||
return next_.literal_chars->length();
|
||||
}
|
||||
|
||||
uc32 ScanHexNumber(int expected_length);
|
||||
|
||||
// Scans a single JavaScript token.
|
||||
|
@ -795,6 +795,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
|
||||
i::Utf8ToUtf16CharacterStream stream(
|
||||
reinterpret_cast<const i::byte*>(re_source),
|
||||
static_cast<unsigned>(strlen(re_source)));
|
||||
i::HandleScope scope(CcTest::i_isolate());
|
||||
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
|
||||
scanner.Initialize(&stream);
|
||||
|
||||
@ -802,8 +803,12 @@ void TestScanRegExp(const char* re_source, const char* expected) {
|
||||
CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
|
||||
CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
|
||||
scanner.Next(); // Current token is now the regexp literal.
|
||||
CHECK(scanner.is_literal_one_byte());
|
||||
i::Vector<const char> actual = scanner.literal_one_byte_string();
|
||||
i::Handle<i::String> val =
|
||||
scanner.AllocateInternalizedString(CcTest::i_isolate());
|
||||
i::DisallowHeapAllocation no_alloc;
|
||||
i::String::FlatContent content = val->GetFlatContent();
|
||||
CHECK(content.IsAscii());
|
||||
i::Vector<const uint8_t> actual = content.ToOneByteVector();
|
||||
for (int i = 0; i < actual.length(); i++) {
|
||||
CHECK_NE('\0', expected[i]);
|
||||
CHECK_EQ(expected[i], actual[i]);
|
||||
|
Loading…
Reference in New Issue
Block a user