move remaining uses of scanner literals into scanner

R=marja@chromium.org

BUG=

Review URL: https://codereview.chromium.org/198713002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19880 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/preparser.cc

@@ -1164,14 +1164,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
       reserved_error_loc = scanner()->location();
     }
 
-    int prev_value;
+    int prev_value = scanner()->FindSymbol(&duplicate_finder, 1);
-    if (scanner()->is_literal_one_byte()) {
-      prev_value = duplicate_finder.AddAsciiSymbol(
-          scanner()->literal_one_byte_string(), 1);
-    } else {
-      prev_value =
-          duplicate_finder.AddUtf16Symbol(scanner()->literal_utf16_string(), 1);
-    }
 
     if (!dupe_error_loc.IsValid() && prev_value != 0) {
       dupe_error_loc = scanner()->location();
@@ -1273,12 +1266,7 @@ PreParser::Expression PreParser::ParseV8Intrinsic(bool* ok) {
 
 
 void PreParser::LogSymbol() {
-  int identifier_pos = position();
+  scanner()->LogSymbol(log_, position());
-  if (scanner()->is_literal_one_byte()) {
-    log_->LogAsciiSymbol(identifier_pos, scanner()->literal_one_byte_string());
-  } else {
-    log_->LogUtf16Symbol(identifier_pos, scanner()->literal_utf16_string());
-  }
 }
 
 

src/preparser.h

@@ -1549,11 +1549,9 @@ void ParserBase<Traits>::ObjectLiteralChecker::CheckProperty(
     bool* ok) {
   int old;
   if (property == Token::NUMBER) {
-    old = finder_.AddNumber(scanner()->literal_one_byte_string(), type);
+    old = scanner()->FindNumber(&finder_, type);
-  } else if (scanner()->is_literal_one_byte()) {
-    old = finder_.AddAsciiSymbol(scanner()->literal_one_byte_string(), type);
   } else {
-    old = finder_.AddUtf16Symbol(scanner()->literal_utf16_string(), type);
+    old = scanner()->FindSymbol(&finder_, type);
   }
   PropertyKind old_type = static_cast<PropertyKind>(old);
   if (HasConflict(old_type, type)) {

src/scanner.cc

@@ -36,6 +36,7 @@
 #include "conversions-inl.h"
 #include "list-inl.h"
 #include "v8.h"
+#include "parser.h"
 
 namespace v8 {
 namespace internal {
@@ -1115,18 +1116,6 @@ bool Scanner::ScanRegExpFlags() {
 }
 
 
-Handle<String> Scanner::AllocateLiteralString(Isolate* isolate,
-                                              PretenureFlag tenured) {
-  if (is_literal_one_byte()) {
-    return isolate->factory()->NewStringFromOneByte(
-        Vector<const uint8_t>::cast(literal_one_byte_string()), tenured);
-  } else {
-    return isolate->factory()->NewStringFromTwoByte(
-          literal_utf16_string(), tenured);
-  }
-}
-
-
 Handle<String> Scanner::AllocateNextLiteralString(Isolate* isolate,
                                                   PretenureFlag tenured) {
   if (is_next_literal_one_byte()) {
@@ -1158,6 +1147,28 @@ double Scanner::DoubleValue() {
 }
 
 
+int Scanner::FindNumber(DuplicateFinder* finder, int value) {
+  return finder->AddNumber(literal_one_byte_string(), value);
+}
+
+
+int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
+  if (is_literal_one_byte()) {
+    return finder->AddAsciiSymbol(literal_one_byte_string(), value);
+  }
+  return finder->AddUtf16Symbol(literal_utf16_string(), value);
+}
+
+
+void Scanner::LogSymbol(ParserRecorder* log, int position) {
+  if (is_literal_one_byte()) {
+    log->LogAsciiSymbol(position, literal_one_byte_string());
+  } else {
+    log->LogUtf16Symbol(position, literal_utf16_string());
+  }
+}
+
+
 int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) {
   return AddSymbol(Vector<const byte>::cast(key), true, value);
 }

src/scanner.h

@@ -44,6 +44,9 @@ namespace v8 {
 namespace internal {
 
 
+class ParserRecorder;
+
+
 // Returns the value (0 .. 15) of a hexadecimal character c.
 // If c is not a legal hexadecimal character, returns a value < 0.
 inline int HexValue(uc32 c) {
@@ -370,32 +373,13 @@ class Scanner {
   // Returns the location information for the current token
   // (the token last returned by Next()).
   Location location() const { return current_.location; }
-  // Returns the literal string, if any, for the current token (the
+
-  // token last returned by Next()). The string is 0-terminated.
+  // Similar functions for the upcoming token.
-  // Literal strings are collected for identifiers, strings, and
+
-  // numbers.
+  // One token look-ahead (past the token returned by Next()).
-  // These functions only give the correct result if the literal
+  Token::Value peek() const { return next_.token; }
-  // was scanned between calls to StartLiteral() and TerminateLiteral().
+
-  Vector<const char> literal_one_byte_string() {
+  Location peek_location() const { return next_.location; }
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->one_byte_literal();
-  }
-  Vector<const uc16> literal_utf16_string() {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->utf16_literal();
-  }
-  bool is_literal_one_byte() {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->is_one_byte();
-  }
-  bool is_literal_contextual_keyword(Vector<const char> keyword) {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->is_contextual_keyword(keyword);
-  }
-  int literal_length() const {
-    ASSERT_NOT_NULL(current_.literal_chars);
-    return current_.literal_chars->length();
-  }
 
   bool literal_contains_escapes() const {
     Location location = current_.location;
@@ -406,38 +390,15 @@ class Scanner {
     }
     return current_.literal_chars->length() != source_length;
   }
-
+  bool is_literal_contextual_keyword(Vector<const char> keyword) {
-  // Similar functions for the upcoming token.
+    ASSERT_NOT_NULL(current_.literal_chars);
-
+    return current_.literal_chars->is_contextual_keyword(keyword);
-  // One token look-ahead (past the token returned by Next()).
-  Token::Value peek() const { return next_.token; }
-
-  Location peek_location() const { return next_.location; }
-
-  // Returns the literal string for the next token (the token that
-  // would be returned if Next() were called).
-  Vector<const char> next_literal_one_byte_string() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->one_byte_literal();
-  }
-  Vector<const uc16> next_literal_utf16_string() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->utf16_literal();
-  }
-  bool is_next_literal_one_byte() {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->is_one_byte();
   }
   bool is_next_contextual_keyword(Vector<const char> keyword) {
     ASSERT_NOT_NULL(next_.literal_chars);
     return next_.literal_chars->is_contextual_keyword(keyword);
   }
-  int next_literal_length() const {
-    ASSERT_NOT_NULL(next_.literal_chars);
-    return next_.literal_chars->length();
-  }
 
-  Handle<String> AllocateLiteralString(Isolate* isolate, PretenureFlag tenured);
   Handle<String> AllocateNextLiteralString(Isolate* isolate,
                                            PretenureFlag tenured);
   Handle<String> AllocateInternalizedString(Isolate* isolate);
@@ -461,13 +422,13 @@ class Scanner {
     }
   }
 
+  int FindNumber(DuplicateFinder* finder, int value);
+  int FindSymbol(DuplicateFinder* finder, int value);
+
+  void LogSymbol(ParserRecorder* log, int position);
+
   UnicodeCache* unicode_cache() { return unicode_cache_; }
 
-  static const int kCharacterLookaheadBufferSize = 1;
-
-  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
-  uc32 ScanOctalEscape(uc32 c, int length);
-
   // Returns the location of the last seen octal literal.
   Location octal_position() const { return octal_pos_; }
   void clear_octal_position() { octal_pos_ = Location::invalid(); }
@@ -519,6 +480,11 @@ class Scanner {
     LiteralBuffer* literal_chars;
   };
 
+  static const int kCharacterLookaheadBufferSize = 1;
+
+  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
+  uc32 ScanOctalEscape(uc32 c, int length);
+
   // Call this after setting source_ to the input.
   void Init() {
     // Set c0_ (one character ahead)
@@ -579,6 +545,47 @@ class Scanner {
     }
   }
 
+  // Returns the literal string, if any, for the current token (the
+  // token last returned by Next()). The string is 0-terminated.
+  // Literal strings are collected for identifiers, strings, and
+  // numbers.
+  // These functions only give the correct result if the literal
+  // was scanned between calls to StartLiteral() and TerminateLiteral().
+  Vector<const char> literal_one_byte_string() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->one_byte_literal();
+  }
+  Vector<const uc16> literal_utf16_string() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->utf16_literal();
+  }
+  bool is_literal_one_byte() {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->is_one_byte();
+  }
+  int literal_length() const {
+    ASSERT_NOT_NULL(current_.literal_chars);
+    return current_.literal_chars->length();
+  }
+  // Returns the literal string for the next token (the token that
+  // would be returned if Next() were called).
+  Vector<const char> next_literal_one_byte_string() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->one_byte_literal();
+  }
+  Vector<const uc16> next_literal_utf16_string() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->utf16_literal();
+  }
+  bool is_next_literal_one_byte() {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->is_one_byte();
+  }
+  int next_literal_length() const {
+    ASSERT_NOT_NULL(next_.literal_chars);
+    return next_.literal_chars->length();
+  }
+
   uc32 ScanHexNumber(int expected_length);
 
   // Scans a single JavaScript token.

test/cctest/test-parsing.cc

@@ -795,6 +795,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
   i::Utf8ToUtf16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
+  i::HandleScope scope(CcTest::i_isolate());
   i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
   scanner.Initialize(&stream);
 
@@ -802,8 +803,12 @@ void TestScanRegExp(const char* re_source, const char* expected) {
   CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
   CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
   scanner.Next();  // Current token is now the regexp literal.
-  CHECK(scanner.is_literal_one_byte());
+  i::Handle<i::String> val =
-  i::Vector<const char> actual = scanner.literal_one_byte_string();
+      scanner.AllocateInternalizedString(CcTest::i_isolate());
+  i::DisallowHeapAllocation no_alloc;
+  i::String::FlatContent content = val->GetFlatContent();
+  CHECK(content.IsAscii());
+  i::Vector<const uint8_t> actual = content.ToOneByteVector();
   for (int i = 0; i < actual.length(); i++) {
     CHECK_NE('\0', expected[i]);
     CHECK_EQ(expected[i], actual[i]);