Fix bug when the scanner does a pushback at the end of input.

We don't advance the input cursor past the end of input, so we shouldn't
decrease it when we pushback the kEndOfInput marker.

Review URL: http://codereview.chromium.org/6246004

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@6308 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/preparser-api.cc

@@ -69,8 +69,12 @@ class InputStreamUTF16Buffer : public UC16CharacterStream {
     }
   }
 
-  virtual void PushBack(uc16 ch) {
+  virtual void PushBack(uc32 ch) {
     ASSERT(pos_ > 0);
+    if (ch == kEndOfInput) {
+      pos_--;
+      return;
+    }
     if (buffer_cursor_ <= pushback_buffer_) {
       // No more room in the current buffer to do pushbacks.
       if (pushback_buffer_end_cache_ == NULL) {
@@ -98,7 +102,8 @@ class InputStreamUTF16Buffer : public UC16CharacterStream {
         buffer_end_ = pushback_buffer_backing_ + pushback_buffer_backing_size_;
       }
     }
-    pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] = ch;
+    pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] =
+        static_cast<uc16>(ch);
     pos_--;
   }
 

src/scanner-base.h

@@ -64,10 +64,10 @@ class UC16CharacterStream {
   // Returns and advances past the next UC16 character in the input
   // stream. If there are no more characters, it returns a negative
   // value.
-  inline int32_t Advance() {
+  inline uc32 Advance() {
     if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
       pos_++;
-      return *(buffer_cursor_++);
+      return static_cast<uc32>(*(buffer_cursor_++));
     }
     // Note: currently the following increment is necessary to avoid a
     // parser problem! The scanner treats the final kEndOfInput as
@@ -97,13 +97,14 @@ class UC16CharacterStream {
     return SlowSeekForward(character_count);
   }
 
-  // Pushes back the most recently read UC16 character, i.e.,
-  // the value returned by the most recent call to Advance.
+  // Pushes back the most recently read UC16 character (or negative
+  // value if at end of input), i.e., the value returned by the most recent
+  // call to Advance.
   // Must not be used right after calling SeekForward.
-  virtual void PushBack(uc16 character) = 0;
+  virtual void PushBack(int32_t character) = 0;
 
  protected:
-  static const int32_t kEndOfInput = -1;
+  static const uc32 kEndOfInput = -1;
 
   // Ensures that the buffer_cursor_ points to the character at
   // position pos_ of the input, if possible. If the position

src/scanner.cc

@@ -48,14 +48,18 @@ BufferedUC16CharacterStream::BufferedUC16CharacterStream()
 
 BufferedUC16CharacterStream::~BufferedUC16CharacterStream() { }
 
-void BufferedUC16CharacterStream::PushBack(uc16 character) {
-  if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) {
-    // buffer_ is writable, buffer_cursor_ is const pointer.
-    buffer_[--buffer_cursor_ - buffer_] = character;
+void BufferedUC16CharacterStream::PushBack(uc32 character) {
+  if (character == kEndOfInput) {
     pos_--;
     return;
   }
-  SlowPushBack(character);
+  if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) {
+    // buffer_ is writable, buffer_cursor_ is const pointer.
+    buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character);
+    pos_--;
+    return;
+  }
+  SlowPushBack(static_cast<uc16>(character));
 }
 
 

src/scanner.h

@@ -43,7 +43,7 @@ class BufferedUC16CharacterStream: public UC16CharacterStream {
   BufferedUC16CharacterStream();
   virtual ~BufferedUC16CharacterStream();
 
-  virtual void PushBack(uc16 character);
+  virtual void PushBack(uc32 character);
 
  protected:
   static const unsigned kBufferSize = 512;
@@ -107,11 +107,12 @@ class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
                                            int end_position);
   virtual ~ExternalTwoByteStringUC16CharacterStream();
 
-  virtual void PushBack(uc16 character) {
+  virtual void PushBack(uc32 character) {
     ASSERT(buffer_cursor_ > raw_data_);
     buffer_cursor_--;
     pos_--;
   }
+
  protected:
   virtual unsigned SlowSeekForward(unsigned delta) {
     // Fast case always handles seeking.