Changed interface to preparser to not require pushback support.

Changed implementation of pushback in preparer character stream.
Removed assert that isn't satisfied by in test-cases, but only by the real code.
Make preparser compile again.

Review URL: http://codereview.chromium.org/5593004

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5946 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

include/v8-preparser.h

@@ -99,13 +99,6 @@ class UnicodeInputStream {
   // Returns the next Unicode code-point in the input, or a negative value when
   // there is no more input in the stream.
   virtual int32_t Next() = 0;
-
-  // Pushes a read character back into the stream, so that it will be the next
-  // to be read by Advance(). The character pushed back must be the most
-  // recently read character that hasn't already been pushed back (i.e., if
-  // pushing back more than one character, they must occur in the opposite order
-  // of the one they were read in).
-  virtual void PushBack(int32_t ch) = 0;
 };
 
 

preparser/preparser-process.cc

@@ -127,7 +127,7 @@ uint32_t ReadUInt32(FILE* source, bool* ok) {
 
 
 bool ReadBuffer(FILE* source, void* buffer, size_t length) {
-  size_t actually_read = fread(buffer, 1, length, stdin);
+  size_t actually_read = fread(buffer, 1, length, source);
   return (actually_read == length);
 }
 
@@ -150,22 +150,25 @@ class ScopedPointer {
 };
 
 
-// Preparse stdin and output result on stdout.
-int PreParseIO() {
+// Preparse input and output result on stdout.
+int PreParseIO(FILE* input) {
   fprintf(stderr, "LOG: Enter parsing loop\n");
   bool ok = true;
-  uint32_t length = ReadUInt32(stdin, &ok);
+  uint32_t length = ReadUInt32(input, &ok);
+  fprintf(stderr, "LOG: Input length: %d\n", length);
   if (!ok) return kErrorReading;
   ScopedPointer<uint8_t> buffer(new uint8_t[length]);
 
-  if (!ReadBuffer(stdin, *buffer, length)) {
+  if (!ReadBuffer(input, *buffer, length)) {
     return kErrorReading;
   }
   UTF8InputStream input_buffer(*buffer, static_cast<size_t>(length));
 
   v8::PreParserData data =
-      v8::Preparse(&input_buffer, 64 * sizeof(void*));  // NOLINT
+      v8::Preparse(&input_buffer, 64 * 1024 * sizeof(void*));  // NOLINT
   if (data.stack_overflow()) {
+    fprintf(stderr, "LOG: Stack overflow\n");
+    fflush(stderr);
     // Report stack overflow error/no-preparser-data.
     WriteUInt32(stdout, 0, &ok);
     if (!ok) return kErrorWriting;
@@ -173,6 +176,8 @@ int PreParseIO() {
   }
 
   uint32_t size = data.size();
+  fprintf(stderr, "LOG: Success, data size: %u\n", size);
+  fflush(stderr);
   WriteUInt32(stdout, size, &ok);
   if (!ok) return kErrorWriting;
   if (!WriteBuffer(stdout, data.data(), size)) {
@@ -185,10 +190,17 @@ int PreParseIO() {
 
 
 int main(int argc, char* argv[]) {
+  FILE* input = stdin;
+  if (argc > 1) {
+    char* arg = argv[1];
+    FILE* file = fopen(arg, "rb");
+    if (!file) return EXIT_FAILURE;
+  }
   int status = 0;
   do {
-    status = v8::internal::PreParseIO();
+    status = v8::internal::PreParseIO(input);
   } while (status == 0);
   fprintf(stderr, "EXIT: Failure %d\n", status);
+  fflush(stderr);
   return EXIT_FAILURE;
 }

src/preparser-api.cc

@@ -41,43 +41,76 @@ namespace internal {
 // UTF16Buffer based on a v8::UnicodeInputStream.
 class InputStreamUTF16Buffer : public UC16CharacterStream {
  public:
+  /* The InputStreamUTF16Buffer maintains an internal buffer
+   * that is filled in chunks from the UC16CharacterStream.
+   * It also maintains unlimited pushback capability, but optimized
+   * for small pushbacks.
+   * The pushback_buffer_ pointer points to the limit of pushbacks
+   * in the current buffer. There is room for a few pushback'ed chars before
+   * the buffer containing the most recently read chunk. If this is overflowed,
+   * an external buffer is allocated/reused to hold further pushbacks, and
+   * pushback_buffer_ and buffer_cursor_/buffer_end_ now points to the
+   * new buffer. When this buffer is read to the end again, the cursor is
+   * switched back to the internal buffer
+   */
   explicit InputStreamUTF16Buffer(v8::UnicodeInputStream* stream)
       : UC16CharacterStream(),
         stream_(stream),
-        pushback_active_(false) {
+        pushback_buffer_(buffer_),
+        pushback_buffer_end_cache_(NULL),
+        pushback_buffer_backing_(NULL),
+        pushback_buffer_backing_size_(0) {
     buffer_cursor_ = buffer_end_ = buffer_ + kPushBackSize;
   }
 
-  virtual ~InputStreamUTF16Buffer() { }
+  virtual ~InputStreamUTF16Buffer() {
+    if (pushback_buffer_backing_ != NULL) {
+      DeleteArray(pushback_buffer_backing_);
+    }
+  }
 
   virtual void PushBack(uc16 ch) {
     ASSERT(pos_ > 0);
-    if (buffer_cursor_ > buffer_) {
-      // While we can stay within the buffer, just do so.
-      *--buffer_cursor_ = ch;
-      pos_--;
-      return;
-    }
-    if (!pushback_active_) {
-      // Push back the entire buffer to the stream and let the
-      // stream handle pushbacks from now.
-      // We leave buffer_cursor_ == buffer_end_, so the next read
-      // will fill the buffer from the current position.
-      // This should happen exceedingly rarely.
-      while (buffer_end_ > buffer_) {
-        stream_->PushBack(*--buffer_end_);
+    if (buffer_cursor_ <= pushback_buffer_) {
+      // No more room in the current buffer to do pushbacks.
+      if (pushback_buffer_end_cache_ == NULL) {
+        // We have overflowed the pushback space at the beginning of buffer_.
+        // Switch to using a separate allocated pushback buffer.
+        if (pushback_buffer_backing_ == NULL) {
+          // Allocate a buffer the first time we need it.
+          pushback_buffer_backing_ = NewArray<uc16>(kPushBackSize);
+          pushback_buffer_backing_size_ = kPushBackSize;
+        }
+        pushback_buffer_ = pushback_buffer_backing_;
+        pushback_buffer_end_cache_ = buffer_end_;
+        buffer_end_ = pushback_buffer_backing_ + pushback_buffer_backing_size_;
+        buffer_cursor_ = buffer_end_ - 1;
+      } else {
+        // Hit the bottom of the allocated pushback buffer.
+        // Double the buffer and continue.
+        uc16* new_buffer = NewArray<uc16>(pushback_buffer_backing_size_ * 2);
+        memcpy(new_buffer + pushback_buffer_backing_size_,
+               pushback_buffer_backing_,
+               pushback_buffer_backing_size_);
+        DeleteArray(pushback_buffer_backing_);
+        buffer_cursor_ = new_buffer + pushback_buffer_backing_size_;
+        pushback_buffer_backing_ = pushback_buffer_ = new_buffer;
+        buffer_end_ = pushback_buffer_backing_ + pushback_buffer_backing_size_;
       }
-      buffer_cursor_ = buffer_end_;
-      pushback_active_ = true;
     }
-    stream_->PushBack(ch);
+    pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] = ch;
     pos_--;
   }
 
  protected:
   virtual bool ReadBlock() {
+    if (pushback_buffer_end_cache_ != NULL) {
+      buffer_cursor_ = buffer_;
+      buffer_end_ = pushback_buffer_end_cache_;
+      pushback_buffer_end_cache_ = NULL;
+      return buffer_end_ > buffer_cursor_;
+    }
     // Copy the top of the buffer into the pushback area.
-    pushback_active_ = false;
     int32_t value;
     uc16* buffer_start = buffer_ + kPushBackSize;
     buffer_cursor_ = buffer_end_ = buffer_start;
@@ -107,8 +140,14 @@ class InputStreamUTF16Buffer : public UC16CharacterStream {
   // then kBufferSize chars of read-ahead.
   // The pushback buffer is only used if pushing back characters past
   // the start of a block.
-  uc16 buffer_[kBufferSize + kPushBackSize];
-  bool pushback_active_;
+  uc16 buffer_[kPushBackSize + kBufferSize];
+  // Limit of pushbacks before new allocation is necessary.
+  uc16* pushback_buffer_;
+  // Only if that pushback buffer at the start of buffer_ isn't sufficient
+  // is the following used.
+  const uc16* pushback_buffer_end_cache_;
+  uc16* pushback_buffer_backing_;
+  unsigned pushback_buffer_backing_size_;
 };
 
 
@@ -135,7 +174,6 @@ void FatalProcessOutOfMemory(const char* reason) {
 
 bool EnableSlowAsserts() { return true; }
 
-
 }  // namespace internal.
 
 

src/scanner-base.cc

@@ -512,7 +512,6 @@ void JavaScriptScanner::SeekForward(int pos) {
     has_line_terminator_before_next_ = false;
   }
   Scan();
-  ASSERT_EQ(Token::RBRACE, next_.token);
 }