Behold, a unit test for Scanner::BookmarkScope (& scanner bookmarking).

This is in preparation for upcmoming scanner + bookmarking cleanups.

Also, drive-by fix for setting a bookmark close to the end of the stream,
when the look-ahead character (c0_) is kEndOfInput, which the bookmarking
logic also used as kNoBookmark.

R=marja@chomium.org
BUG=v8:4947

Review-Url: https://codereview.chromium.org/2345053003
Cr-Commit-Position: refs/heads/master@{#39507}

src/parsing/scanner.cc

@@ -1608,9 +1608,9 @@ void Scanner::ResetToBookmark() {
   bookmark_c0_ = kBookmarkWasApplied;
 }
 
-
-bool Scanner::BookmarkHasBeenSet() { return bookmark_c0_ >= 0; }
-
+bool Scanner::BookmarkHasBeenSet() {
+  return bookmark_c0_ != kNoBookmark && bookmark_c0_ != kBookmarkWasApplied;
+}
 
 bool Scanner::BookmarkHasBeenReset() {
   return bookmark_c0_ == kBookmarkWasApplied;

src/parsing/scanner.h

@@ -842,8 +842,8 @@ class Scanner {
   // To be able to restore this state, we will keep copies of current_, next_,
   // and c0_; we'll ask the stream to bookmark itself, and we'll copy the
   // contents of current_'s and next_'s literal buffers to bookmark_*_literal_.
-  static const uc32 kNoBookmark = -1;
-  static const uc32 kBookmarkWasApplied = -2;
+  static const uc32 kNoBookmark = -2;
+  static const uc32 kBookmarkWasApplied = -3;
   uc32 bookmark_c0_;
   TokenDesc bookmark_current_;
   TokenDesc bookmark_next_;

test/cctest/BUILD.gn

@@ -91,6 +91,7 @@ v8_executable("cctest") {
     "libplatform/test-tracing.cc",
     "libsampler/test-sampler.cc",
     "parsing/test-scanner-streams.cc",
+    "parsing/test-scanner.cc",
     "print-extension.cc",
     "print-extension.h",
     "profiler-extension.cc",

test/cctest/cctest.gyp

@@ -114,6 +114,7 @@
       'libplatform/test-tracing.cc',
       'libsampler/test-sampler.cc',
       'parsing/test-scanner-streams.cc',
+      'parsing/test-scanner.cc',
       'print-extension.cc',
       'print-extension.h',
       'profiler-extension.cc',

test/cctest/parsing/test-scanner.cc

@@ -0,0 +1,64 @@
+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Tests v8::internal::Scanner. Note that presently most unit tests for the
+// Scanner are in cctest/test-parsing.cc, rather than here.
+
+#include "src/handles-inl.h"
+#include "src/parsing/scanner-character-streams.h"
+#include "src/parsing/scanner.h"
+#include "src/unicode-cache.h"
+#include "test/cctest/cctest.h"
+
+using namespace v8::internal;
+
+namespace {
+
+const char src_simple[] = "function foo() { var x = 2 * a() + b; }";
+
+static UnicodeCache* unicode_cache = new UnicodeCache();
+
+std::unique_ptr<Scanner> make_scanner(const char* src) {
+  std::unique_ptr<Scanner> scanner(new Scanner(new UnicodeCache()));
+  scanner->Initialize(ScannerStream::ForTesting(src).release());
+  return scanner;
+}
+
+}  // anonymous namespace
+
+TEST(Bookmarks) {
+  // Scan through the given source and record the tokens for use as reference
+  // below.
+  std::vector<Token::Value> tokens;
+  {
+    auto scanner = make_scanner(src_simple);
+    do {
+      tokens.push_back(scanner->Next());
+    } while (scanner->current_token() != Token::EOS);
+  }
+
+  // For each position:
+  // - Scan through file,
+  // - set a bookmark once the position is reached,
+  // - scan a bit more,
+  // - reset to the bookmark, and
+  // - scan until the end.
+  // At each step, compare to the reference token sequence generated above.
+  for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) {
+    auto scanner = make_scanner(src_simple);
+    Scanner::BookmarkScope bookmark(scanner.get());
+
+    for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) {
+      if (i == bookmark_pos) {
+        bookmark.Set();
+      }
+      DCHECK_EQ(tokens[i], scanner->Next());
+    }
+
+    bookmark.Reset();
+    for (size_t i = bookmark_pos; i < tokens.size(); i++) {
+      DCHECK_EQ(tokens[i], scanner->Next());
+    }
+  }
+}