Make the preparser standalone library and process build in debug mode.

It should now be possible to build the preparser using 'scons preparser' in both release and debug modes.
Remove v8.h include from scanner-base.h and other files.
Remove NativeAllocationChecker and all of its kind.
Moved Isolate::PreallocatedStorage* to isolate.cc

Review URL: http://codereview.chromium.org/6749029

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@7413 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/allocation.cc

@@ -25,58 +25,16 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "v8.h"
-#include "isolate.h"
-#include "allocation.h"
-
-/* TODO(isolates): this is what's included in bleeding_edge
-   including of v8.h was replaced with these in
-   http://codereview.chromium.org/5005001/
-   we need Isolate and Isolate needs a lot more so I'm including v8.h back.
 #include "../include/v8stdint.h"
 #include "globals.h"
 #include "checks.h"
 #include "allocation.h"
 #include "utils.h"
-*/
 
 namespace v8 {
 namespace internal {
 
-#ifdef DEBUG
-
-NativeAllocationChecker::NativeAllocationChecker(
-    NativeAllocationChecker::NativeAllocationAllowed allowed)
-    : allowed_(allowed) {
-  if (allowed == DISALLOW) {
-    Isolate* isolate = Isolate::Current();
-    isolate->set_allocation_disallowed(isolate->allocation_disallowed() + 1);
-  }
-}
-
-
-NativeAllocationChecker::~NativeAllocationChecker() {
-  Isolate* isolate = Isolate::Current();
-  if (allowed_ == DISALLOW) {
-    isolate->set_allocation_disallowed(isolate->allocation_disallowed() - 1);
-  }
-  ASSERT(isolate->allocation_disallowed() >= 0);
-}
-
-
-bool NativeAllocationChecker::allocation_allowed() {
-  // TODO(isolates): either find a way to make this work that doesn't
-  // require initializing an isolate before we can use malloc or drop
-  // it completely.
-  return true;
-  // return Isolate::Current()->allocation_disallowed() == 0;
-}
-
-#endif  // DEBUG
-
-
 void* Malloced::New(size_t size) {
-  ASSERT(NativeAllocationChecker::allocation_allowed());
   void* result = malloc(size);
   if (result == NULL) {
     v8::internal::FatalProcessOutOfMemory("Malloced operator new");
@@ -142,77 +100,6 @@ char* StrNDup(const char* str, int n) {
 }
 
 
-void Isolate::PreallocatedStorageInit(size_t size) {
-  ASSERT(free_list_.next_ == &free_list_);
-  ASSERT(free_list_.previous_ == &free_list_);
-  PreallocatedStorage* free_chunk =
-      reinterpret_cast<PreallocatedStorage*>(new char[size]);
-  free_list_.next_ = free_list_.previous_ = free_chunk;
-  free_chunk->next_ = free_chunk->previous_ = &free_list_;
-  free_chunk->size_ = size - sizeof(PreallocatedStorage);
-  preallocated_storage_preallocated_ = true;
-}
-
-
-void* Isolate::PreallocatedStorageNew(size_t size) {
-  if (!preallocated_storage_preallocated_) {
-    return FreeStoreAllocationPolicy::New(size);
-  }
-  ASSERT(free_list_.next_ != &free_list_);
-  ASSERT(free_list_.previous_ != &free_list_);
-
-  size = (size + kPointerSize - 1) & ~(kPointerSize - 1);
-  // Search for exact fit.
-  for (PreallocatedStorage* storage = free_list_.next_;
-       storage != &free_list_;
-       storage = storage->next_) {
-    if (storage->size_ == size) {
-      storage->Unlink();
-      storage->LinkTo(&in_use_list_);
-      return reinterpret_cast<void*>(storage + 1);
-    }
-  }
-  // Search for first fit.
-  for (PreallocatedStorage* storage = free_list_.next_;
-       storage != &free_list_;
-       storage = storage->next_) {
-    if (storage->size_ >= size + sizeof(PreallocatedStorage)) {
-      storage->Unlink();
-      storage->LinkTo(&in_use_list_);
-      PreallocatedStorage* left_over =
-          reinterpret_cast<PreallocatedStorage*>(
-              reinterpret_cast<char*>(storage + 1) + size);
-      left_over->size_ = storage->size_ - size - sizeof(PreallocatedStorage);
-      ASSERT(size + left_over->size_ + sizeof(PreallocatedStorage) ==
-             storage->size_);
-      storage->size_ = size;
-      left_over->LinkTo(&free_list_);
-      return reinterpret_cast<void*>(storage + 1);
-    }
-  }
-  // Allocation failure.
-  ASSERT(false);
-  return NULL;
-}
-
-
-// We don't attempt to coalesce.
-void Isolate::PreallocatedStorageDelete(void* p) {
-  if (p == NULL) {
-    return;
-  }
-  if (!preallocated_storage_preallocated_) {
-    FreeStoreAllocationPolicy::Delete(p);
-    return;
-  }
-  PreallocatedStorage* storage = reinterpret_cast<PreallocatedStorage*>(p) - 1;
-  ASSERT(storage->next_->previous_ == storage);
-  ASSERT(storage->previous_->next_ == storage);
-  storage->Unlink();
-  storage->LinkTo(&free_list_);
-}
-
-
 void PreallocatedStorage::LinkTo(PreallocatedStorage* other) {
   next_ = other->next_;
   other->next_->previous_ = this;

src/allocation.h

@@ -39,25 +39,6 @@ namespace internal {
 // processing.
 void FatalProcessOutOfMemory(const char* message);
 
-// A class that controls whether allocation is allowed.  This is for
-// the C++ heap only!
-class NativeAllocationChecker {
- public:
-  enum NativeAllocationAllowed { ALLOW, DISALLOW };
-#ifdef DEBUG
-  explicit NativeAllocationChecker(NativeAllocationAllowed allowed);
-  ~NativeAllocationChecker();
-  static bool allocation_allowed();
- private:
-  // This flag applies to this particular instance.
-  NativeAllocationAllowed allowed_;
-#else
-  explicit inline NativeAllocationChecker(NativeAllocationAllowed allowed) {}
-  static inline bool allocation_allowed() { return true; }
-#endif
-};
-
-
 // Superclass for classes managed with new & delete.
 class Malloced {
  public:
@@ -101,7 +82,6 @@ class AllStatic {
 
 template <typename T>
 static T* NewArray(int size) {
-  ASSERT(NativeAllocationChecker::allocation_allowed());
   T* result = new T[size];
   if (result == NULL) Malloced::FatalProcessOutOfMemory();
   return result;

src/isolate.cc

@@ -168,6 +168,77 @@ void Isolate::PreallocatedMemoryThreadStop() {
 }
 
 
+void Isolate::PreallocatedStorageInit(size_t size) {
+  ASSERT(free_list_.next_ == &free_list_);
+  ASSERT(free_list_.previous_ == &free_list_);
+  PreallocatedStorage* free_chunk =
+      reinterpret_cast<PreallocatedStorage*>(new char[size]);
+  free_list_.next_ = free_list_.previous_ = free_chunk;
+  free_chunk->next_ = free_chunk->previous_ = &free_list_;
+  free_chunk->size_ = size - sizeof(PreallocatedStorage);
+  preallocated_storage_preallocated_ = true;
+}
+
+
+void* Isolate::PreallocatedStorageNew(size_t size) {
+  if (!preallocated_storage_preallocated_) {
+    return FreeStoreAllocationPolicy::New(size);
+  }
+  ASSERT(free_list_.next_ != &free_list_);
+  ASSERT(free_list_.previous_ != &free_list_);
+
+  size = (size + kPointerSize - 1) & ~(kPointerSize - 1);
+  // Search for exact fit.
+  for (PreallocatedStorage* storage = free_list_.next_;
+       storage != &free_list_;
+       storage = storage->next_) {
+    if (storage->size_ == size) {
+      storage->Unlink();
+      storage->LinkTo(&in_use_list_);
+      return reinterpret_cast<void*>(storage + 1);
+    }
+  }
+  // Search for first fit.
+  for (PreallocatedStorage* storage = free_list_.next_;
+       storage != &free_list_;
+       storage = storage->next_) {
+    if (storage->size_ >= size + sizeof(PreallocatedStorage)) {
+      storage->Unlink();
+      storage->LinkTo(&in_use_list_);
+      PreallocatedStorage* left_over =
+          reinterpret_cast<PreallocatedStorage*>(
+              reinterpret_cast<char*>(storage + 1) + size);
+      left_over->size_ = storage->size_ - size - sizeof(PreallocatedStorage);
+      ASSERT(size + left_over->size_ + sizeof(PreallocatedStorage) ==
+             storage->size_);
+      storage->size_ = size;
+      left_over->LinkTo(&free_list_);
+      return reinterpret_cast<void*>(storage + 1);
+    }
+  }
+  // Allocation failure.
+  ASSERT(false);
+  return NULL;
+}
+
+
+// We don't attempt to coalesce.
+void Isolate::PreallocatedStorageDelete(void* p) {
+  if (p == NULL) {
+    return;
+  }
+  if (!preallocated_storage_preallocated_) {
+    FreeStoreAllocationPolicy::Delete(p);
+    return;
+  }
+  PreallocatedStorage* storage = reinterpret_cast<PreallocatedStorage*>(p) - 1;
+  ASSERT(storage->next_->previous_ == storage);
+  ASSERT(storage->previous_->next_ == storage);
+  storage->Unlink();
+  storage->LinkTo(&free_list_);
+}
+
+
 Isolate* Isolate::default_isolate_ = NULL;
 Thread::LocalStorageKey Isolate::isolate_key_;
 Thread::LocalStorageKey Isolate::thread_id_key_;

src/isolate.h

@@ -292,8 +292,6 @@ typedef List<HeapObject*, PreallocatedStorage> DebugObjectCache;
   /* Assembler state. */                                                       \
   /* A previously allocated buffer of kMinimalBufferSize bytes, or NULL. */    \
   V(byte*, assembler_spare_buffer, NULL)                                       \
-  /*This static counter ensures that NativeAllocationCheckers can be nested.*/ \
-  V(int, allocation_disallowed, 0)                                             \
   V(FatalErrorCallback, exception_behavior, NULL)                              \
   V(v8::Debug::MessageHandler, message_handler, NULL)                          \
   /* To distinguish the function templates, so that we can find them in the */ \

src/objects.cc

@@ -4514,7 +4514,6 @@ SmartPointer<char> String::ToCString(AllowNullsFlag allow_nulls,
                                      int offset,
                                      int length,
                                      int* length_return) {
-  ASSERT(NativeAllocationChecker::allocation_allowed());
   if (robust_flag == ROBUST_STRING_TRAVERSAL && !LooksValid()) {
     return SmartPointer<char>(NULL);
   }
@@ -4594,7 +4593,6 @@ const uc16* String::GetTwoByteData(unsigned start) {
 
 
 SmartPointer<uc16> String::ToWideCString(RobustnessFlag robust_flag) {
-  ASSERT(NativeAllocationChecker::allocation_allowed());
   if (robust_flag == ROBUST_STRING_TRAVERSAL && !LooksValid()) {
     return SmartPointer<uc16>();
   }

src/parser.cc

@@ -579,7 +579,7 @@ Parser::Parser(Handle<Script> script,
     : isolate_(script->GetIsolate()),
       symbol_cache_(pre_data ? pre_data->symbol_count() : 0),
       script_(script),
-      scanner_(isolate_),
+      scanner_(isolate_->scanner_constants()),
       top_scope_(NULL),
       with_nesting_level_(0),
       lexical_scope_(NULL),
@@ -5053,7 +5053,7 @@ static ScriptDataImpl* DoPreParse(UC16CharacterStream* source,
                                   bool allow_lazy,
                                   ParserRecorder* recorder) {
   Isolate* isolate = Isolate::Current();
-  V8JavaScriptScanner scanner(isolate);
+  V8JavaScriptScanner scanner(isolate->scanner_constants());
   scanner.Initialize(source);
   intptr_t stack_limit = isolate->stack_guard()->real_climit();
   if (!preparser::PreParser::PreParseProgram(&scanner,

src/parser.h

@@ -776,7 +776,9 @@ class JsonParser BASE_EMBEDDED {
   }
 
  private:
-  JsonParser() : isolate_(Isolate::Current()), scanner_(isolate_) { }
+  JsonParser()
+      : isolate_(Isolate::Current()),
+        scanner_(isolate_->scanner_constants()) { }
   ~JsonParser() { }
 
   Isolate* isolate() { return isolate_; }

src/preparse-data.cc

@@ -25,20 +25,13 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "v8.h"
-
-/* 
-TODO(isolates): I incldue v8.h instead of these because we need Isolate and 
-some classes (NativeAllocationChecker) are moved into isolate.h 
 #include "../include/v8stdint.h"
 #include "globals.h"
 #include "checks.h"
 #include "allocation.h"
-#include "allocation-inl.h"
 #include "utils.h"
 #include "list-inl.h"
-#include "hashmap.h" 
-*/
+#include "hashmap.h"
 
 #include "preparse-data.h"
 

src/preparser-api.cc

@@ -27,8 +27,6 @@
 
 #include "../include/v8-preparser.h"
 
-#include "v8.h"
-
 #include "globals.h"
 #include "checks.h"
 #include "allocation.h"
@@ -161,8 +159,8 @@ class InputStreamUTF16Buffer : public UC16CharacterStream {
 
 class StandAloneJavaScriptScanner : public JavaScriptScanner {
  public:
-  StandAloneJavaScriptScanner()
-    : JavaScriptScanner(Isolate::Current()) { }
+  explicit StandAloneJavaScriptScanner(ScannerConstants* scanner_constants)
+      : JavaScriptScanner(scanner_constants) { }
 
   void Initialize(UC16CharacterStream* source) {
     source_ = source;
@@ -176,7 +174,8 @@ class StandAloneJavaScriptScanner : public JavaScriptScanner {
 };
 
 
-// Functions declared by allocation.h
+// Functions declared by allocation.h and implemented in both api.cc (for v8)
+// or here (for a stand-alone preparser).
 
 void FatalProcessOutOfMemory(const char* reason) {
   V8_Fatal(__FILE__, __LINE__, reason);
@@ -193,7 +192,8 @@ UnicodeInputStream::~UnicodeInputStream() { }
 PreParserData Preparse(UnicodeInputStream* input, size_t max_stack) {
   internal::InputStreamUTF16Buffer buffer(input);
   uintptr_t stack_limit = reinterpret_cast<uintptr_t>(&buffer) - max_stack;
-  internal::StandAloneJavaScriptScanner scanner;
+  ScannerConstants scanner_constants;
+  internal::StandAloneJavaScriptScanner scanner(&scanner_constants);
   scanner.Initialize(&buffer);
   internal::CompleteParserRecorder recorder;
   preparser::PreParser::PreParseResult result =

src/preparser.cc

@@ -25,11 +25,6 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "v8.h"
-
-/*
-TODO(isolates): I incldue v8.h instead of these because we need Isolate and 
-some classes (NativeAllocationChecker) are moved into isolate.h 
 #include "../include/v8stdint.h"
 #include "unicode.h"
 #include "globals.h"
@@ -37,7 +32,6 @@ some classes (NativeAllocationChecker) are moved into isolate.h
 #include "allocation.h"
 #include "utils.h"
 #include "list.h"
-*/
 
 #include "scanner-base.h"
 #include "preparse-data.h"

src/scanner-base.cc

@@ -27,13 +27,7 @@
 
 // Features shared by parsing and pre-parsing scanners.
 
-#include "v8.h"
-
-/*
-TODO(isolates): I incldue v8.h instead of these because we need Isolate and 
-some classes (NativeAllocationChecker) are moved into isolate.h 
 #include "../include/v8stdint.h"
-*/
 #include "scanner-base.h"
 #include "char-predicates-inl.h"
 
@@ -60,8 +54,8 @@ bool ScannerConstants::IsIdentifier(unibrow::CharacterStream* buffer) {
 // ----------------------------------------------------------------------------
 // Scanner
 
-Scanner::Scanner(Isolate* isolate)
-    : scanner_constants_(isolate->scanner_constants()),
+Scanner::Scanner(ScannerConstants* scanner_constants)
+    : scanner_constants_(scanner_constants),
       octal_pos_(kNoOctalLocation) {
 }
 
@@ -120,7 +114,8 @@ uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
 // ----------------------------------------------------------------------------
 // JavaScriptScanner
 
-JavaScriptScanner::JavaScriptScanner(Isolate* isolate) : Scanner(isolate) {}
+JavaScriptScanner::JavaScriptScanner(ScannerConstants* scanner_contants)
+    : Scanner(scanner_contants) { }
 
 
 Token::Value JavaScriptScanner::Next() {

src/scanner-base.h

@@ -123,6 +123,7 @@ class ScannerConstants {
 // ---------------------------------------------------------------------
 // Constants used by scanners.
  public:
+  ScannerConstants() {}
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
 
   StaticResource<Utf8Decoder>* utf8_decoder() {
@@ -137,7 +138,6 @@ class ScannerConstants {
   bool IsIdentifier(unibrow::CharacterStream* buffer);
 
  private:
-  ScannerConstants() {}
 
   unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart;
   unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart;
@@ -145,7 +145,6 @@ class ScannerConstants {
   unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
   StaticResource<Utf8Decoder> utf8_decoder_;
 
-  friend class Isolate;
   DISALLOW_COPY_AND_ASSIGN(ScannerConstants);
 };
 
@@ -273,7 +272,7 @@ class Scanner {
     bool complete_;
   };
 
-  explicit Scanner(Isolate* isolate);
+  explicit Scanner(ScannerConstants* scanner_contants);
 
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
@@ -474,7 +473,7 @@ class JavaScriptScanner : public Scanner {
     bool complete_;
   };
 
-  explicit JavaScriptScanner(Isolate* isolate);
+  explicit JavaScriptScanner(ScannerConstants* scanner_contants);
 
   // Returns the next token.
   Token::Value Next();

src/scanner.cc

@@ -345,7 +345,8 @@ void V8JavaScriptScanner::Initialize(UC16CharacterStream* source) {
 // ----------------------------------------------------------------------------
 // JsonScanner
 
-JsonScanner::JsonScanner(Isolate* isolate) : Scanner(isolate) { }
+JsonScanner::JsonScanner(ScannerConstants* scanner_constants)
+    : Scanner(scanner_constants) { }
 
 
 void JsonScanner::Initialize(UC16CharacterStream* source) {

src/scanner.h

@@ -134,8 +134,8 @@ class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
 
 class V8JavaScriptScanner : public JavaScriptScanner {
  public:
-  explicit V8JavaScriptScanner(Isolate* isolate)
-      : JavaScriptScanner(isolate) {}
+  explicit V8JavaScriptScanner(ScannerConstants* scanner_constants)
+      : JavaScriptScanner(scanner_constants) {}
 
   void Initialize(UC16CharacterStream* source);
 };
@@ -143,7 +143,7 @@ class V8JavaScriptScanner : public JavaScriptScanner {
 
 class JsonScanner : public Scanner {
  public:
-  explicit JsonScanner(Isolate* isolate);
+  explicit JsonScanner(ScannerConstants* scanner_constants);
 
   void Initialize(UC16CharacterStream* source);
 

src/top.cc

@@ -306,11 +306,6 @@ void Isolate::PrintStack() {
       allocator = preallocated_message_space_;
     }
 
-    NativeAllocationChecker allocation_checker(
-      !FLAG_preallocate_message_memory ?
-      NativeAllocationChecker::ALLOW :
-      NativeAllocationChecker::DISALLOW);
-
     StringStream::ClearMentionedObjectCache();
     StringStream accumulator(allocator);
     incomplete_message_ = &accumulator;

test/cctest/test-parsing.cc

@@ -265,7 +265,7 @@ TEST(StandAlonePreParser) {
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
-    i::V8JavaScriptScanner scanner(ISOLATE);
+    i::V8JavaScriptScanner scanner(ISOLATE->scanner_constants());
     scanner.Initialize(&stream);
 
     v8::preparser::PreParser::PreParseResult result =
@@ -358,7 +358,7 @@ TEST(PreParseOverflow) {
       reinterpret_cast<const i::byte*>(*program),
       static_cast<unsigned>(kProgramSize));
   i::CompleteParserRecorder log;
-  i::V8JavaScriptScanner scanner(ISOLATE);
+  i::V8JavaScriptScanner scanner(ISOLATE->scanner_constants());
   scanner.Initialize(&stream);
 
 
@@ -576,7 +576,7 @@ void TestStreamScanner(i::UC16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
-  i::V8JavaScriptScanner scanner(ISOLATE);
+  i::V8JavaScriptScanner scanner(ISOLATE->scanner_constants());
   scanner.Initialize(stream);
 
   int i = 0;
@@ -655,7 +655,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
   i::Utf8ToUC16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
-  i::V8JavaScriptScanner scanner(ISOLATE);
+  i::V8JavaScriptScanner scanner(ISOLATE->scanner_constants());
   scanner.Initialize(&stream);
 
   i::Token::Value start = scanner.peek();