[heap] Implement simple concurrent marking deque.

This patch adds a concurrent marking deque that exposes the same interface
for the main thread as the existing marking deque.

The matching interface makes the concurrent marking deque a drop-in
replacement for the sequential marking deque without any change in
mark-compactor and incremental marker.

BUG=chromium:694255

Review-Url: https://codereview.chromium.org/2810893002
Cr-Commit-Position: refs/heads/master@{#45042}

BUILD.gn

@@ -1567,6 +1567,7 @@ v8_source_set("v8_base") {
     "src/heap/array-buffer-tracker.h",
     "src/heap/code-stats.cc",
     "src/heap/code-stats.h",
+    "src/heap/concurrent-marking-deque.h",
     "src/heap/concurrent-marking.cc",
     "src/heap/concurrent-marking.h",
     "src/heap/embedder-tracing.cc",

src/heap/concurrent-marking-deque.h

@@ -0,0 +1,177 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CONCURRENT_MARKING_DEQUE_
+#define V8_HEAP_CONCURRENT_MARKING_DEQUE_
+
+#include <deque>
+
+#include "src/base/platform/mutex.h"
+
+namespace v8 {
+namespace internal {
+
+class Heap;
+class Isolate;
+class HeapObject;
+
+enum class MarkingThread { kMain, kConcurrent };
+
+enum class TargetDeque { kShared, kBailout };
+
+// The concurrent marking deque supports deque operations for two threads:
+// main and concurrent. It is implemented using two deques: shared and bailout.
+//
+// The concurrent thread can use the push and pop operations with the
+// MarkingThread::kConcurrent argument. All other operations are intended
+// to be used by the main thread only.
+//
+// The interface of the concurrent marking deque for the main thread matches
+// that of the sequential marking deque, so they can be easily switched
+// at compile time without updating the main thread call-sites.
+//
+// The shared deque is shared between the main thread and the concurrent
+// thread, so both threads can push to and pop from the shared deque.
+// The bailout deque stores objects that cannot be processed by the concurrent
+// thread. Only the concurrent thread can push to it and only the main thread
+// can pop from it.
+class ConcurrentMarkingDeque {
+ public:
+  // The heap parameter is needed to match the interface
+  // of the sequential marking deque.
+  explicit ConcurrentMarkingDeque(Heap* heap) {}
+
+  // Pushes the object into the specified deque assuming that the function is
+  // called on the specified thread. The main thread can push only to the shared
+  // deque. The concurrent thread can push to both deques.
+  bool Push(HeapObject* object, MarkingThread thread = MarkingThread::kMain,
+            TargetDeque target = TargetDeque::kShared) {
+    DCHECK_IMPLIES(thread == MarkingThread::kMain,
+                   target == TargetDeque::kShared);
+    switch (target) {
+      case TargetDeque::kShared:
+        shared_deque_.Push(object);
+        break;
+      case TargetDeque::kBailout:
+        bailout_deque_.Push(object);
+        break;
+    }
+    return true;
+  }
+
+  // Pops an object from the bailout or shared deque assuming that the function
+  // is called on the specified thread. The main thread first tries to pop the
+  // bailout deque. If the deque is empty then it tries the shared deque.
+  // If the shared deque is also empty, then the function returns nullptr.
+  // The concurrent thread pops only from the shared deque.
+  HeapObject* Pop(MarkingThread thread = MarkingThread::kMain) {
+    if (thread == MarkingThread::kMain) {
+      HeapObject* result = bailout_deque_.Pop();
+      if (result != nullptr) return result;
+    }
+    return shared_deque_.Pop();
+  }
+
+  // All the following operations can used only by the main thread.
+  void Clear() {
+    bailout_deque_.Clear();
+    shared_deque_.Clear();
+  }
+
+  bool IsFull() { return false; }
+
+  bool IsEmpty() { return bailout_deque_.IsEmpty() && shared_deque_.IsEmpty(); }
+
+  int Size() { return bailout_deque_.Size() + shared_deque_.Size(); }
+
+  // This is used for a large array with a progress bar.
+  // For simpicity, unshift to the bailout deque so that the concurrent thread
+  // does not see such objects.
+  bool Unshift(HeapObject* object) {
+    bailout_deque_.Unshift(object);
+    return true;
+  }
+
+  // Calls the specified callback on each element of the deques and replaces
+  // the element with the result of the callback. If the callback returns
+  // nullptr then the element is removed from the deque.
+  // The callback must accept HeapObject* and return HeapObject*.
+  template <typename Callback>
+  void Update(Callback callback) {
+    bailout_deque_.Update(callback);
+    shared_deque_.Update(callback);
+  }
+
+  // These empty functions are needed to match the interface
+  // of the sequential marking deque.
+  void SetUp() {}
+  void TearDown() {}
+  void StartUsing() {}
+  void StopUsing() {}
+  void ClearOverflowed() {}
+  void SetOverflowed() {}
+  bool overflowed() const { return false; }
+
+ private:
+  // Simple, slow, and thread-safe deque that forwards all operations to
+  // a lock-protected std::deque.
+  class Deque {
+   public:
+    Deque() { cache_padding_[0] = 0; }
+    void Clear() {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      return deque_.clear();
+    }
+    bool IsEmpty() {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      return deque_.empty();
+    }
+    int Size() {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      return static_cast<int>(deque_.size());
+    }
+    void Push(HeapObject* object) {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      deque_.push_back(object);
+    }
+    HeapObject* Pop() {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      if (deque_.empty()) return nullptr;
+      HeapObject* result = deque_.back();
+      deque_.pop_back();
+      return result;
+    }
+    void Unshift(HeapObject* object) {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      deque_.push_front(object);
+    }
+    template <typename Callback>
+    void Update(Callback callback) {
+      base::LockGuard<base::Mutex> guard(&mutex_);
+      std::deque<HeapObject*> new_deque;
+      for (auto object : deque_) {
+        HeapObject* new_object = callback(object);
+        if (new_object) {
+          new_deque.push_back(new_object);
+        }
+      }
+      deque_.swap(new_deque);
+    }
+
+   private:
+    base::Mutex mutex_;
+    std::deque<HeapObject*> deque_;
+    // Ensure that two deques do not share the same cache line.
+    static int const kCachePadding = 64;
+    char cache_padding_[kCachePadding];
+  };
+  Deque bailout_deque_;
+  Deque shared_deque_;
+  DISALLOW_COPY_AND_ASSIGN(ConcurrentMarkingDeque);
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_CONCURRENT_MARKING_DEQUE_

src/heap/concurrent-marking.cc

@@ -7,6 +7,7 @@
 #include <stack>
 #include <unordered_map>
 
+#include "src/heap/concurrent-marking-deque.h"
 #include "src/heap/heap-inl.h"
 #include "src/heap/heap.h"
 #include "src/heap/marking.h"
@@ -21,43 +22,13 @@
 namespace v8 {
 namespace internal {
 
-class ConcurrentMarkingMarkbits {
- public:
-  ConcurrentMarkingMarkbits() {}
-  ~ConcurrentMarkingMarkbits() {
-    for (auto chunk_bitmap : bitmap_) {
-      FreeBitmap(chunk_bitmap.second);
-    }
-  }
-  bool Mark(HeapObject* obj) {
-    Address address = obj->address();
-    MemoryChunk* chunk = MemoryChunk::FromAddress(address);
-    if (bitmap_.count(chunk) == 0) {
-      bitmap_[chunk] = AllocateBitmap();
-    }
-    MarkBit mark_bit =
-        bitmap_[chunk]->MarkBitFromIndex(chunk->AddressToMarkbitIndex(address));
-    if (mark_bit.Get()) return false;
-    mark_bit.Set();
-    return true;
-  }
-
-  Bitmap* AllocateBitmap() {
-    return static_cast<Bitmap*>(calloc(1, Bitmap::kSize));
-  }
-
-  void FreeBitmap(Bitmap* bitmap) { free(bitmap); }
-
- private:
-  std::unordered_map<MemoryChunk*, Bitmap*> bitmap_;
-};
-
 class ConcurrentMarkingVisitor final
     : public HeapVisitor<int, ConcurrentMarkingVisitor> {
  public:
   using BaseClass = HeapVisitor<int, ConcurrentMarkingVisitor>;
 
-  ConcurrentMarkingVisitor() : bytes_marked_(0) {}
+  explicit ConcurrentMarkingVisitor(ConcurrentMarkingDeque* deque)
+      : deque_(deque) {}
 
   void VisitPointers(HeapObject* host, Object** start, Object** end) override {
     for (Object** p = start; p < end; p++) {
@@ -154,84 +125,71 @@ class ConcurrentMarkingVisitor final
   }
 
   void MarkObject(HeapObject* obj) {
-    if (markbits_.Mark(obj)) {
+    deque_->Push(obj, MarkingThread::kConcurrent, TargetDeque::kShared);
-      marking_stack_.push(obj);
   }
-  }
-
-  void MarkTransitively() {
-    while (!marking_stack_.empty()) {
-      HeapObject* obj = marking_stack_.top();
-      marking_stack_.pop();
-      bytes_marked_ += IterateBody(obj);
-    }
-  }
-
-  size_t bytes_marked() { return bytes_marked_; }
 
  private:
-  size_t bytes_marked_;
+  ConcurrentMarkingDeque* deque_;
-  std::stack<HeapObject*> marking_stack_;
-  ConcurrentMarkingMarkbits markbits_;
 };
 
 class ConcurrentMarking::Task : public CancelableTask {
  public:
-  Task(Heap* heap, std::vector<HeapObject*>* root_set,
+  Task(Isolate* isolate, ConcurrentMarking* concurrent_marking,
        base::Semaphore* on_finish)
-      : CancelableTask(heap->isolate()),
+      : CancelableTask(isolate),
-        heap_(heap),
+        concurrent_marking_(concurrent_marking),
-        on_finish_(on_finish),
+        on_finish_(on_finish) {}
-        root_set_(root_set) {}
 
   virtual ~Task() {}
 
  private:
   // v8::internal::CancelableTask overrides.
   void RunInternal() override {
-    double time_ms = heap_->MonotonicallyIncreasingTimeInMs();
+    concurrent_marking_->Run();
-    {
-      TimedScope scope(&time_ms);
-      for (HeapObject* obj : *root_set_) {
-        marking_visitor_.MarkObject(obj);
-      }
-      marking_visitor_.MarkTransitively();
-    }
-    if (FLAG_trace_concurrent_marking) {
-      heap_->isolate()->PrintWithTimestamp(
-          "concurrently marked %dKB in %.2fms\n",
-          static_cast<int>(marking_visitor_.bytes_marked() / KB), time_ms);
-    }
     on_finish_->Signal();
   }
 
-  Heap* heap_;
+  ConcurrentMarking* concurrent_marking_;
   base::Semaphore* on_finish_;
-  ConcurrentMarkingVisitor marking_visitor_;
-  std::vector<HeapObject*>* root_set_;
   DISALLOW_COPY_AND_ASSIGN(Task);
 };
 
-ConcurrentMarking::ConcurrentMarking(Heap* heap)
+ConcurrentMarking::ConcurrentMarking(Heap* heap, ConcurrentMarkingDeque* deque)
-    : heap_(heap), pending_task_semaphore_(0), is_task_pending_(false) {
+    : heap_(heap),
+      pending_task_semaphore_(0),
+      deque_(deque),
+      visitor_(new ConcurrentMarkingVisitor(deque_)),
+      is_task_pending_(false) {
   // Concurrent marking does not work with double unboxing.
   STATIC_ASSERT(!(V8_CONCURRENT_MARKING && V8_DOUBLE_FIELDS_UNBOXING));
   // The runtime flag should be set only if the compile time flag was set.
   CHECK(!FLAG_concurrent_marking || V8_CONCURRENT_MARKING);
 }
 
-ConcurrentMarking::~ConcurrentMarking() {}
+ConcurrentMarking::~ConcurrentMarking() { delete visitor_; }
 
-void ConcurrentMarking::AddRoot(HeapObject* object) {
+void ConcurrentMarking::Run() {
-  root_set_.push_back(object);
+  double time_ms = heap_->MonotonicallyIncreasingTimeInMs();
+  size_t bytes_marked = 0;
+  {
+    TimedScope scope(&time_ms);
+    HeapObject* object;
+    while ((object = deque_->Pop(MarkingThread::kConcurrent)) != nullptr) {
+      bytes_marked += visitor_->IterateBody(object);
+    }
+  }
+  if (FLAG_trace_concurrent_marking) {
+    heap_->isolate()->PrintWithTimestamp("concurrently marked %dKB in %.2fms\n",
+                                         static_cast<int>(bytes_marked / KB),
+                                         time_ms);
+  }
 }
 
 void ConcurrentMarking::StartTask() {
   if (!FLAG_concurrent_marking) return;
   is_task_pending_ = true;
-
   V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new Task(heap_, &root_set_, &pending_task_semaphore_),
+      new Task(heap_->isolate(), this, &pending_task_semaphore_),
       v8::Platform::kShortRunningTask);
 }
 
@@ -239,7 +197,6 @@ void ConcurrentMarking::WaitForTaskToComplete() {
   if (!FLAG_concurrent_marking) return;
   pending_task_semaphore_.Wait();
   is_task_pending_ = false;
-  root_set_.clear();
 }
 
 void ConcurrentMarking::EnsureTaskCompleted() {

src/heap/concurrent-marking.h

@@ -5,8 +5,6 @@
 #ifndef V8_HEAP_CONCURRENT_MARKING_
 #define V8_HEAP_CONCURRENT_MARKING_
 
-#include <vector>
-
 #include "src/allocation.h"
 #include "src/cancelable-task.h"
 #include "src/utils.h"
@@ -15,16 +13,16 @@
 namespace v8 {
 namespace internal {
 
+class ConcurrentMarkingDeque;
+class ConcurrentMarkingVisitor;
 class Heap;
 class Isolate;
 
 class ConcurrentMarking {
  public:
-  explicit ConcurrentMarking(Heap* heap);
+  ConcurrentMarking(Heap* heap, ConcurrentMarkingDeque* deque_);
   ~ConcurrentMarking();
 
-  void AddRoot(HeapObject* object);
-
   void StartTask();
   void WaitForTaskToComplete();
   bool IsTaskPending() { return is_task_pending_; }
@@ -32,10 +30,12 @@ class ConcurrentMarking {
 
  private:
   class Task;
+  void Run();
   Heap* heap_;
   base::Semaphore pending_task_semaphore_;
+  ConcurrentMarkingDeque* deque_;
+  ConcurrentMarkingVisitor* visitor_;
   bool is_task_pending_;
-  std::vector<HeapObject*> root_set_;
 };
 
 }  // namespace internal

src/heap/heap.cc

@@ -5494,7 +5494,6 @@ bool Heap::SetUp() {
   store_buffer_ = new StoreBuffer(this);
 
   incremental_marking_ = new IncrementalMarking(this);
-  concurrent_marking_ = new ConcurrentMarking(this);
 
   for (int i = 0; i <= LAST_SPACE; i++) {
     space_[i] = nullptr;
@@ -5543,6 +5542,8 @@ bool Heap::SetUp() {
   mark_compact_collector_ = new MarkCompactCollector(this);
   incremental_marking_->set_marking_deque(
       mark_compact_collector_->marking_deque());
+  concurrent_marking_ =
+      new ConcurrentMarking(this, mark_compact_collector_->marking_deque());
   if (FLAG_minor_mc)
     minor_mark_compact_collector_ = new MinorMarkCompactCollector(this);
   gc_idle_time_handler_ = new GCIdleTimeHandler();

src/heap/incremental-marking.cc

@@ -551,9 +551,6 @@ void IncrementalMarking::StartMarking() {
 
   if (FLAG_concurrent_marking) {
     ConcurrentMarking* concurrent_marking = heap_->concurrent_marking();
-    marking_deque()->Iterate([concurrent_marking](HeapObject* obj) {
-      concurrent_marking->AddRoot(obj);
-    });
     concurrent_marking->StartTask();
   }
 

src/heap/mark-compact.h

@@ -10,6 +10,7 @@
 #include "src/base/bits.h"
 #include "src/base/platform/condition-variable.h"
 #include "src/cancelable-task.h"
+#include "src/heap/concurrent-marking-deque.h"
 #include "src/heap/marking.h"
 #include "src/heap/sequential-marking-deque.h"
 #include "src/heap/spaces.h"
@@ -31,7 +32,11 @@ class PageParallelJob;
 class RecordMigratedSlotVisitor;
 class ThreadLocalTop;
 
+#ifdef V8_CONCURRENT_MARKING
+using MarkingDeque = ConcurrentMarkingDeque;
+#else
 using MarkingDeque = SequentialMarkingDeque;
+#endif
 
 class ObjectMarking : public AllStatic {
  public:

src/heap/sequential-marking-deque.h

@@ -91,15 +91,6 @@ class SequentialMarkingDeque {
     }
   }
 
-  template <typename Callback>
-  void Iterate(Callback callback) {
-    int i = bottom_;
-    while (i != top_) {
-      callback(array_[i]);
-      i = (i + 1) & mask_;
-    }
-  }
-
   // Calls the specified callback on each element of the deque and replaces
   // the element with the result of the callback. If the callback returns
   // nullptr then the element is removed from the deque.

src/v8.gyp

@@ -951,6 +951,7 @@
         'heap/array-buffer-tracker.h',
         'heap/code-stats.cc',
         'heap/code-stats.h',
+        'heap/concurrent-marking-deque.h',
         'heap/concurrent-marking.cc',
         'heap/concurrent-marking.h',
         'heap/embedder-tracing.cc',

test/cctest/heap/test-concurrent-marking.cc

@@ -18,8 +18,9 @@ TEST(ConcurrentMarking) {
   if (!i::FLAG_concurrent_marking) return;
   CcTest::InitializeVM();
   Heap* heap = CcTest::heap();
-  ConcurrentMarking* concurrent_marking = new ConcurrentMarking(heap);
+  ConcurrentMarkingDeque deque(heap);
-  concurrent_marking->AddRoot(heap->undefined_value());
+  deque.Push(heap->undefined_value());
+  ConcurrentMarking* concurrent_marking = new ConcurrentMarking(heap, &deque);
   concurrent_marking->StartTask();
   concurrent_marking->WaitForTaskToComplete();
   delete concurrent_marking;

test/cctest/heap/test-mark-compact.cc

@@ -43,6 +43,7 @@
 #include "src/global-handles.h"
 #include "src/heap/mark-compact-inl.h"
 #include "src/heap/mark-compact.h"
+#include "src/heap/sequential-marking-deque.h"
 #include "src/objects-inl.h"
 #include "test/cctest/cctest.h"
 #include "test/cctest/heap/heap-tester.h"
@@ -51,10 +52,9 @@
 using namespace v8::internal;
 using v8::Just;
 
-
+TEST(SequentialMarkingDeque) {
-TEST(MarkingDeque) {
   CcTest::InitializeVM();
-  MarkingDeque s(CcTest::i_isolate()->heap());
+  SequentialMarkingDeque s(CcTest::i_isolate()->heap());
   s.SetUp();
   s.StartUsing();
   Address original_address = reinterpret_cast<Address>(&s);

test/unittests/BUILD.gn

@@ -100,6 +100,7 @@ v8_executable("unittests") {
     "eh-frame-iterator-unittest.cc",
     "eh-frame-writer-unittest.cc",
     "heap/bitmap-unittest.cc",
+    "heap/concurrent-marking-deque-unittest.cc",
     "heap/embedder-tracing-unittest.cc",
     "heap/gc-idle-time-handler-unittest.cc",
     "heap/gc-tracer-unittest.cc",

test/unittests/heap/concurrent-marking-deque-unittest.cc

@@ -0,0 +1,57 @@
+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdlib.h>
+
+#include "src/globals.h"
+#include "src/heap/concurrent-marking-deque.h"
+#include "src/heap/heap-inl.h"
+#include "src/isolate.h"
+#include "test/unittests/test-utils.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace v8 {
+namespace internal {
+
+class ConcurrentMarkingDequeTest : public TestWithIsolate {
+ public:
+  ConcurrentMarkingDequeTest() {
+    marking_deque_ = new ConcurrentMarkingDeque(i_isolate()->heap());
+    object_ = i_isolate()->heap()->undefined_value();
+  }
+
+  ~ConcurrentMarkingDequeTest() { delete marking_deque_; }
+
+  ConcurrentMarkingDeque* marking_deque() { return marking_deque_; }
+
+  HeapObject* object() { return object_; }
+
+ private:
+  ConcurrentMarkingDeque* marking_deque_;
+  HeapObject* object_;
+  DISALLOW_COPY_AND_ASSIGN(ConcurrentMarkingDequeTest);
+};
+
+TEST_F(ConcurrentMarkingDequeTest, Empty) {
+  EXPECT_TRUE(marking_deque()->IsEmpty());
+  EXPECT_EQ(0, marking_deque()->Size());
+}
+
+TEST_F(ConcurrentMarkingDequeTest, SharedDeque) {
+  marking_deque()->Push(object());
+  EXPECT_FALSE(marking_deque()->IsEmpty());
+  EXPECT_EQ(1, marking_deque()->Size());
+  EXPECT_EQ(object(), marking_deque()->Pop(MarkingThread::kConcurrent));
+}
+
+TEST_F(ConcurrentMarkingDequeTest, BailoutDeque) {
+  marking_deque()->Push(object(), MarkingThread::kConcurrent,
+                        TargetDeque::kBailout);
+  EXPECT_FALSE(marking_deque()->IsEmpty());
+  EXPECT_EQ(1, marking_deque()->Size());
+  EXPECT_EQ(nullptr, marking_deque()->Pop(MarkingThread::kConcurrent));
+}
+
+}  // namespace internal
+}  // namespace v8

test/unittests/unittests.gyp

@@ -97,6 +97,7 @@
       'eh-frame-iterator-unittest.cc',
       'eh-frame-writer-unittest.cc',
       'heap/bitmap-unittest.cc',
+      'heap/concurrent-marking-deque-unittest.cc',
       'heap/embedder-tracing-unittest.cc',
       'heap/gc-idle-time-handler-unittest.cc',
       'heap/gc-tracer-unittest.cc',