diff --git a/BUILD.gn b/BUILD.gn index cb37b7abec..e0ee3a0067 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1567,6 +1567,7 @@ v8_source_set("v8_base") { "src/heap/array-buffer-tracker.h", "src/heap/code-stats.cc", "src/heap/code-stats.h", + "src/heap/concurrent-marking-deque.h", "src/heap/concurrent-marking.cc", "src/heap/concurrent-marking.h", "src/heap/embedder-tracing.cc", diff --git a/src/heap/concurrent-marking-deque.h b/src/heap/concurrent-marking-deque.h new file mode 100644 index 0000000000..daa0e2ba4c --- /dev/null +++ b/src/heap/concurrent-marking-deque.h @@ -0,0 +1,177 @@ +// Copyright 2017 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_HEAP_CONCURRENT_MARKING_DEQUE_ +#define V8_HEAP_CONCURRENT_MARKING_DEQUE_ + +#include + +#include "src/base/platform/mutex.h" + +namespace v8 { +namespace internal { + +class Heap; +class Isolate; +class HeapObject; + +enum class MarkingThread { kMain, kConcurrent }; + +enum class TargetDeque { kShared, kBailout }; + +// The concurrent marking deque supports deque operations for two threads: +// main and concurrent. It is implemented using two deques: shared and bailout. +// +// The concurrent thread can use the push and pop operations with the +// MarkingThread::kConcurrent argument. All other operations are intended +// to be used by the main thread only. +// +// The interface of the concurrent marking deque for the main thread matches +// that of the sequential marking deque, so they can be easily switched +// at compile time without updating the main thread call-sites. +// +// The shared deque is shared between the main thread and the concurrent +// thread, so both threads can push to and pop from the shared deque. +// The bailout deque stores objects that cannot be processed by the concurrent +// thread. Only the concurrent thread can push to it and only the main thread +// can pop from it. +class ConcurrentMarkingDeque { + public: + // The heap parameter is needed to match the interface + // of the sequential marking deque. + explicit ConcurrentMarkingDeque(Heap* heap) {} + + // Pushes the object into the specified deque assuming that the function is + // called on the specified thread. The main thread can push only to the shared + // deque. The concurrent thread can push to both deques. + bool Push(HeapObject* object, MarkingThread thread = MarkingThread::kMain, + TargetDeque target = TargetDeque::kShared) { + DCHECK_IMPLIES(thread == MarkingThread::kMain, + target == TargetDeque::kShared); + switch (target) { + case TargetDeque::kShared: + shared_deque_.Push(object); + break; + case TargetDeque::kBailout: + bailout_deque_.Push(object); + break; + } + return true; + } + + // Pops an object from the bailout or shared deque assuming that the function + // is called on the specified thread. The main thread first tries to pop the + // bailout deque. If the deque is empty then it tries the shared deque. + // If the shared deque is also empty, then the function returns nullptr. + // The concurrent thread pops only from the shared deque. + HeapObject* Pop(MarkingThread thread = MarkingThread::kMain) { + if (thread == MarkingThread::kMain) { + HeapObject* result = bailout_deque_.Pop(); + if (result != nullptr) return result; + } + return shared_deque_.Pop(); + } + + // All the following operations can used only by the main thread. + void Clear() { + bailout_deque_.Clear(); + shared_deque_.Clear(); + } + + bool IsFull() { return false; } + + bool IsEmpty() { return bailout_deque_.IsEmpty() && shared_deque_.IsEmpty(); } + + int Size() { return bailout_deque_.Size() + shared_deque_.Size(); } + + // This is used for a large array with a progress bar. + // For simpicity, unshift to the bailout deque so that the concurrent thread + // does not see such objects. + bool Unshift(HeapObject* object) { + bailout_deque_.Unshift(object); + return true; + } + + // Calls the specified callback on each element of the deques and replaces + // the element with the result of the callback. If the callback returns + // nullptr then the element is removed from the deque. + // The callback must accept HeapObject* and return HeapObject*. + template + void Update(Callback callback) { + bailout_deque_.Update(callback); + shared_deque_.Update(callback); + } + + // These empty functions are needed to match the interface + // of the sequential marking deque. + void SetUp() {} + void TearDown() {} + void StartUsing() {} + void StopUsing() {} + void ClearOverflowed() {} + void SetOverflowed() {} + bool overflowed() const { return false; } + + private: + // Simple, slow, and thread-safe deque that forwards all operations to + // a lock-protected std::deque. + class Deque { + public: + Deque() { cache_padding_[0] = 0; } + void Clear() { + base::LockGuard guard(&mutex_); + return deque_.clear(); + } + bool IsEmpty() { + base::LockGuard guard(&mutex_); + return deque_.empty(); + } + int Size() { + base::LockGuard guard(&mutex_); + return static_cast(deque_.size()); + } + void Push(HeapObject* object) { + base::LockGuard guard(&mutex_); + deque_.push_back(object); + } + HeapObject* Pop() { + base::LockGuard guard(&mutex_); + if (deque_.empty()) return nullptr; + HeapObject* result = deque_.back(); + deque_.pop_back(); + return result; + } + void Unshift(HeapObject* object) { + base::LockGuard guard(&mutex_); + deque_.push_front(object); + } + template + void Update(Callback callback) { + base::LockGuard guard(&mutex_); + std::deque new_deque; + for (auto object : deque_) { + HeapObject* new_object = callback(object); + if (new_object) { + new_deque.push_back(new_object); + } + } + deque_.swap(new_deque); + } + + private: + base::Mutex mutex_; + std::deque deque_; + // Ensure that two deques do not share the same cache line. + static int const kCachePadding = 64; + char cache_padding_[kCachePadding]; + }; + Deque bailout_deque_; + Deque shared_deque_; + DISALLOW_COPY_AND_ASSIGN(ConcurrentMarkingDeque); +}; + +} // namespace internal +} // namespace v8 + +#endif // V8_CONCURRENT_MARKING_DEQUE_ diff --git a/src/heap/concurrent-marking.cc b/src/heap/concurrent-marking.cc index 2a3377da3e..618cb1139b 100644 --- a/src/heap/concurrent-marking.cc +++ b/src/heap/concurrent-marking.cc @@ -7,6 +7,7 @@ #include #include +#include "src/heap/concurrent-marking-deque.h" #include "src/heap/heap-inl.h" #include "src/heap/heap.h" #include "src/heap/marking.h" @@ -21,43 +22,13 @@ namespace v8 { namespace internal { -class ConcurrentMarkingMarkbits { - public: - ConcurrentMarkingMarkbits() {} - ~ConcurrentMarkingMarkbits() { - for (auto chunk_bitmap : bitmap_) { - FreeBitmap(chunk_bitmap.second); - } - } - bool Mark(HeapObject* obj) { - Address address = obj->address(); - MemoryChunk* chunk = MemoryChunk::FromAddress(address); - if (bitmap_.count(chunk) == 0) { - bitmap_[chunk] = AllocateBitmap(); - } - MarkBit mark_bit = - bitmap_[chunk]->MarkBitFromIndex(chunk->AddressToMarkbitIndex(address)); - if (mark_bit.Get()) return false; - mark_bit.Set(); - return true; - } - - Bitmap* AllocateBitmap() { - return static_cast(calloc(1, Bitmap::kSize)); - } - - void FreeBitmap(Bitmap* bitmap) { free(bitmap); } - - private: - std::unordered_map bitmap_; -}; - class ConcurrentMarkingVisitor final : public HeapVisitor { public: using BaseClass = HeapVisitor; - ConcurrentMarkingVisitor() : bytes_marked_(0) {} + explicit ConcurrentMarkingVisitor(ConcurrentMarkingDeque* deque) + : deque_(deque) {} void VisitPointers(HeapObject* host, Object** start, Object** end) override { for (Object** p = start; p < end; p++) { @@ -154,84 +125,71 @@ class ConcurrentMarkingVisitor final } void MarkObject(HeapObject* obj) { - if (markbits_.Mark(obj)) { - marking_stack_.push(obj); - } + deque_->Push(obj, MarkingThread::kConcurrent, TargetDeque::kShared); } - void MarkTransitively() { - while (!marking_stack_.empty()) { - HeapObject* obj = marking_stack_.top(); - marking_stack_.pop(); - bytes_marked_ += IterateBody(obj); - } - } - - size_t bytes_marked() { return bytes_marked_; } - private: - size_t bytes_marked_; - std::stack marking_stack_; - ConcurrentMarkingMarkbits markbits_; + ConcurrentMarkingDeque* deque_; }; class ConcurrentMarking::Task : public CancelableTask { public: - Task(Heap* heap, std::vector* root_set, + Task(Isolate* isolate, ConcurrentMarking* concurrent_marking, base::Semaphore* on_finish) - : CancelableTask(heap->isolate()), - heap_(heap), - on_finish_(on_finish), - root_set_(root_set) {} + : CancelableTask(isolate), + concurrent_marking_(concurrent_marking), + on_finish_(on_finish) {} virtual ~Task() {} private: // v8::internal::CancelableTask overrides. void RunInternal() override { - double time_ms = heap_->MonotonicallyIncreasingTimeInMs(); - { - TimedScope scope(&time_ms); - for (HeapObject* obj : *root_set_) { - marking_visitor_.MarkObject(obj); - } - marking_visitor_.MarkTransitively(); - } - if (FLAG_trace_concurrent_marking) { - heap_->isolate()->PrintWithTimestamp( - "concurrently marked %dKB in %.2fms\n", - static_cast(marking_visitor_.bytes_marked() / KB), time_ms); - } + concurrent_marking_->Run(); on_finish_->Signal(); } - Heap* heap_; + ConcurrentMarking* concurrent_marking_; base::Semaphore* on_finish_; - ConcurrentMarkingVisitor marking_visitor_; - std::vector* root_set_; DISALLOW_COPY_AND_ASSIGN(Task); }; -ConcurrentMarking::ConcurrentMarking(Heap* heap) - : heap_(heap), pending_task_semaphore_(0), is_task_pending_(false) { +ConcurrentMarking::ConcurrentMarking(Heap* heap, ConcurrentMarkingDeque* deque) + : heap_(heap), + pending_task_semaphore_(0), + deque_(deque), + visitor_(new ConcurrentMarkingVisitor(deque_)), + is_task_pending_(false) { // Concurrent marking does not work with double unboxing. STATIC_ASSERT(!(V8_CONCURRENT_MARKING && V8_DOUBLE_FIELDS_UNBOXING)); // The runtime flag should be set only if the compile time flag was set. CHECK(!FLAG_concurrent_marking || V8_CONCURRENT_MARKING); } -ConcurrentMarking::~ConcurrentMarking() {} +ConcurrentMarking::~ConcurrentMarking() { delete visitor_; } -void ConcurrentMarking::AddRoot(HeapObject* object) { - root_set_.push_back(object); +void ConcurrentMarking::Run() { + double time_ms = heap_->MonotonicallyIncreasingTimeInMs(); + size_t bytes_marked = 0; + { + TimedScope scope(&time_ms); + HeapObject* object; + while ((object = deque_->Pop(MarkingThread::kConcurrent)) != nullptr) { + bytes_marked += visitor_->IterateBody(object); + } + } + if (FLAG_trace_concurrent_marking) { + heap_->isolate()->PrintWithTimestamp("concurrently marked %dKB in %.2fms\n", + static_cast(bytes_marked / KB), + time_ms); + } } void ConcurrentMarking::StartTask() { if (!FLAG_concurrent_marking) return; is_task_pending_ = true; - V8::GetCurrentPlatform()->CallOnBackgroundThread( - new Task(heap_, &root_set_, &pending_task_semaphore_), + new Task(heap_->isolate(), this, &pending_task_semaphore_), v8::Platform::kShortRunningTask); } @@ -239,7 +197,6 @@ void ConcurrentMarking::WaitForTaskToComplete() { if (!FLAG_concurrent_marking) return; pending_task_semaphore_.Wait(); is_task_pending_ = false; - root_set_.clear(); } void ConcurrentMarking::EnsureTaskCompleted() { diff --git a/src/heap/concurrent-marking.h b/src/heap/concurrent-marking.h index 9d7b6b58ca..134fa38f64 100644 --- a/src/heap/concurrent-marking.h +++ b/src/heap/concurrent-marking.h @@ -5,8 +5,6 @@ #ifndef V8_HEAP_CONCURRENT_MARKING_ #define V8_HEAP_CONCURRENT_MARKING_ -#include - #include "src/allocation.h" #include "src/cancelable-task.h" #include "src/utils.h" @@ -15,16 +13,16 @@ namespace v8 { namespace internal { +class ConcurrentMarkingDeque; +class ConcurrentMarkingVisitor; class Heap; class Isolate; class ConcurrentMarking { public: - explicit ConcurrentMarking(Heap* heap); + ConcurrentMarking(Heap* heap, ConcurrentMarkingDeque* deque_); ~ConcurrentMarking(); - void AddRoot(HeapObject* object); - void StartTask(); void WaitForTaskToComplete(); bool IsTaskPending() { return is_task_pending_; } @@ -32,10 +30,12 @@ class ConcurrentMarking { private: class Task; + void Run(); Heap* heap_; base::Semaphore pending_task_semaphore_; + ConcurrentMarkingDeque* deque_; + ConcurrentMarkingVisitor* visitor_; bool is_task_pending_; - std::vector root_set_; }; } // namespace internal diff --git a/src/heap/heap.cc b/src/heap/heap.cc index d8372733ee..89bc3a7562 100644 --- a/src/heap/heap.cc +++ b/src/heap/heap.cc @@ -5494,7 +5494,6 @@ bool Heap::SetUp() { store_buffer_ = new StoreBuffer(this); incremental_marking_ = new IncrementalMarking(this); - concurrent_marking_ = new ConcurrentMarking(this); for (int i = 0; i <= LAST_SPACE; i++) { space_[i] = nullptr; @@ -5543,6 +5542,8 @@ bool Heap::SetUp() { mark_compact_collector_ = new MarkCompactCollector(this); incremental_marking_->set_marking_deque( mark_compact_collector_->marking_deque()); + concurrent_marking_ = + new ConcurrentMarking(this, mark_compact_collector_->marking_deque()); if (FLAG_minor_mc) minor_mark_compact_collector_ = new MinorMarkCompactCollector(this); gc_idle_time_handler_ = new GCIdleTimeHandler(); diff --git a/src/heap/incremental-marking.cc b/src/heap/incremental-marking.cc index 5767ad5596..856819d37d 100644 --- a/src/heap/incremental-marking.cc +++ b/src/heap/incremental-marking.cc @@ -551,9 +551,6 @@ void IncrementalMarking::StartMarking() { if (FLAG_concurrent_marking) { ConcurrentMarking* concurrent_marking = heap_->concurrent_marking(); - marking_deque()->Iterate([concurrent_marking](HeapObject* obj) { - concurrent_marking->AddRoot(obj); - }); concurrent_marking->StartTask(); } diff --git a/src/heap/mark-compact.h b/src/heap/mark-compact.h index c63c0b0a66..a76936fc92 100644 --- a/src/heap/mark-compact.h +++ b/src/heap/mark-compact.h @@ -10,6 +10,7 @@ #include "src/base/bits.h" #include "src/base/platform/condition-variable.h" #include "src/cancelable-task.h" +#include "src/heap/concurrent-marking-deque.h" #include "src/heap/marking.h" #include "src/heap/sequential-marking-deque.h" #include "src/heap/spaces.h" @@ -31,7 +32,11 @@ class PageParallelJob; class RecordMigratedSlotVisitor; class ThreadLocalTop; +#ifdef V8_CONCURRENT_MARKING +using MarkingDeque = ConcurrentMarkingDeque; +#else using MarkingDeque = SequentialMarkingDeque; +#endif class ObjectMarking : public AllStatic { public: diff --git a/src/heap/sequential-marking-deque.h b/src/heap/sequential-marking-deque.h index 89b762f3ce..86098dd730 100644 --- a/src/heap/sequential-marking-deque.h +++ b/src/heap/sequential-marking-deque.h @@ -91,15 +91,6 @@ class SequentialMarkingDeque { } } - template - void Iterate(Callback callback) { - int i = bottom_; - while (i != top_) { - callback(array_[i]); - i = (i + 1) & mask_; - } - } - // Calls the specified callback on each element of the deque and replaces // the element with the result of the callback. If the callback returns // nullptr then the element is removed from the deque. diff --git a/src/v8.gyp b/src/v8.gyp index a24f0e5312..0a425c7e59 100644 --- a/src/v8.gyp +++ b/src/v8.gyp @@ -951,6 +951,7 @@ 'heap/array-buffer-tracker.h', 'heap/code-stats.cc', 'heap/code-stats.h', + 'heap/concurrent-marking-deque.h', 'heap/concurrent-marking.cc', 'heap/concurrent-marking.h', 'heap/embedder-tracing.cc', diff --git a/test/cctest/heap/test-concurrent-marking.cc b/test/cctest/heap/test-concurrent-marking.cc index ce52018e29..03ee6c4b8c 100644 --- a/test/cctest/heap/test-concurrent-marking.cc +++ b/test/cctest/heap/test-concurrent-marking.cc @@ -18,8 +18,9 @@ TEST(ConcurrentMarking) { if (!i::FLAG_concurrent_marking) return; CcTest::InitializeVM(); Heap* heap = CcTest::heap(); - ConcurrentMarking* concurrent_marking = new ConcurrentMarking(heap); - concurrent_marking->AddRoot(heap->undefined_value()); + ConcurrentMarkingDeque deque(heap); + deque.Push(heap->undefined_value()); + ConcurrentMarking* concurrent_marking = new ConcurrentMarking(heap, &deque); concurrent_marking->StartTask(); concurrent_marking->WaitForTaskToComplete(); delete concurrent_marking; diff --git a/test/cctest/heap/test-mark-compact.cc b/test/cctest/heap/test-mark-compact.cc index 7242e1e7c4..36c5ce9e11 100644 --- a/test/cctest/heap/test-mark-compact.cc +++ b/test/cctest/heap/test-mark-compact.cc @@ -43,6 +43,7 @@ #include "src/global-handles.h" #include "src/heap/mark-compact-inl.h" #include "src/heap/mark-compact.h" +#include "src/heap/sequential-marking-deque.h" #include "src/objects-inl.h" #include "test/cctest/cctest.h" #include "test/cctest/heap/heap-tester.h" @@ -51,10 +52,9 @@ using namespace v8::internal; using v8::Just; - -TEST(MarkingDeque) { +TEST(SequentialMarkingDeque) { CcTest::InitializeVM(); - MarkingDeque s(CcTest::i_isolate()->heap()); + SequentialMarkingDeque s(CcTest::i_isolate()->heap()); s.SetUp(); s.StartUsing(); Address original_address = reinterpret_cast
(&s); diff --git a/test/unittests/BUILD.gn b/test/unittests/BUILD.gn index 25aba250c4..18517bf44f 100644 --- a/test/unittests/BUILD.gn +++ b/test/unittests/BUILD.gn @@ -100,6 +100,7 @@ v8_executable("unittests") { "eh-frame-iterator-unittest.cc", "eh-frame-writer-unittest.cc", "heap/bitmap-unittest.cc", + "heap/concurrent-marking-deque-unittest.cc", "heap/embedder-tracing-unittest.cc", "heap/gc-idle-time-handler-unittest.cc", "heap/gc-tracer-unittest.cc", diff --git a/test/unittests/heap/concurrent-marking-deque-unittest.cc b/test/unittests/heap/concurrent-marking-deque-unittest.cc new file mode 100644 index 0000000000..25369217e3 --- /dev/null +++ b/test/unittests/heap/concurrent-marking-deque-unittest.cc @@ -0,0 +1,57 @@ +// Copyright 2017 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "src/globals.h" +#include "src/heap/concurrent-marking-deque.h" +#include "src/heap/heap-inl.h" +#include "src/isolate.h" +#include "test/unittests/test-utils.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace v8 { +namespace internal { + +class ConcurrentMarkingDequeTest : public TestWithIsolate { + public: + ConcurrentMarkingDequeTest() { + marking_deque_ = new ConcurrentMarkingDeque(i_isolate()->heap()); + object_ = i_isolate()->heap()->undefined_value(); + } + + ~ConcurrentMarkingDequeTest() { delete marking_deque_; } + + ConcurrentMarkingDeque* marking_deque() { return marking_deque_; } + + HeapObject* object() { return object_; } + + private: + ConcurrentMarkingDeque* marking_deque_; + HeapObject* object_; + DISALLOW_COPY_AND_ASSIGN(ConcurrentMarkingDequeTest); +}; + +TEST_F(ConcurrentMarkingDequeTest, Empty) { + EXPECT_TRUE(marking_deque()->IsEmpty()); + EXPECT_EQ(0, marking_deque()->Size()); +} + +TEST_F(ConcurrentMarkingDequeTest, SharedDeque) { + marking_deque()->Push(object()); + EXPECT_FALSE(marking_deque()->IsEmpty()); + EXPECT_EQ(1, marking_deque()->Size()); + EXPECT_EQ(object(), marking_deque()->Pop(MarkingThread::kConcurrent)); +} + +TEST_F(ConcurrentMarkingDequeTest, BailoutDeque) { + marking_deque()->Push(object(), MarkingThread::kConcurrent, + TargetDeque::kBailout); + EXPECT_FALSE(marking_deque()->IsEmpty()); + EXPECT_EQ(1, marking_deque()->Size()); + EXPECT_EQ(nullptr, marking_deque()->Pop(MarkingThread::kConcurrent)); +} + +} // namespace internal +} // namespace v8 diff --git a/test/unittests/unittests.gyp b/test/unittests/unittests.gyp index 0c9c4a664a..6dc486d506 100644 --- a/test/unittests/unittests.gyp +++ b/test/unittests/unittests.gyp @@ -97,6 +97,7 @@ 'eh-frame-iterator-unittest.cc', 'eh-frame-writer-unittest.cc', 'heap/bitmap-unittest.cc', + 'heap/concurrent-marking-deque-unittest.cc', 'heap/embedder-tracing-unittest.cc', 'heap/gc-idle-time-handler-unittest.cc', 'heap/gc-tracer-unittest.cc',