diff --git a/BUILD.gn b/BUILD.gn index 57a813bdc1..2af183e1f1 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -4014,6 +4014,8 @@ v8_source_set("cppgc_base") { "include/cppgc/visitor.h", "include/v8config.h", "src/heap/cppgc/allocation.cc", + "src/heap/cppgc/free-list.cc", + "src/heap/cppgc/free-list.h", "src/heap/cppgc/gc-info-table.cc", "src/heap/cppgc/gc-info-table.h", "src/heap/cppgc/gc-info.cc", diff --git a/src/heap/cppgc/free-list.cc b/src/heap/cppgc/free-list.cc new file mode 100644 index 0000000000..4483386d34 --- /dev/null +++ b/src/heap/cppgc/free-list.cc @@ -0,0 +1,168 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/heap/cppgc/free-list.h" + +#include + +#include "include/cppgc/internal/logging.h" +#include "src/base/bits.h" +#include "src/heap/cppgc/globals.h" +#include "src/heap/cppgc/heap-object-header-inl.h" + +namespace cppgc { +namespace internal { + +namespace { +uint32_t BucketIndexForSize(uint32_t size) { + return v8::base::bits::WhichPowerOfTwo( + v8::base::bits::RoundDownToPowerOfTwo32(size)); +} +} // namespace + +class FreeList::Entry : public HeapObjectHeader { + public: + explicit Entry(size_t size) : HeapObjectHeader(size, kFreeListGCInfoIndex) { + static_assert(sizeof(Entry) == kFreeListEntrySize, "Sizes must match"); + } + + Entry* Next() const { return next_; } + void SetNext(Entry* next) { next_ = next; } + + void Link(Entry** previous_next) { + next_ = *previous_next; + *previous_next = this; + } + void Unlink(Entry** previous_next) { + *previous_next = next_; + next_ = nullptr; + } + + private: + Entry* next_ = nullptr; +}; + +FreeList::FreeList() { Clear(); } + +FreeList::FreeList(FreeList&& other) V8_NOEXCEPT + : free_list_heads_(std::move(other.free_list_heads_)), + free_list_tails_(std::move(other.free_list_tails_)), + biggest_free_list_index_(std::move(other.biggest_free_list_index_)) { + other.Clear(); +} + +FreeList& FreeList::operator=(FreeList&& other) V8_NOEXCEPT { + Clear(); + Append(std::move(other)); + DCHECK(other.IsEmpty()); + return *this; +} + +void FreeList::Add(FreeList::Block block) { + const size_t size = block.size; + DCHECK_GT(kPageSize, size); + DCHECK_LE(kFreeListEntrySize, size); + + Entry* entry = new (block.address) Entry(size); + const size_t index = BucketIndexForSize(static_cast(size)); + entry->Link(&free_list_heads_[index]); + biggest_free_list_index_ = std::max(biggest_free_list_index_, index); + if (!entry->Next()) { + free_list_tails_[index] = entry; + } +} + +void FreeList::Append(FreeList&& other) { +#if DEBUG + const size_t expected_size = Size() + other.Size(); +#endif + // Newly created entries get added to the head. + for (size_t index = 0; index < free_list_tails_.size(); ++index) { + Entry* other_tail = other.free_list_tails_[index]; + Entry*& this_head = this->free_list_heads_[index]; + if (other_tail) { + other_tail->SetNext(this_head); + if (!this_head) { + this->free_list_tails_[index] = other_tail; + } + this_head = other.free_list_heads_[index]; + other.free_list_heads_[index] = nullptr; + other.free_list_tails_[index] = nullptr; + } + } + + biggest_free_list_index_ = + std::max(biggest_free_list_index_, other.biggest_free_list_index_); + other.biggest_free_list_index_ = 0; +#if DEBUG + DCHECK_EQ(expected_size, Size()); +#endif + DCHECK(other.IsEmpty()); +} + +FreeList::Block FreeList::Allocate(size_t allocation_size) { + // Try reusing a block from the largest bin. The underlying reasoning + // being that we want to amortize this slow allocation call by carving + // off as a large a free block as possible in one go; a block that will + // service this block and let following allocations be serviced quickly + // by bump allocation. + // bucket_size represents minimal size of entries in a bucket. + size_t bucket_size = static_cast(1) << biggest_free_list_index_; + size_t index = biggest_free_list_index_; + for (; index > 0; --index, bucket_size >>= 1) { + DCHECK(IsConsistent(index)); + Entry* entry = free_list_heads_[index]; + if (allocation_size > bucket_size) { + // Final bucket candidate; check initial entry if it is able + // to service this allocation. Do not perform a linear scan, + // as it is considered too costly. + if (!entry || entry->GetSize() < allocation_size) break; + } + if (entry) { + if (!entry->Next()) { + DCHECK_EQ(entry, free_list_tails_[index]); + free_list_tails_[index] = nullptr; + } + entry->Unlink(&free_list_heads_[index]); + biggest_free_list_index_ = index; + return {entry, entry->GetSize()}; + } + } + biggest_free_list_index_ = index; + return {nullptr, 0u}; +} + +void FreeList::Clear() { + std::fill(free_list_heads_.begin(), free_list_heads_.end(), nullptr); + std::fill(free_list_tails_.begin(), free_list_tails_.end(), nullptr); + biggest_free_list_index_ = 0; +} + +size_t FreeList::Size() const { + size_t size = 0; + for (auto* entry : free_list_heads_) { + while (entry) { + size += entry->GetSize(); + entry = entry->Next(); + } + } + return size; +} + +bool FreeList::IsEmpty() const { + return std::all_of(free_list_heads_.cbegin(), free_list_heads_.cend(), + [](const auto* entry) { return !entry; }); +} + +bool FreeList::IsConsistent(size_t index) const { + // Check that freelist head and tail pointers are consistent, i.e. + // - either both are nulls (no entries in the bucket); + // - or both are non-nulls and the tail points to the end. + return (!free_list_heads_[index] && !free_list_tails_[index]) || + (free_list_heads_[index] && free_list_tails_[index] && + !free_list_tails_[index]->Next()); +} + +} // namespace internal +} // namespace cppgc diff --git a/src/heap/cppgc/free-list.h b/src/heap/cppgc/free-list.h new file mode 100644 index 0000000000..54c59cd9c2 --- /dev/null +++ b/src/heap/cppgc/free-list.h @@ -0,0 +1,60 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_HEAP_CPPGC_FREE_LIST_H_ +#define V8_HEAP_CPPGC_FREE_LIST_H_ + +#include + +#include "src/base/macros.h" +#include "src/heap/cppgc/globals.h" +#include "src/heap/cppgc/heap-object-header.h" + +namespace cppgc { +namespace internal { + +class V8_EXPORT_PRIVATE FreeList { + public: + struct Block { + void* address; + size_t size; + }; + + FreeList(); + + FreeList(const FreeList&) = delete; + FreeList& operator=(const FreeList&) = delete; + + FreeList(FreeList&& freelist) V8_NOEXCEPT; + FreeList& operator=(FreeList&& freelist) V8_NOEXCEPT; + + // Allocates entries which are at least of the provided size. + Block Allocate(size_t); + + // Adds block to the freelist. The minimal block size is two words. + void Add(Block); + + // Append other freelist into this. + void Append(FreeList&&); + + void Clear(); + + size_t Size() const; + bool IsEmpty() const; + + private: + class Entry; + + bool IsConsistent(size_t) const; + + // All |Entry|s in the nth list have size >= 2^n. + std::array free_list_heads_; + std::array free_list_tails_; + size_t biggest_free_list_index_ = 0; +}; + +} // namespace internal +} // namespace cppgc + +#endif // V8_HEAP_CPPGC_FREE_LIST_H_ diff --git a/src/heap/cppgc/globals.h b/src/heap/cppgc/globals.h index 6ca6a92e0b..734abd508e 100644 --- a/src/heap/cppgc/globals.h +++ b/src/heap/cppgc/globals.h @@ -8,6 +8,8 @@ #include #include +#include "include/cppgc/internal/gc-info.h" + namespace cppgc { namespace internal { @@ -37,6 +39,9 @@ constexpr size_t kGuardPageSize = 4096; constexpr size_t kLargeObjectSizeThreshold = kPageSize / 2; +constexpr GCInfoIndex kFreeListGCInfoIndex = 0; +constexpr size_t kFreeListEntrySize = 2 * sizeof(uintptr_t); + } // namespace internal } // namespace cppgc diff --git a/src/heap/cppgc/heap-object-header-inl.h b/src/heap/cppgc/heap-object-header-inl.h index 129ea605e7..184e151658 100644 --- a/src/heap/cppgc/heap-object-header-inl.h +++ b/src/heap/cppgc/heap-object-header-inl.h @@ -11,6 +11,7 @@ #include "src/base/logging.h" #include "src/base/macros.h" #include "src/heap/cppgc/gc-info-table.h" +#include "src/heap/cppgc/globals.h" #include "src/heap/cppgc/heap-object-header.h" namespace cppgc { @@ -33,7 +34,7 @@ HeapObjectHeader::HeapObjectHeader(size_t size, GCInfoIndex gc_info_index) { USE(padding_); #endif // defined(V8_TARGET_ARCH_64_BIT) DCHECK_LT(gc_info_index, GCInfoTable::kMaxIndex); - DCHECK_EQ(0u, size & kAllocationMask); + DCHECK_EQ(0u, size & (sizeof(HeapObjectHeader) - 1)); DCHECK_GE(kMaxSize, size); encoded_high_ = GCInfoIndexField::encode(gc_info_index); encoded_low_ = EncodeSize(size); @@ -111,6 +112,11 @@ bool HeapObjectHeader::TryMarkAtomic() { std::memory_order_relaxed); } +template +bool HeapObjectHeader::IsFree() const { + return GetGCInfoIndex() == kFreeListGCInfoIndex; +} + template uint16_t HeapObjectHeader::LoadEncoded() const { diff --git a/src/heap/cppgc/heap-object-header.h b/src/heap/cppgc/heap-object-header.h index c635842090..c8a2a8378f 100644 --- a/src/heap/cppgc/heap-object-header.h +++ b/src/heap/cppgc/heap-object-header.h @@ -41,7 +41,7 @@ namespace internal { // stored in |LargeObjectPage::PayloadSize()|. // - |mark bit| and |in construction| bits are located in separate 16-bit halves // to allow potentially accessing them non-atomically. -class HeapObjectHeader final { +class HeapObjectHeader { public: enum class AccessMode : uint8_t { kNonAtomic, kAtomic }; @@ -77,6 +77,9 @@ class HeapObjectHeader final { void Unmark(); inline bool TryMarkAtomic(); + template + bool IsFree() const; + void Finalize(); private: @@ -102,7 +105,7 @@ class HeapObjectHeader final { static constexpr uint16_t EncodeSize(size_t size) { // Essentially, gets optimized to >> 1. using SizeField = UnusedField2::Next; - return SizeField::encode(size) / kAllocationGranularity; + return SizeField::encode(size / kAllocationGranularity); } V8_EXPORT_PRIVATE void CheckApiConstants(); diff --git a/test/unittests/BUILD.gn b/test/unittests/BUILD.gn index ec43cf1358..2fa7d2ca4c 100644 --- a/test/unittests/BUILD.gn +++ b/test/unittests/BUILD.gn @@ -45,6 +45,7 @@ v8_source_set("cppgc_unittests_sources") { sources = [ "heap/cppgc/finalizer-trait_unittest.cc", + "heap/cppgc/free-list_unittest.cc", "heap/cppgc/garbage-collected_unittest.cc", "heap/cppgc/gc-info_unittest.cc", "heap/cppgc/heap-object-header_unittest.cc", diff --git a/test/unittests/heap/cppgc/free-list_unittest.cc b/test/unittests/heap/cppgc/free-list_unittest.cc new file mode 100644 index 0000000000..59946e9cbd --- /dev/null +++ b/test/unittests/heap/cppgc/free-list_unittest.cc @@ -0,0 +1,169 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include +#include + +#include "src/base/bits.h" +#include "src/heap/cppgc/free-list.h" +#include "src/heap/cppgc/globals.h" +#include "src/heap/cppgc/heap-object-header.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace cppgc { +namespace internal { +namespace { + +class Block { + public: + Block() = default; + explicit Block(size_t size) : address_(calloc(1, size)), size_(size) {} + + Block(Block&& other) V8_NOEXCEPT : address_(other.address_), + size_(other.size_) { + other.address_ = nullptr; + other.size_ = 0; + } + + Block& operator=(Block&& other) V8_NOEXCEPT { + address_ = other.address_; + size_ = other.size_; + other.address_ = nullptr; + other.size_ = 0; + return *this; + } + + ~Block() { free(address_); } + + void* Address() const { return address_; } + size_t Size() const { return size_; } + + private: + void* address_ = nullptr; + size_t size_ = 0; +}; + +std::vector CreateEntries() { + static constexpr size_t kFreeListEntrySizeLog2 = + v8::base::bits::WhichPowerOfTwo(kFreeListEntrySize); + std::vector vector; + vector.reserve(kPageSizeLog2); + for (size_t i = kFreeListEntrySizeLog2; i < kPageSizeLog2; ++i) { + vector.emplace_back(static_cast(1u) << i); + } + return vector; +} + +FreeList CreatePopulatedFreeList(const std::vector& blocks) { + FreeList list; + for (const auto& block : blocks) { + list.Add({block.Address(), block.Size()}); + } + return list; +} + +} // namespace + +TEST(FreeListTest, Empty) { + FreeList list; + EXPECT_TRUE(list.IsEmpty()); + EXPECT_EQ(0u, list.Size()); + + auto block = list.Allocate(16); + EXPECT_EQ(nullptr, block.address); + EXPECT_EQ(0u, block.size); +} + +TEST(FreeListTest, Add) { + auto blocks = CreateEntries(); + FreeList list = CreatePopulatedFreeList(blocks); + EXPECT_FALSE(list.IsEmpty()); + const size_t allocated_size = std::accumulate( + blocks.cbegin(), blocks.cend(), 0u, + [](size_t acc, const Block& b) { return acc + b.Size(); }); + EXPECT_EQ(allocated_size, list.Size()); +} + +TEST(FreeListTest, Clear) { + auto blocks = CreateEntries(); + FreeList list = CreatePopulatedFreeList(blocks); + list.Clear(); + EXPECT_EQ(0u, list.Size()); + EXPECT_TRUE(list.IsEmpty()); +} + +TEST(FreeListTest, Move) { + { + auto blocks = CreateEntries(); + FreeList list1 = CreatePopulatedFreeList(blocks); + const size_t expected_size = list1.Size(); + FreeList list2 = std::move(list1); + EXPECT_EQ(expected_size, list2.Size()); + EXPECT_FALSE(list2.IsEmpty()); + EXPECT_EQ(0u, list1.Size()); + EXPECT_TRUE(list1.IsEmpty()); + } + { + auto blocks1 = CreateEntries(); + FreeList list1 = CreatePopulatedFreeList(blocks1); + const size_t expected_size = list1.Size(); + + auto blocks2 = CreateEntries(); + FreeList list2 = CreatePopulatedFreeList(blocks2); + + list2 = std::move(list1); + EXPECT_EQ(expected_size, list2.Size()); + EXPECT_FALSE(list2.IsEmpty()); + EXPECT_EQ(0u, list1.Size()); + EXPECT_TRUE(list1.IsEmpty()); + } +} + +TEST(FreeListTest, Append) { + auto blocks1 = CreateEntries(); + FreeList list1 = CreatePopulatedFreeList(blocks1); + const size_t list1_size = list1.Size(); + + auto blocks2 = CreateEntries(); + FreeList list2 = CreatePopulatedFreeList(blocks2); + const size_t list2_size = list1.Size(); + + list2.Append(std::move(list1)); + EXPECT_EQ(list1_size + list2_size, list2.Size()); + EXPECT_FALSE(list2.IsEmpty()); + EXPECT_EQ(0u, list1.Size()); + EXPECT_TRUE(list1.IsEmpty()); +} + +TEST(FreeListTest, Allocate) { + static constexpr size_t kFreeListEntrySizeLog2 = + v8::base::bits::WhichPowerOfTwo(kFreeListEntrySize); + + std::vector blocks; + blocks.reserve(kPageSizeLog2); + for (size_t i = kFreeListEntrySizeLog2; i < kPageSizeLog2; ++i) { + blocks.emplace_back(static_cast(1u) << i); + } + + FreeList list = CreatePopulatedFreeList(blocks); + + // Try allocate from the biggest block. + for (auto it = blocks.rbegin(); it < blocks.rend(); ++it) { + const auto result = list.Allocate(it->Size()); + EXPECT_EQ(it->Address(), result.address); + EXPECT_EQ(it->Size(), result.size); + } + + EXPECT_EQ(0u, list.Size()); + EXPECT_TRUE(list.IsEmpty()); + + // Check that allocation fails for empty list: + const auto empty_block = list.Allocate(8); + EXPECT_EQ(nullptr, empty_block.address); + EXPECT_EQ(0u, empty_block.size); +} + +} // namespace internal +} // namespace cppgc