diff --git a/BUILD.bazel b/BUILD.bazel index 66af9b1f81..84a2497381 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1855,6 +1855,9 @@ filegroup( "src/objects/string-inl.h", "src/objects/string-set-inl.h", "src/objects/string-set.h", + "src/objects/string-forwarding-table-inl.h", + "src/objects/string-forwarding-table.cc", + "src/objects/string-forwarding-table.h", "src/objects/string-table-inl.h", "src/objects/string-table.cc", "src/objects/symbol-table.cc", diff --git a/BUILD.gn b/BUILD.gn index cbac619a72..d1b4bae608 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -3410,6 +3410,8 @@ v8_header_set("v8_internal_headers") { "src/objects/source-text-module-inl.h", "src/objects/source-text-module.h", "src/objects/string-comparator.h", + "src/objects/string-forwarding-table-inl.h", + "src/objects/string-forwarding-table.h", "src/objects/string-inl.h", "src/objects/string-set-inl.h", "src/objects/string-set.h", @@ -4637,6 +4639,7 @@ v8_source_set("v8_base_without_compiler") { "src/objects/simd.cc", "src/objects/source-text-module.cc", "src/objects/string-comparator.cc", + "src/objects/string-forwarding-table.cc", "src/objects/string-table.cc", "src/objects/string.cc", "src/objects/swiss-name-dictionary.cc", diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc index 0b201e159d..2113343df2 100644 --- a/src/heap/mark-compact.cc +++ b/src/heap/mark-compact.cc @@ -66,6 +66,7 @@ #include "src/objects/objects.h" #include "src/objects/slots-inl.h" #include "src/objects/smi.h" +#include "src/objects/string-forwarding-table-inl.h" #include "src/objects/transitions-inl.h" #include "src/objects/visitors.h" #include "src/snapshot/shared-heap-serializer.h" @@ -1436,64 +1437,6 @@ class InternalizedStringTableCleaner final : public RootVisitor { int pointers_removed_ = 0; }; -class StringForwardingTableCleaner final : public RootVisitor { - public: - explicit StringForwardingTableCleaner(Heap* heap) : heap_(heap) {} - - void VisitRootPointers(Root root, const char* description, - FullObjectSlot start, FullObjectSlot end) override { - UNREACHABLE(); - } - - void VisitRootPointers(Root root, const char* description, - OffHeapObjectSlot start, - OffHeapObjectSlot end) override { - DCHECK_EQ(root, Root::kStringForwardingTable); - // Visit all HeapObject pointers in [start, end). - // The forwarding table is organized in pairs of [orig string, forward - // string]. - auto* marking_state = - heap_->mark_compact_collector()->non_atomic_marking_state(); - Isolate* isolate = heap_->isolate(); - for (OffHeapObjectSlot p = start; p < end; p += 2) { - Object original = p.load(isolate); - if (!original.IsHeapObject()) { - // Only if we always use the forwarding table, the string could be a - // smi, indicating that the entry died during scavenge. - DCHECK(FLAG_always_use_string_forwarding_table); - DCHECK_EQ(original, StringForwardingTable::deleted_element()); - continue; - } - if (marking_state->IsBlack(HeapObject::cast(original))) { - String original_string = String::cast(original); - // Check if the string was already transitioned. This happens if we have - // multiple entries for the same string in the table. - if (original_string.IsThinString()) continue; - - // The second slot of each record is the forward string. - Object forward = (p + 1).load(isolate); - String forward_string = String::cast(forward); - - // Mark the forwarded string. - marking_state->WhiteToBlack(forward_string); - - // Transition the original string to a ThinString and override the - // forwarding index with the correct hash. - original_string.MakeThin(isolate, forward_string); - original_string.set_raw_hash_field(forward_string.raw_hash_field()); - // Record the slot in the old-to-old remembered set. This is required as - // the internalized string could be relocated during compaction. - ObjectSlot slot = ThinString::cast(original_string) - .RawField(ThinString::kActualOffset); - MarkCompactCollector::RecordSlot(original_string, slot, forward_string); - } - } - } - - private: - Heap* heap_; -}; - class ExternalStringTableCleaner : public RootVisitor { public: explicit ExternalStringTableCleaner(Heap* heap) : heap_(heap) {} @@ -2866,6 +2809,62 @@ class ClearStringTableJobItem final : public ParallelClearingJob::ClearingItem { Isolate* const isolate_; }; +class StringForwardingTableCleaner final { + public: + explicit StringForwardingTableCleaner(Heap* heap) + : heap_(heap), + isolate_(heap_->isolate()), + marking_state_( + heap_->mark_compact_collector()->non_atomic_marking_state()) {} + void Run() { + StringForwardingTable* forwarding_table = + isolate_->string_forwarding_table(); + forwarding_table->IterateElements( + isolate_, [&](StringForwardingTable::Record* record) { + TransitionStrings(record); + }); + forwarding_table->Reset(); + } + + private: + void TransitionStrings(StringForwardingTable::Record* record) { + Object original = record->OriginalStringObject(isolate_); + if (!original.IsHeapObject()) { + // Only if we always use the forwarding table, the string could be a + // smi, indicating that the entry died during scavenge. + DCHECK(FLAG_always_use_string_forwarding_table); + DCHECK_EQ(original, StringForwardingTable::deleted_element()); + return; + } + if (marking_state_->IsBlack(HeapObject::cast(original))) { + String original_string = String::cast(original); + TryInternalize(original_string, record); + original_string.set_raw_hash_field(record->raw_hash(isolate_)); + } + } + + void TryInternalize(String original_string, + StringForwardingTable::Record* record) { + if (original_string.IsThinString()) return; + String forward_string = record->forward_string(isolate_); + + // Mark the forwarded string to keep it alive. + marking_state_->WhiteToBlack(forward_string); + // Transition the original string to a ThinString and override the + // forwarding index with the correct hash. + original_string.MakeThin(isolate_, forward_string); + // Record the slot in the old-to-old remembered set. This is + // required as the internalized string could be relocated during + // compaction. + ObjectSlot slot = + ThinString::cast(original_string).RawField(ThinString::kActualOffset); + MarkCompactCollector::RecordSlot(original_string, slot, forward_string); + } + Heap* heap_; + Isolate* isolate_; + NonAtomicMarkingState* marking_state_; +}; + } // namespace void MarkCompactCollector::ClearNonLiveReferences() { @@ -2879,11 +2878,8 @@ void MarkCompactCollector::ClearNonLiveReferences() { // Clearing the string forwarding table must happen before clearing the // string table, as entries in the forwarding table can keep internalized // strings alive. - StringForwardingTable* forwarding_table = - isolate()->string_forwarding_table(); - StringForwardingTableCleaner visitor(heap()); - forwarding_table->IterateElements(&visitor); - forwarding_table->Reset(); + StringForwardingTableCleaner forwarding_table_cleaner(heap()); + forwarding_table_cleaner.Run(); } auto clearing_job = std::make_unique(); diff --git a/src/objects/all-objects-inl.h b/src/objects/all-objects-inl.h index 51f1d653b9..568f731f0c 100644 --- a/src/objects/all-objects-inl.h +++ b/src/objects/all-objects-inl.h @@ -76,6 +76,7 @@ #include "src/objects/shared-function-info-inl.h" #include "src/objects/slots-atomic-inl.h" #include "src/objects/slots-inl.h" +#include "src/objects/string-forwarding-table-inl.h" #include "src/objects/string-inl.h" #include "src/objects/string-set-inl.h" #include "src/objects/string-table-inl.h" diff --git a/src/objects/name-inl.h b/src/objects/name-inl.h index 608a2b986f..26de099738 100644 --- a/src/objects/name-inl.h +++ b/src/objects/name-inl.h @@ -10,8 +10,8 @@ #include "src/objects/map-inl.h" #include "src/objects/name.h" #include "src/objects/primitive-heap-object-inl.h" +#include "src/objects/string-forwarding-table.h" #include "src/objects/string-inl.h" -#include "src/objects/string-table.h" // Has to be the last include (doesn't have include guards): #include "src/objects/object-macros.h" diff --git a/src/objects/string-forwarding-table-inl.h b/src/objects/string-forwarding-table-inl.h new file mode 100644 index 0000000000..eefecefe56 --- /dev/null +++ b/src/objects/string-forwarding-table-inl.h @@ -0,0 +1,207 @@ +// Copyright 2022 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_ +#define V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_ + +#include "src/base/atomicops.h" +#include "src/common/globals.h" +#include "src/objects/name-inl.h" +#include "src/objects/slots-inl.h" +#include "src/objects/slots.h" +#include "src/objects/string-forwarding-table.h" +#include "src/objects/string-inl.h" +// Has to be the last include (doesn't have include guards): +#include "src/objects/object-macros.h" + +namespace v8 { +namespace internal { + +class StringForwardingTable::Record final { + public: + String original_string(PtrComprCageBase cage_base) const { + return String::cast(OriginalStringObject(cage_base)); + } + + String forward_string(PtrComprCageBase cage_base) const { + return String::cast(ForwardStringObject(cage_base)); + } + + inline uint32_t raw_hash(PtrComprCageBase cage_base) const; + + Object OriginalStringObject(PtrComprCageBase cage_base) const { + return OriginalStringSlot().Acquire_Load(cage_base); + } + + Object ForwardStringObject(PtrComprCageBase cage_base) const { + return ForwardStringSlot().Acquire_Load(cage_base); + } + + void set_original_string(Object object) { + OriginalStringSlot().Release_Store(object); + } + + void set_forward_string(Object object) { + ForwardStringSlot().Release_Store(object); + } + + inline void SetInternalized(String string, String forward_to); + + private: + OffHeapObjectSlot OriginalStringSlot() const { + return OffHeapObjectSlot(&original_string_); + } + + OffHeapObjectSlot ForwardStringSlot() const { + return OffHeapObjectSlot(&forward_string_); + } + + Tagged_t original_string_; + Tagged_t forward_string_; + + friend class StringForwardingTable::Block; +}; + +uint32_t StringForwardingTable::Record::raw_hash( + PtrComprCageBase cage_base) const { + String internalized = forward_string(cage_base); + uint32_t raw_hash = internalized.raw_hash_field(); + DCHECK(Name::IsHashFieldComputed(raw_hash)); + return raw_hash; +} + +void StringForwardingTable::Record::SetInternalized(String string, + String forward_to) { + set_original_string(string); + set_forward_string(forward_to); +} + +class StringForwardingTable::Block { + public: + static std::unique_ptr New(int capacity); + explicit Block(int capacity); + int capacity() const { return capacity_; } + void* operator new(size_t size, int capacity); + void* operator new(size_t size) = delete; + void operator delete(void* data); + + Record* record(int index) { + DCHECK_LT(index, capacity()); + return &elements_[index]; + } + + const Record* record(int index) const { + DCHECK_LT(index, capacity()); + return &elements_[index]; + } + + void UpdateAfterEvacuation(PtrComprCageBase cage_base); + void UpdateAfterEvacuation(PtrComprCageBase cage_base, int up_to_index); + + private: + const int capacity_; + Record elements_[1]; +}; + +class StringForwardingTable::BlockVector { + public: + using Block = StringForwardingTable::Block; + using Allocator = std::allocator; + + explicit BlockVector(size_t capacity); + ~BlockVector(); + size_t capacity() const { return capacity_; } + + Block* LoadBlock(size_t index, AcquireLoadTag) { + DCHECK_LT(index, size()); + return base::AsAtomicPointer::Acquire_Load(&begin_[index]); + } + + Block* LoadBlock(size_t index) { + DCHECK_LT(index, size()); + return begin_[index]; + } + + void AddBlock(std::unique_ptr block) { + DCHECK_LT(size(), capacity()); + base::AsAtomicPointer::Release_Store(&begin_[size_], block.release()); + size_++; + } + + static std::unique_ptr Grow(BlockVector* data, size_t capacity, + const base::Mutex& mutex); + + size_t size() const { return size_; } + + private: + V8_NO_UNIQUE_ADDRESS Allocator allocator_; + const size_t capacity_; + std::atomic size_; + Block** begin_; +}; + +int StringForwardingTable::size() const { return next_free_index_; } +bool StringForwardingTable::empty() const { return size() == 0; } + +// static +uint32_t StringForwardingTable::BlockForIndex(int index, + uint32_t* index_in_block) { + DCHECK_GE(index, 0); + DCHECK_NOT_NULL(index_in_block); + // The block is the leftmost set bit of the index, corrected by the size + // of the first block. + const uint32_t block_index = + kBitsPerInt - + base::bits::CountLeadingZeros( + static_cast(index + kInitialBlockSize)) - + kInitialBlockSizeHighestBit - 1; + *index_in_block = IndexInBlock(index, block_index); + return block_index; +} + +// static +uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block_index) { + DCHECK_GE(index, 0); + // Clear out the leftmost set bit (the block index) to get the index within + // the block. + return static_cast(index + kInitialBlockSize) & + ~(1u << (block_index + kInitialBlockSizeHighestBit)); +} + +// static +uint32_t StringForwardingTable::CapacityForBlock(uint32_t block_index) { + return 1u << (block_index + kInitialBlockSizeHighestBit); +} + +template +void StringForwardingTable::IterateElements(Isolate* isolate, Func&& callback) { + isolate->heap()->safepoint()->AssertActive(); + DCHECK_NE(isolate->heap()->gc_state(), Heap::NOT_IN_GC); + + if (empty()) return; + BlockVector* blocks = blocks_.load(std::memory_order_relaxed); + const uint32_t last_block_index = static_cast(blocks->size() - 1); + for (uint32_t block_index = 0; block_index < last_block_index; + ++block_index) { + Block* block = blocks->LoadBlock(block_index); + for (int index = 0; index < block->capacity(); ++index) { + Record* rec = block->record(index); + callback(rec); + } + } + // Handle last block separately, as it is not filled to capacity. + const uint32_t max_index = IndexInBlock(size() - 1, last_block_index) + 1; + Block* block = blocks->LoadBlock(last_block_index); + for (uint32_t index = 0; index < max_index; ++index) { + Record* rec = block->record(index); + callback(rec); + } +} + +} // namespace internal +} // namespace v8 + +#include "src/objects/object-macros-undef.h" + +#endif // V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_ diff --git a/src/objects/string-forwarding-table.cc b/src/objects/string-forwarding-table.cc new file mode 100644 index 0000000000..bfc5e418fb --- /dev/null +++ b/src/objects/string-forwarding-table.cc @@ -0,0 +1,239 @@ +// Copyright 2022 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/objects/string-forwarding-table.h" + +#include "src/base/atomicops.h" +#include "src/common/globals.h" +#include "src/objects/objects-inl.h" +#include "src/objects/slots-inl.h" +#include "src/objects/slots.h" +#include "src/objects/string-forwarding-table-inl.h" +#include "src/utils/allocation.h" + +namespace v8 { +namespace internal { + +StringForwardingTable::Block::Block(int capacity) : capacity_(capacity) { + static_assert(unused_element().ptr() == 0); + static_assert(kNullAddress == 0); + static_assert(sizeof(Record) % sizeof(Address) == 0); + static_assert(offsetof(Record, original_string_) == 0); + constexpr int kRecordPointerSize = sizeof(Record) / sizeof(Address); + MemsetPointer(reinterpret_cast(&elements_[0]), 0, + capacity_ * kRecordPointerSize); +} + +void* StringForwardingTable::Block::operator new(size_t size, int capacity) { + // Make sure the size given is the size of the Block structure. + DCHECK_EQ(size, sizeof(StringForwardingTable::Block)); + // Make sure the Record class is trivial and has standard layout. + static_assert(std::is_trivial_v); + static_assert(std::is_standard_layout_v); + // Make sure that the elements_ array is at the end of Block, with no padding, + // so that subsequent elements can be accessed as offsets from elements_. + static_assert(offsetof(StringForwardingTable::Block, elements_) == + sizeof(StringForwardingTable::Block) - sizeof(Record)); + // Make sure that elements_ is aligned when StringTable::Block is aligned. + static_assert((alignof(StringForwardingTable::Block) + + offsetof(StringForwardingTable::Block, elements_)) % + kTaggedSize == + 0); + + const size_t elements_size = capacity * sizeof(Record); + // Storage for the first element is already supplied by elements_, so subtract + // sizeof(Record). + const size_t new_size = size + elements_size - sizeof(Record); + DCHECK_LE(alignof(StringForwardingTable::Block), kSystemPointerSize); + return AlignedAlloc(new_size, kSystemPointerSize); +} + +void StringForwardingTable::Block::operator delete(void* block) { + AlignedFree(block); +} + +std::unique_ptr StringForwardingTable::Block::New( + int capacity) { + return std::unique_ptr(new (capacity) Block(capacity)); +} + +void StringForwardingTable::Block::UpdateAfterEvacuation( + PtrComprCageBase cage_base) { + UpdateAfterEvacuation(cage_base, capacity_); +} + +void StringForwardingTable::Block::UpdateAfterEvacuation( + PtrComprCageBase cage_base, int up_to_index) { + // This is only used for Scavenger. + DCHECK(!FLAG_minor_mc); + DCHECK(FLAG_always_use_string_forwarding_table); + for (int index = 0; index < up_to_index; ++index) { + Object original = record(index)->OriginalStringObject(cage_base); + if (!original.IsHeapObject()) continue; + HeapObject object = HeapObject::cast(original); + if (Heap::InFromPage(object)) { + DCHECK(!object.InSharedWritableHeap()); + MapWord map_word = object.map_word(kRelaxedLoad); + if (map_word.IsForwardingAddress()) { + HeapObject forwarded_object = map_word.ToForwardingAddress(); + record(index)->set_original_string(forwarded_object); + } else { + record(index)->set_original_string(deleted_element()); + } + } else { + DCHECK(!object.map_word(kRelaxedLoad).IsForwardingAddress()); + } + } +} + +StringForwardingTable::BlockVector::BlockVector(size_t capacity) + : allocator_(Allocator()), capacity_(capacity), size_(0) { + begin_ = allocator_.allocate(capacity); +} + +StringForwardingTable::BlockVector::~BlockVector() { + allocator_.deallocate(begin_, capacity()); +} + +// static +std::unique_ptr +StringForwardingTable::BlockVector::Grow( + StringForwardingTable::BlockVector* data, size_t capacity, + const base::Mutex& mutex) { + mutex.AssertHeld(); + std::unique_ptr new_data = + std::make_unique(capacity); + // Copy pointers to blocks from the old to the new vector. + for (size_t i = 0; i < data->size(); i++) { + new_data->begin_[i] = data->LoadBlock(i); + } + new_data->size_ = data->size(); + return new_data; +} + +StringForwardingTable::StringForwardingTable(Isolate* isolate) + : isolate_(isolate), next_free_index_(0) { + InitializeBlockVector(); +} + +StringForwardingTable::~StringForwardingTable() { + BlockVector* blocks = blocks_.load(std::memory_order_relaxed); + for (uint32_t block_index = 0; block_index < blocks->size(); block_index++) { + delete blocks->LoadBlock(block_index); + } +} + +void StringForwardingTable::InitializeBlockVector() { + BlockVector* blocks = block_vector_storage_ + .emplace_back(std::make_unique( + kInitialBlockVectorCapacity)) + .get(); + blocks->AddBlock(Block::New(kInitialBlockSize)); + blocks_.store(blocks, std::memory_order_relaxed); +} + +StringForwardingTable::BlockVector* StringForwardingTable::EnsureCapacity( + uint32_t block_index) { + BlockVector* blocks = blocks_.load(std::memory_order_acquire); + if V8_UNLIKELY (block_index >= blocks->size()) { + base::MutexGuard table_grow_guard(&grow_mutex_); + // Reload the vector, as another thread could have grown it. + blocks = blocks_.load(std::memory_order_relaxed); + // Check again if we need to grow under lock. + if (block_index >= blocks->size()) { + // Grow the vector if the block to insert is greater than the vectors + // capacity. + if (block_index >= blocks->capacity()) { + std::unique_ptr new_blocks = + BlockVector::Grow(blocks, blocks->capacity() * 2, grow_mutex_); + block_vector_storage_.push_back(std::move(new_blocks)); + blocks = block_vector_storage_.back().get(); + blocks_.store(blocks, std::memory_order_release); + } + const uint32_t capacity = CapacityForBlock(block_index); + std::unique_ptr new_block = Block::New(capacity); + blocks->AddBlock(std::move(new_block)); + } + } + return blocks; +} + +int StringForwardingTable::AddForwardString(String string, String forward_to) { + DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table, + string.InSharedHeap()); + DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table, + forward_to.InSharedHeap()); + int index = next_free_index_++; + uint32_t index_in_block; + const uint32_t block_index = BlockForIndex(index, &index_in_block); + + BlockVector* blocks = EnsureCapacity(block_index); + Block* block = blocks->LoadBlock(block_index, kAcquireLoad); + block->record(index_in_block)->SetInternalized(string, forward_to); + return index; +} + +String StringForwardingTable::GetForwardString(PtrComprCageBase cage_base, + int index) const { + CHECK_LT(index, size()); + uint32_t index_in_block; + const uint32_t block_index = BlockForIndex(index, &index_in_block); + Block* block = blocks_.load(std::memory_order_acquire) + ->LoadBlock(block_index, kAcquireLoad); + return block->record(index_in_block)->forward_string(cage_base); +} + +// static +Address StringForwardingTable::GetForwardStringAddress(Isolate* isolate, + int index) { + return isolate->string_forwarding_table() + ->GetForwardString(isolate, index) + .ptr(); +} + +uint32_t StringForwardingTable::GetRawHash(PtrComprCageBase cage_base, + int index) const { + CHECK_LT(index, size()); + uint32_t index_in_block; + const uint32_t block_index = BlockForIndex(index, &index_in_block); + Block* block = blocks_.load(std::memory_order_acquire) + ->LoadBlock(block_index, kAcquireLoad); + return block->record(index_in_block)->raw_hash(cage_base); +} + +void StringForwardingTable::Reset() { + isolate_->heap()->safepoint()->AssertActive(); + DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC); + + BlockVector* blocks = blocks_.load(std::memory_order_relaxed); + for (uint32_t block_index = 0; block_index < blocks->size(); ++block_index) { + delete blocks->LoadBlock(block_index); + } + + block_vector_storage_.clear(); + InitializeBlockVector(); + next_free_index_ = 0; +} + +void StringForwardingTable::UpdateAfterEvacuation() { + DCHECK(FLAG_always_use_string_forwarding_table); + + if (empty()) return; + + BlockVector* blocks = blocks_.load(std::memory_order_relaxed); + const unsigned int last_block_index = + static_cast(blocks->size() - 1); + for (unsigned int block_index = 0; block_index < last_block_index; + ++block_index) { + Block* block = blocks->LoadBlock(block_index, kAcquireLoad); + block->UpdateAfterEvacuation(isolate_); + } + // Handle last block separately, as it is not filled to capacity. + const int max_index = IndexInBlock(size() - 1, last_block_index) + 1; + blocks->LoadBlock(last_block_index, kAcquireLoad) + ->UpdateAfterEvacuation(isolate_, max_index); +} + +} // namespace internal +} // namespace v8 diff --git a/src/objects/string-forwarding-table.h b/src/objects/string-forwarding-table.h new file mode 100644 index 0000000000..6eed6dbb6b --- /dev/null +++ b/src/objects/string-forwarding-table.h @@ -0,0 +1,87 @@ +// Copyright 2022 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_H_ +#define V8_OBJECTS_STRING_FORWARDING_TABLE_H_ + +#include "src/objects/string.h" + +// Has to be the last include (doesn't have include guards): +#include "src/objects/object-macros.h" + +namespace v8 { +namespace internal { + +// Mapping from forwarding indices (stored in a string's hash field) to +// internalized strings. +// The table is organised in "blocks". As writes only append new entries, the +// organisation in blocks allows lock-free writes. We need a lock only for +// growing the table (adding more blocks). When the vector holding the blocks +// needs to grow, we keep a copy of the old vector alive to allow concurrent +// reads while the vector is relocated. +class StringForwardingTable { + public: + // Capacity for the first block. + static constexpr int kInitialBlockSize = 16; + static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize)); + static constexpr int kInitialBlockSizeHighestBit = + kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1; + // Initial capacity in the block vector. + static constexpr int kInitialBlockVectorCapacity = 4; + static constexpr Smi unused_element() { return Smi::FromInt(0); } + static constexpr Smi deleted_element() { return Smi::FromInt(1); } + + explicit StringForwardingTable(Isolate* isolate); + ~StringForwardingTable(); + + inline int size() const; + inline bool empty() const; + // Returns the index of the added record. + int AddForwardString(String string, String forward_to); + String GetForwardString(PtrComprCageBase cage_base, int index) const; + static Address GetForwardStringAddress(Isolate* isolate, int index); + V8_EXPORT_PRIVATE uint32_t GetRawHash(PtrComprCageBase cage_base, + int index) const; + template + V8_INLINE void IterateElements(Isolate* isolate, Func&& callback); + void Reset(); + void UpdateAfterEvacuation(); + + class Record; + + private: + class Block; + class BlockVector; + + // Returns the block for a given index and sets the index within this block + // as out parameter. + static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out); + static inline uint32_t IndexInBlock(int index, uint32_t block); + static inline uint32_t CapacityForBlock(uint32_t block); + + void InitializeBlockVector(); + // Ensure that |block| exists in the BlockVector already. If not, a new block + // is created (with capacity double the capacity of the last block) and + // inserted into the BlockVector. The BlockVector itself might grow (to double + // the capacity). + BlockVector* EnsureCapacity(uint32_t block); + + Isolate* isolate_; + std::atomic blocks_; + // We need a vector of BlockVectors to keep old BlockVectors alive when we + // grow the table, due to concurrent reads that may still hold a pointer to + // them. |block_vector_sotrage_| is only accessed while we grow with the mutex + // held. All regular access go through |block_|, which holds a pointer to the + // current BlockVector. + std::vector> block_vector_storage_; + std::atomic next_free_index_; + base::Mutex grow_mutex_; +}; + +} // namespace internal +} // namespace v8 + +#include "src/objects/object-macros-undef.h" + +#endif // V8_OBJECTS_STRING_FORWARDING_TABLE_H_ diff --git a/src/objects/string-table-inl.h b/src/objects/string-table-inl.h index e8f49386c1..511a821216 100644 --- a/src/objects/string-table-inl.h +++ b/src/objects/string-table-inl.h @@ -24,37 +24,6 @@ uint32_t StringTableKey::hash() const { return Name::HashBits::decode(raw_hash_field_); } -int StringForwardingTable::Size() const { return next_free_index_; } - -// static -uint32_t StringForwardingTable::BlockForIndex(int index, - uint32_t* index_in_block) { - DCHECK_GE(index, 0); - DCHECK_NOT_NULL(index_in_block); - // The block is the leftmost set bit of the index, corrected by the size - // of the first block. - const uint32_t block = kBitsPerInt - - base::bits::CountLeadingZeros( - static_cast(index + kInitialBlockSize)) - - kInitialBlockSizeHighestBit - 1; - *index_in_block = IndexInBlock(index, block); - return block; -} - -// static -uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block) { - DCHECK_GE(index, 0); - // Clear out the leftmost set bit (the block) to get the index within the - // block. - return static_cast(index + kInitialBlockSize) & - ~(1u << (block + kInitialBlockSizeHighestBit)); -} - -// static -uint32_t StringForwardingTable::CapacityForBlock(uint32_t block) { - return 1u << (block + kInitialBlockSizeHighestBit); -} - } // namespace internal } // namespace v8 diff --git a/src/objects/string-table.cc b/src/objects/string-table.cc index 371306b22d..1768c2423c 100644 --- a/src/objects/string-table.cc +++ b/src/objects/string-table.cc @@ -448,7 +448,8 @@ void SetInternalizedReference(Isolate* isolate, String string, if (Name::IsInternalizedForwardingIndex(field)) return; const int forwarding_index = - isolate->string_forwarding_table()->Add(isolate, string, internalized); + isolate->string_forwarding_table()->AddForwardString(string, + internalized); string.set_raw_hash_field( String::CreateInternalizedForwardingIndex(forwarding_index), kReleaseStore); @@ -830,337 +831,5 @@ void StringTable::NotifyElementsRemoved(int count) { data_.load(std::memory_order_relaxed)->ElementsRemoved(count); } -class StringForwardingTable::Block { - public: - static std::unique_ptr New(int capacity); - explicit Block(int capacity); - int capacity() const { return capacity_; } - void* operator new(size_t size, int capacity); - void* operator new(size_t size) = delete; - void operator delete(void* data); - - void Set(int index, String string, String forward_to) { - DCHECK_LT(index, capacity()); - Set(IndexOfOriginalString(index), string); - Set(IndexOfForwardString(index), forward_to); - } - - String GetOriginalString(Isolate* isolate, int index) const { - DCHECK_LT(index, capacity()); - return String::cast(Get(isolate, IndexOfOriginalString(index))); - } - - String GetForwardString(Isolate* isolate, int index) const { - DCHECK_LT(index, capacity()); - return String::cast(Get(isolate, IndexOfForwardString(index))); - } - - uint32_t GetRawHash(Isolate* isolate, int index) const { - DCHECK_LT(index, capacity()); - String internalized = GetForwardString(isolate, index); - uint32_t raw_hash = internalized.raw_hash_field(); - DCHECK(Name::IsHashFieldComputed(raw_hash)); - return raw_hash; - } - - void IterateElements(RootVisitor* visitor, int up_to_index) { - OffHeapObjectSlot first_slot = slot(0); - OffHeapObjectSlot end_slot = slot(IndexOfOriginalString(up_to_index)); - visitor->VisitRootPointers(Root::kStringForwardingTable, nullptr, - first_slot, end_slot); - } - void UpdateAfterEvacuation(Isolate* isolate); - void UpdateAfterEvacuation(Isolate* isolate, int up_to_index); - - private: - static constexpr int kRecordSize = 2; - static constexpr int kOriginalStringOffset = 0; - static constexpr int kForwardStringOffset = 1; - - int IndexOfOriginalString(int index) const { - return index * kRecordSize + kOriginalStringOffset; - } - - int IndexOfForwardString(int index) const { - return index * kRecordSize + kForwardStringOffset; - } - - OffHeapObjectSlot slot(int index) const { - return OffHeapObjectSlot(&elements_[index]); - } - - Object Get(PtrComprCageBase cage_base, int internal_index) const { - return slot(internal_index).Acquire_Load(cage_base); - } - - void Set(int internal_index, Object object) { - slot(internal_index).Release_Store(object); - } - - const int capacity_; - Tagged_t elements_[1]; -}; - -StringForwardingTable::Block::Block(int capacity) : capacity_(capacity) {} - -void* StringForwardingTable::Block::operator new(size_t size, int capacity) { - // Make sure the size given is the size of the Block structure. - DCHECK_EQ(size, sizeof(StringForwardingTable::Block)); - // Make sure that the elements_ array is at the end of Block, with no padding, - // so that subsequent elements can be accessed as offsets from elements_. - static_assert(offsetof(StringForwardingTable::Block, elements_) == - sizeof(StringForwardingTable::Block) - sizeof(Tagged_t) * 1); - // Make sure that elements_ is aligned when StringTable::Block is aligned. - static_assert((alignof(StringForwardingTable::Block) + - offsetof(StringForwardingTable::Block, elements_)) % - kTaggedSize == - 0); - - const size_t elements_size = capacity * kRecordSize * sizeof(Tagged_t); - // Storage for the first element is already supplied by elements_, so subtract - // sizeof(Tagged_t). - const size_t new_size = size + elements_size - sizeof(Tagged_t); - DCHECK_LE(alignof(StringForwardingTable::Block), kSystemPointerSize); - return AlignedAlloc(new_size, kSystemPointerSize); -} - -void StringForwardingTable::Block::operator delete(void* block) { - AlignedFree(block); -} - -std::unique_ptr StringForwardingTable::Block::New( - int capacity) { - return std::unique_ptr(new (capacity) Block(capacity)); -} - -void StringForwardingTable::Block::UpdateAfterEvacuation(Isolate* isolate) { - UpdateAfterEvacuation(isolate, capacity_); -} - -void StringForwardingTable::Block::UpdateAfterEvacuation(Isolate* isolate, - int up_to_index) { - // This is only used for Scavenger. - DCHECK(!FLAG_minor_mc); - DCHECK(FLAG_always_use_string_forwarding_table); - for (int index = 0; index < up_to_index; ++index) { - Object original = Get(isolate, IndexOfOriginalString(index)); - if (!original.IsHeapObject()) continue; - HeapObject object = HeapObject::cast(original); - if (Heap::InFromPage(object)) { - DCHECK(!object.InSharedWritableHeap()); - MapWord map_word = object.map_word(kRelaxedLoad); - if (map_word.IsForwardingAddress()) { - HeapObject forwarded_object = map_word.ToForwardingAddress(); - Set(IndexOfOriginalString(index), String::cast(forwarded_object)); - } else { - Set(IndexOfOriginalString(index), deleted_element()); - } - } else { - DCHECK(!object.map_word(kRelaxedLoad).IsForwardingAddress()); - } - } -} - -class StringForwardingTable::BlockVector { - public: - using Block = StringForwardingTable::Block; - using Allocator = std::allocator; - - explicit BlockVector(size_t capacity); - ~BlockVector(); - size_t capacity() const { return capacity_; } - - Block* LoadBlock(size_t index, AcquireLoadTag) { - DCHECK_LT(index, size()); - return base::AsAtomicPointer::Acquire_Load(&begin_[index]); - } - - Block* LoadBlock(size_t index) { - DCHECK_LT(index, size()); - return begin_[index]; - } - - void AddBlock(std::unique_ptr block) { - DCHECK_LT(size(), capacity()); - base::AsAtomicPointer::Release_Store(&begin_[size_], block.release()); - size_++; - } - - static std::unique_ptr Grow(BlockVector* data, size_t capacity, - const base::Mutex& mutex); - - size_t size() const { return size_; } - - private: - V8_NO_UNIQUE_ADDRESS Allocator allocator_; - const size_t capacity_; - std::atomic size_; - Block** begin_; -}; - -StringForwardingTable::BlockVector::BlockVector(size_t capacity) - : allocator_(Allocator()), capacity_(capacity), size_(0) { - begin_ = allocator_.allocate(capacity); -} - -StringForwardingTable::BlockVector::~BlockVector() { - allocator_.deallocate(begin_, capacity()); -} - -// static -std::unique_ptr -StringForwardingTable::BlockVector::Grow( - StringForwardingTable::BlockVector* data, size_t capacity, - const base::Mutex& mutex) { - mutex.AssertHeld(); - std::unique_ptr new_data = - std::make_unique(capacity); - // Copy pointers to blocks from the old to the new vector. - for (size_t i = 0; i < data->size(); i++) { - new_data->begin_[i] = data->LoadBlock(i); - } - new_data->size_ = data->size(); - return new_data; -} - -StringForwardingTable::StringForwardingTable(Isolate* isolate) - : isolate_(isolate), next_free_index_(0) { - InitializeBlockVector(); -} - -StringForwardingTable::~StringForwardingTable() { - BlockVector* blocks = blocks_.load(std::memory_order_relaxed); - for (uint32_t block = 0; block < blocks->size(); block++) { - delete blocks->LoadBlock(block); - } -} - -void StringForwardingTable::InitializeBlockVector() { - BlockVector* blocks = block_vector_storage_ - .emplace_back(std::make_unique( - kInitialBlockVectorCapacity)) - .get(); - blocks->AddBlock(Block::New(kInitialBlockSize)); - blocks_.store(blocks, std::memory_order_relaxed); -} - -StringForwardingTable::BlockVector* StringForwardingTable::EnsureCapacity( - uint32_t block) { - BlockVector* blocks = blocks_.load(std::memory_order_acquire); - if V8_UNLIKELY (block >= blocks->size()) { - base::MutexGuard table_grow_guard(&grow_mutex_); - // Reload the vector, as another thread could have grown it. - blocks = blocks_.load(std::memory_order_relaxed); - // Check again if we need to grow under lock. - if (block >= blocks->size()) { - const uint32_t capacity = CapacityForBlock(block); - std::unique_ptr new_block = Block::New(capacity); - // Grow the vector if the block to insert is greater than the vectors - // capacity. - if (block >= blocks->capacity()) { - std::unique_ptr new_blocks = - BlockVector::Grow(blocks, blocks->capacity() * 2, grow_mutex_); - block_vector_storage_.push_back(std::move(new_blocks)); - blocks = block_vector_storage_.back().get(); - blocks_.store(blocks, std::memory_order_release); - } - blocks->AddBlock(std::move(new_block)); - } - } - return blocks; -} - -int StringForwardingTable::Add(Isolate* isolate, String string, - String forward_to) { - DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table, - string.InSharedHeap()); - DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table, - forward_to.InSharedHeap()); - int index = next_free_index_++; - uint32_t index_in_block; - const uint32_t block = BlockForIndex(index, &index_in_block); - - BlockVector* blocks = EnsureCapacity(block); - Block* data = blocks->LoadBlock(block, kAcquireLoad); - data->Set(index_in_block, string, forward_to); - return index; -} - -String StringForwardingTable::GetForwardString(Isolate* isolate, - int index) const { - CHECK_LT(index, Size()); - uint32_t index_in_block; - const uint32_t block = BlockForIndex(index, &index_in_block); - Block* data = - blocks_.load(std::memory_order_acquire)->LoadBlock(block, kAcquireLoad); - return data->GetForwardString(isolate, index_in_block); -} - -// static -Address StringForwardingTable::GetForwardStringAddress(Isolate* isolate, - int index) { - return isolate->string_forwarding_table() - ->GetForwardString(isolate, index) - .ptr(); -} - -uint32_t StringForwardingTable::GetRawHash(Isolate* isolate, int index) const { - CHECK_LT(index, Size()); - uint32_t index_in_block; - const uint32_t block = BlockForIndex(index, &index_in_block); - Block* data = - blocks_.load(std::memory_order_acquire)->LoadBlock(block, kAcquireLoad); - return data->GetRawHash(isolate, index_in_block); -} - -void StringForwardingTable::IterateElements(RootVisitor* visitor) { - isolate_->heap()->safepoint()->AssertActive(); - DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC); - - if (next_free_index_ == 0) return; // Early exit if table is empty. - - BlockVector* blocks = blocks_.load(std::memory_order_relaxed); - const uint32_t last_block = static_cast(blocks->size() - 1); - for (uint32_t block = 0; block < last_block; ++block) { - Block* data = blocks->LoadBlock(block); - data->IterateElements(visitor, data->capacity()); - } - // Handle last block separately, as it is not filled to capacity. - const uint32_t max_index = IndexInBlock(next_free_index_ - 1, last_block) + 1; - Block* data = blocks->LoadBlock(last_block); - data->IterateElements(visitor, max_index); -} - -void StringForwardingTable::Reset() { - isolate_->heap()->safepoint()->AssertActive(); - DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC); - - BlockVector* blocks = blocks_.load(std::memory_order_relaxed); - for (uint32_t block = 0; block < blocks->size(); ++block) { - delete blocks->LoadBlock(block); - } - - block_vector_storage_.clear(); - InitializeBlockVector(); - next_free_index_ = 0; -} - -void StringForwardingTable::UpdateAfterEvacuation() { - DCHECK(FLAG_always_use_string_forwarding_table); - - if (next_free_index_ == 0) return; // Early exit if table is empty. - - BlockVector* blocks = blocks_.load(std::memory_order_relaxed); - const unsigned int last_block = static_cast(blocks->size() - 1); - for (unsigned int block = 0; block < last_block; ++block) { - Block* data = blocks->LoadBlock(block, kAcquireLoad); - data->UpdateAfterEvacuation(isolate_); - } - // Handle last block separately, as it is not filled to capacity. - const int max_index = IndexInBlock(next_free_index_ - 1, last_block) + 1; - blocks->LoadBlock(last_block, kAcquireLoad) - ->UpdateAfterEvacuation(isolate_, max_index); -} - } // namespace internal } // namespace v8 diff --git a/src/objects/string-table.h b/src/objects/string-table.h index 2ca3780d00..9c1f0d1a97 100644 --- a/src/objects/string-table.h +++ b/src/objects/string-table.h @@ -95,66 +95,6 @@ class V8_EXPORT_PRIVATE StringTable { Isolate* isolate_; }; -// Mapping from forwarding indices (stored in a string's hash field) to -// internalized strings. -// The table is organised in "blocks". As writes only append new entries, the -// organisation in blocks allows lock-free writes. We need a lock only for -// growing the table (adding more blocks). When the vector holding the blocks -// needs to grow, we keep a copy of the old vector alive to allow concurrent -// reads while the vector is relocated. -class StringForwardingTable { - public: - // Capacity for the first block. - static constexpr int kInitialBlockSize = 16; - static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize)); - static constexpr int kInitialBlockSizeHighestBit = - kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1; - // Initial capacity in the block vector. - static constexpr int kInitialBlockVectorCapacity = 4; - static constexpr Smi deleted_element() { return Smi::FromInt(0); } - - explicit StringForwardingTable(Isolate* isolate); - ~StringForwardingTable(); - - inline int Size() const; - // Returns the index of the added string pair. - int Add(Isolate* isolate, String string, String forward_to); - String GetForwardString(Isolate* isolate, int index) const; - static Address GetForwardStringAddress(Isolate* isolate, int index); - V8_EXPORT_PRIVATE uint32_t GetRawHash(Isolate* isolate, int index) const; - void IterateElements(RootVisitor* visitor); - void Reset(); - void UpdateAfterEvacuation(); - - private: - class Block; - class BlockVector; - - // Returns the block for a given index and sets the index within this block - // as out parameter. - static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out); - static inline uint32_t IndexInBlock(int index, uint32_t block); - static inline uint32_t CapacityForBlock(uint32_t block); - - void InitializeBlockVector(); - // Ensure that |block| exists in the BlockVector already. If not, a new block - // is created (with capacity double the capacity of the last block) and - // inserted into the BlockVector. The BlockVector itself might grow (to double - // the capacity). - BlockVector* EnsureCapacity(uint32_t block); - - Isolate* isolate_; - std::atomic blocks_; - // We need a vector of BlockVectors to keep old BlockVectors alive when we - // grow the table, due to concurrent reads that may still hold a pointer to - // them. |block_vector_sotrage_| is only accessed while we grow with the mutex - // held. All regular access go through |block_|, which holds a pointer to the - // current BlockVector. - std::vector> block_vector_storage_; - std::atomic next_free_index_; - base::Mutex grow_mutex_; -}; - } // namespace internal } // namespace v8 diff --git a/src/objects/visitors.h b/src/objects/visitors.h index ea4525d2d9..52c8114ff1 100644 --- a/src/objects/visitors.h +++ b/src/objects/visitors.h @@ -18,7 +18,6 @@ class CodeDataContainer; #define ROOT_ID_LIST(V) \ V(kStringTable, "(Internalized strings)") \ - V(kStringForwardingTable, "(Forwarded strings)") \ V(kExternalStringsTable, "(External strings)") \ V(kReadOnlyRootList, "(Read-only roots)") \ V(kStrongRootList, "(Strong roots)") \ diff --git a/test/cctest/test-shared-strings.cc b/test/cctest/test-shared-strings.cc index ac7d2f523e..671b1dd57f 100644 --- a/test/cctest/test-shared-strings.cc +++ b/test/cctest/test-shared-strings.cc @@ -13,6 +13,7 @@ #include "src/heap/remembered-set.h" #include "src/objects/fixed-array.h" #include "src/objects/objects-inl.h" +#include "src/objects/string-forwarding-table-inl.h" #include "test/cctest/cctest.h" #include "test/cctest/heap/heap-utils.h" @@ -901,7 +902,7 @@ UNINITIALIZED_TEST(SharedStringsTransitionDuringGC) { i_isolate->heap()->CollectSharedGarbage(GarbageCollectionReason::kTesting); // Check that GC cleared the forwarding table. - CHECK_EQ(i_isolate->string_forwarding_table()->Size(), 0); + CHECK_EQ(i_isolate->string_forwarding_table()->size(), 0); // Check all strings are transitioned to ThinStrings for (int i = 0; i < shared_strings->length(); i++) {