AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

Prepare StringForwardingTable for external strings

Browse Source 
- Move StringForwardingTable implementation to own compilation unit.
- Refactoring preparing for layout change (Introduce explicit record
  class to make transition from contiguous Tagged_t fields to a
  heterogeneous record layout easier).
- Replace RootVisitor pattern for transitioning/cleanup during GC with
  callback.
- Minor cleanups.

Bug: v8:12957
Change-Id: Iae343393f470130eac0c54148a1303b67fb95aa4
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3845635
Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Commit-Queue: Patrick Thier <pthier@chromium.org>
Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82730}
This commit is contained in:
Patrick Thier 2022-08-25 14:36:06 +00:00 committed by V8 LUCI CQ
parent a31e8f242f
commit 348adb07ca
13 changed files with 604 additions and 490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
BUILD.bazel
View File

@ -1855,6 +1855,9 @@ filegroup(
"src/objects/string-inl.h",
"src/objects/string-set-inl.h",
"src/objects/string-set.h",
"src/objects/string-forwarding-table-inl.h",
"src/objects/string-forwarding-table.cc",
"src/objects/string-forwarding-table.h",
"src/objects/string-table-inl.h",
"src/objects/string-table.cc",
"src/objects/symbol-table.cc",

3
BUILD.gn
View File

@ -3410,6 +3410,8 @@ v8_header_set("v8_internal_headers") {
"src/objects/source-text-module-inl.h",
"src/objects/source-text-module.h",
"src/objects/string-comparator.h",
"src/objects/string-forwarding-table-inl.h",
"src/objects/string-forwarding-table.h",
"src/objects/string-inl.h",
"src/objects/string-set-inl.h",
"src/objects/string-set.h",
@ -4637,6 +4639,7 @@ v8_source_set("v8_base_without_compiler") {
"src/objects/simd.cc",
"src/objects/source-text-module.cc",
"src/objects/string-comparator.cc",
"src/objects/string-forwarding-table.cc",
"src/objects/string-table.cc",
"src/objects/string.cc",
"src/objects/swiss-name-dictionary.cc",

122
src/heap/mark-compact.cc
View File

@ -66,6 +66,7 @@
#include "src/objects/objects.h"
#include "src/objects/slots-inl.h"
#include "src/objects/smi.h"
#include "src/objects/string-forwarding-table-inl.h"
#include "src/objects/transitions-inl.h"
#include "src/objects/visitors.h"
#include "src/snapshot/shared-heap-serializer.h"
@ -1436,64 +1437,6 @@ class InternalizedStringTableCleaner final : public RootVisitor {
int pointers_removed_ = 0;
};
class StringForwardingTableCleaner final : public RootVisitor {
public:
explicit StringForwardingTableCleaner(Heap* heap) : heap_(heap) {}
void VisitRootPointers(Root root, const char* description,
FullObjectSlot start, FullObjectSlot end) override {
UNREACHABLE();
}
void VisitRootPointers(Root root, const char* description,
OffHeapObjectSlot start,
OffHeapObjectSlot end) override {
DCHECK_EQ(root, Root::kStringForwardingTable);
// Visit all HeapObject pointers in [start, end).
// The forwarding table is organized in pairs of [orig string, forward
// string].
auto* marking_state =
heap_->mark_compact_collector()->non_atomic_marking_state();
Isolate* isolate = heap_->isolate();
for (OffHeapObjectSlot p = start; p < end; p += 2) {
Object original = p.load(isolate);
if (!original.IsHeapObject()) {
// Only if we always use the forwarding table, the string could be a
// smi, indicating that the entry died during scavenge.
DCHECK(FLAG_always_use_string_forwarding_table);
DCHECK_EQ(original, StringForwardingTable::deleted_element());
continue;
}
if (marking_state->IsBlack(HeapObject::cast(original))) {
String original_string = String::cast(original);
// Check if the string was already transitioned. This happens if we have
// multiple entries for the same string in the table.
if (original_string.IsThinString()) continue;
// The second slot of each record is the forward string.
Object forward = (p + 1).load(isolate);
String forward_string = String::cast(forward);
// Mark the forwarded string.
marking_state->WhiteToBlack(forward_string);
// Transition the original string to a ThinString and override the
// forwarding index with the correct hash.
original_string.MakeThin(isolate, forward_string);
original_string.set_raw_hash_field(forward_string.raw_hash_field());
// Record the slot in the old-to-old remembered set. This is required as
// the internalized string could be relocated during compaction.
ObjectSlot slot = ThinString::cast(original_string)
.RawField(ThinString::kActualOffset);
MarkCompactCollector::RecordSlot(original_string, slot, forward_string);
}
}
}
private:
Heap* heap_;
};
class ExternalStringTableCleaner : public RootVisitor {
public:
explicit ExternalStringTableCleaner(Heap* heap) : heap_(heap) {}
@ -2866,6 +2809,62 @@ class ClearStringTableJobItem final : public ParallelClearingJob::ClearingItem {
Isolate* const isolate_;
};
class StringForwardingTableCleaner final {
public:
explicit StringForwardingTableCleaner(Heap* heap)
: heap_(heap),
isolate_(heap_->isolate()),
marking_state_(
heap_->mark_compact_collector()->non_atomic_marking_state()) {}
void Run() {
StringForwardingTable* forwarding_table =
isolate_->string_forwarding_table();
forwarding_table->IterateElements(
isolate_, [&](StringForwardingTable::Record* record) {
TransitionStrings(record);
});
forwarding_table->Reset();
}
private:
void TransitionStrings(StringForwardingTable::Record* record) {
Object original = record->OriginalStringObject(isolate_);
if (!original.IsHeapObject()) {
// Only if we always use the forwarding table, the string could be a
// smi, indicating that the entry died during scavenge.
DCHECK(FLAG_always_use_string_forwarding_table);
DCHECK_EQ(original, StringForwardingTable::deleted_element());
return;
}
if (marking_state_->IsBlack(HeapObject::cast(original))) {
String original_string = String::cast(original);
TryInternalize(original_string, record);
original_string.set_raw_hash_field(record->raw_hash(isolate_));
}
}
void TryInternalize(String original_string,
StringForwardingTable::Record* record) {
if (original_string.IsThinString()) return;
String forward_string = record->forward_string(isolate_);
// Mark the forwarded string to keep it alive.
marking_state_->WhiteToBlack(forward_string);
// Transition the original string to a ThinString and override the
// forwarding index with the correct hash.
original_string.MakeThin(isolate_, forward_string);
// Record the slot in the old-to-old remembered set. This is
// required as the internalized string could be relocated during
// compaction.
ObjectSlot slot =
ThinString::cast(original_string).RawField(ThinString::kActualOffset);
MarkCompactCollector::RecordSlot(original_string, slot, forward_string);
}
Heap* heap_;
Isolate* isolate_;
NonAtomicMarkingState* marking_state_;
};
} // namespace
void MarkCompactCollector::ClearNonLiveReferences() {
@ -2879,11 +2878,8 @@ void MarkCompactCollector::ClearNonLiveReferences() {
// Clearing the string forwarding table must happen before clearing the
// string table, as entries in the forwarding table can keep internalized
// strings alive.
StringForwardingTable* forwarding_table =
isolate()->string_forwarding_table();
StringForwardingTableCleaner visitor(heap());
forwarding_table->IterateElements(&visitor);
forwarding_table->Reset();
StringForwardingTableCleaner forwarding_table_cleaner(heap());
forwarding_table_cleaner.Run();
}
auto clearing_job = std::make_unique<ParallelClearingJob>();

1
src/objects/all-objects-inl.h
View File

@ -76,6 +76,7 @@
#include "src/objects/shared-function-info-inl.h"
#include "src/objects/slots-atomic-inl.h"
#include "src/objects/slots-inl.h"
#include "src/objects/string-forwarding-table-inl.h"
#include "src/objects/string-inl.h"
#include "src/objects/string-set-inl.h"
#include "src/objects/string-table-inl.h"

2
src/objects/name-inl.h
View File

@ -10,8 +10,8 @@
#include "src/objects/map-inl.h"
#include "src/objects/name.h"
#include "src/objects/primitive-heap-object-inl.h"
#include "src/objects/string-forwarding-table.h"
#include "src/objects/string-inl.h"
#include "src/objects/string-table.h"
// Has to be the last include (doesn't have include guards):
#include "src/objects/object-macros.h"

207
src/objects/string-forwarding-table-inl.h Normal file
View File

@ -0,0 +1,207 @@
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_
#define V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_
#include "src/base/atomicops.h"
#include "src/common/globals.h"
#include "src/objects/name-inl.h"
#include "src/objects/slots-inl.h"
#include "src/objects/slots.h"
#include "src/objects/string-forwarding-table.h"
#include "src/objects/string-inl.h"
// Has to be the last include (doesn't have include guards):
#include "src/objects/object-macros.h"
namespace v8 {
namespace internal {
class StringForwardingTable::Record final {
public:
String original_string(PtrComprCageBase cage_base) const {
return String::cast(OriginalStringObject(cage_base));
}
String forward_string(PtrComprCageBase cage_base) const {
return String::cast(ForwardStringObject(cage_base));
}
inline uint32_t raw_hash(PtrComprCageBase cage_base) const;
Object OriginalStringObject(PtrComprCageBase cage_base) const {
return OriginalStringSlot().Acquire_Load(cage_base);
}
Object ForwardStringObject(PtrComprCageBase cage_base) const {
return ForwardStringSlot().Acquire_Load(cage_base);
}
void set_original_string(Object object) {
OriginalStringSlot().Release_Store(object);
}
void set_forward_string(Object object) {
ForwardStringSlot().Release_Store(object);
}
inline void SetInternalized(String string, String forward_to);
private:
OffHeapObjectSlot OriginalStringSlot() const {
return OffHeapObjectSlot(&original_string_);
}
OffHeapObjectSlot ForwardStringSlot() const {
return OffHeapObjectSlot(&forward_string_);
}
Tagged_t original_string_;
Tagged_t forward_string_;
friend class StringForwardingTable::Block;
};
uint32_t StringForwardingTable::Record::raw_hash(
PtrComprCageBase cage_base) const {
String internalized = forward_string(cage_base);
uint32_t raw_hash = internalized.raw_hash_field();
DCHECK(Name::IsHashFieldComputed(raw_hash));
return raw_hash;
}
void StringForwardingTable::Record::SetInternalized(String string,
String forward_to) {
set_original_string(string);
set_forward_string(forward_to);
}
class StringForwardingTable::Block {
public:
static std::unique_ptr<Block> New(int capacity);
explicit Block(int capacity);
int capacity() const { return capacity_; }
void* operator new(size_t size, int capacity);
void* operator new(size_t size) = delete;
void operator delete(void* data);
Record* record(int index) {
DCHECK_LT(index, capacity());
return &elements_[index];
}
const Record* record(int index) const {
DCHECK_LT(index, capacity());
return &elements_[index];
}
void UpdateAfterEvacuation(PtrComprCageBase cage_base);
void UpdateAfterEvacuation(PtrComprCageBase cage_base, int up_to_index);
private:
const int capacity_;
Record elements_[1];
};
class StringForwardingTable::BlockVector {
public:
using Block = StringForwardingTable::Block;
using Allocator = std::allocator<Block*>;
explicit BlockVector(size_t capacity);
~BlockVector();
size_t capacity() const { return capacity_; }
Block* LoadBlock(size_t index, AcquireLoadTag) {
DCHECK_LT(index, size());
return base::AsAtomicPointer::Acquire_Load(&begin_[index]);
}
Block* LoadBlock(size_t index) {
DCHECK_LT(index, size());
return begin_[index];
}
void AddBlock(std::unique_ptr<Block> block) {
DCHECK_LT(size(), capacity());
base::AsAtomicPointer::Release_Store(&begin_[size_], block.release());
size_++;
}
static std::unique_ptr<BlockVector> Grow(BlockVector* data, size_t capacity,
const base::Mutex& mutex);
size_t size() const { return size_; }
private:
V8_NO_UNIQUE_ADDRESS Allocator allocator_;
const size_t capacity_;
std::atomic<size_t> size_;
Block** begin_;
};
int StringForwardingTable::size() const { return next_free_index_; }
bool StringForwardingTable::empty() const { return size() == 0; }
// static
uint32_t StringForwardingTable::BlockForIndex(int index,
uint32_t* index_in_block) {
DCHECK_GE(index, 0);
DCHECK_NOT_NULL(index_in_block);
// The block is the leftmost set bit of the index, corrected by the size
// of the first block.
const uint32_t block_index =
kBitsPerInt -
base::bits::CountLeadingZeros(
static_cast<uint32_t>(index + kInitialBlockSize)) -
kInitialBlockSizeHighestBit - 1;
*index_in_block = IndexInBlock(index, block_index);
return block_index;
}
// static
uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block_index) {
DCHECK_GE(index, 0);
// Clear out the leftmost set bit (the block index) to get the index within
// the block.
return static_cast<uint32_t>(index + kInitialBlockSize) &
~(1u << (block_index + kInitialBlockSizeHighestBit));
}
// static
uint32_t StringForwardingTable::CapacityForBlock(uint32_t block_index) {
return 1u << (block_index + kInitialBlockSizeHighestBit);
}
template <typename Func>
void StringForwardingTable::IterateElements(Isolate* isolate, Func&& callback) {
isolate->heap()->safepoint()->AssertActive();
DCHECK_NE(isolate->heap()->gc_state(), Heap::NOT_IN_GC);
if (empty()) return;
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
const uint32_t last_block_index = static_cast<uint32_t>(blocks->size() - 1);
for (uint32_t block_index = 0; block_index < last_block_index;
++block_index) {
Block* block = blocks->LoadBlock(block_index);
for (int index = 0; index < block->capacity(); ++index) {
Record* rec = block->record(index);
callback(rec);
}
}
// Handle last block separately, as it is not filled to capacity.
const uint32_t max_index = IndexInBlock(size() - 1, last_block_index) + 1;
Block* block = blocks->LoadBlock(last_block_index);
for (uint32_t index = 0; index < max_index; ++index) {
Record* rec = block->record(index);
callback(rec);
}
}
} // namespace internal
} // namespace v8
#include "src/objects/object-macros-undef.h"
#endif // V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_

239
src/objects/string-forwarding-table.cc Normal file
View File

@ -0,0 +1,239 @@
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/objects/string-forwarding-table.h"
#include "src/base/atomicops.h"
#include "src/common/globals.h"
#include "src/objects/objects-inl.h"
#include "src/objects/slots-inl.h"
#include "src/objects/slots.h"
#include "src/objects/string-forwarding-table-inl.h"
#include "src/utils/allocation.h"
namespace v8 {
namespace internal {
StringForwardingTable::Block::Block(int capacity) : capacity_(capacity) {
static_assert(unused_element().ptr() == 0);
static_assert(kNullAddress == 0);
static_assert(sizeof(Record) % sizeof(Address) == 0);
static_assert(offsetof(Record, original_string_) == 0);
constexpr int kRecordPointerSize = sizeof(Record) / sizeof(Address);
MemsetPointer(reinterpret_cast<Address*>(&elements_[0]), 0,
capacity_ * kRecordPointerSize);
}
void* StringForwardingTable::Block::operator new(size_t size, int capacity) {
// Make sure the size given is the size of the Block structure.
DCHECK_EQ(size, sizeof(StringForwardingTable::Block));
// Make sure the Record class is trivial and has standard layout.
static_assert(std::is_trivial_v<Record>);
static_assert(std::is_standard_layout_v<Record>);
// Make sure that the elements_ array is at the end of Block, with no padding,
// so that subsequent elements can be accessed as offsets from elements_.
static_assert(offsetof(StringForwardingTable::Block, elements_) ==
sizeof(StringForwardingTable::Block) - sizeof(Record));
// Make sure that elements_ is aligned when StringTable::Block is aligned.
static_assert((alignof(StringForwardingTable::Block) +
offsetof(StringForwardingTable::Block, elements_)) %
kTaggedSize ==
0);
const size_t elements_size = capacity * sizeof(Record);
// Storage for the first element is already supplied by elements_, so subtract
// sizeof(Record).
const size_t new_size = size + elements_size - sizeof(Record);
DCHECK_LE(alignof(StringForwardingTable::Block), kSystemPointerSize);
return AlignedAlloc(new_size, kSystemPointerSize);
}
void StringForwardingTable::Block::operator delete(void* block) {
AlignedFree(block);
}
std::unique_ptr<StringForwardingTable::Block> StringForwardingTable::Block::New(
int capacity) {
return std::unique_ptr<Block>(new (capacity) Block(capacity));
}
void StringForwardingTable::Block::UpdateAfterEvacuation(
PtrComprCageBase cage_base) {
UpdateAfterEvacuation(cage_base, capacity_);
}
void StringForwardingTable::Block::UpdateAfterEvacuation(
PtrComprCageBase cage_base, int up_to_index) {
// This is only used for Scavenger.
DCHECK(!FLAG_minor_mc);
DCHECK(FLAG_always_use_string_forwarding_table);
for (int index = 0; index < up_to_index; ++index) {
Object original = record(index)->OriginalStringObject(cage_base);
if (!original.IsHeapObject()) continue;
HeapObject object = HeapObject::cast(original);
if (Heap::InFromPage(object)) {
DCHECK(!object.InSharedWritableHeap());
MapWord map_word = object.map_word(kRelaxedLoad);
if (map_word.IsForwardingAddress()) {
HeapObject forwarded_object = map_word.ToForwardingAddress();
record(index)->set_original_string(forwarded_object);
} else {
record(index)->set_original_string(deleted_element());
}
} else {
DCHECK(!object.map_word(kRelaxedLoad).IsForwardingAddress());
}
}
}
StringForwardingTable::BlockVector::BlockVector(size_t capacity)
: allocator_(Allocator()), capacity_(capacity), size_(0) {
begin_ = allocator_.allocate(capacity);
}
StringForwardingTable::BlockVector::~BlockVector() {
allocator_.deallocate(begin_, capacity());
}
// static
std::unique_ptr<StringForwardingTable::BlockVector>
StringForwardingTable::BlockVector::Grow(
StringForwardingTable::BlockVector* data, size_t capacity,
const base::Mutex& mutex) {
mutex.AssertHeld();
std::unique_ptr<BlockVector> new_data =
std::make_unique<BlockVector>(capacity);
// Copy pointers to blocks from the old to the new vector.
for (size_t i = 0; i < data->size(); i++) {
new_data->begin_[i] = data->LoadBlock(i);
}
new_data->size_ = data->size();
return new_data;
}
StringForwardingTable::StringForwardingTable(Isolate* isolate)
: isolate_(isolate), next_free_index_(0) {
InitializeBlockVector();
}
StringForwardingTable::~StringForwardingTable() {
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
for (uint32_t block_index = 0; block_index < blocks->size(); block_index++) {
delete blocks->LoadBlock(block_index);
}
}
void StringForwardingTable::InitializeBlockVector() {
BlockVector* blocks = block_vector_storage_
.emplace_back(std::make_unique<BlockVector>(
kInitialBlockVectorCapacity))
.get();
blocks->AddBlock(Block::New(kInitialBlockSize));
blocks_.store(blocks, std::memory_order_relaxed);
}
StringForwardingTable::BlockVector* StringForwardingTable::EnsureCapacity(
uint32_t block_index) {
BlockVector* blocks = blocks_.load(std::memory_order_acquire);
if V8_UNLIKELY (block_index >= blocks->size()) {
base::MutexGuard table_grow_guard(&grow_mutex_);
// Reload the vector, as another thread could have grown it.
blocks = blocks_.load(std::memory_order_relaxed);
// Check again if we need to grow under lock.
if (block_index >= blocks->size()) {
// Grow the vector if the block to insert is greater than the vectors
// capacity.
if (block_index >= blocks->capacity()) {
std::unique_ptr<BlockVector> new_blocks =
BlockVector::Grow(blocks, blocks->capacity() * 2, grow_mutex_);
block_vector_storage_.push_back(std::move(new_blocks));
blocks = block_vector_storage_.back().get();
blocks_.store(blocks, std::memory_order_release);
}
const uint32_t capacity = CapacityForBlock(block_index);
std::unique_ptr<Block> new_block = Block::New(capacity);
blocks->AddBlock(std::move(new_block));
}
}
return blocks;
}
int StringForwardingTable::AddForwardString(String string, String forward_to) {
DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
string.InSharedHeap());
DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
forward_to.InSharedHeap());
int index = next_free_index_++;
uint32_t index_in_block;
const uint32_t block_index = BlockForIndex(index, &index_in_block);
BlockVector* blocks = EnsureCapacity(block_index);
Block* block = blocks->LoadBlock(block_index, kAcquireLoad);
block->record(index_in_block)->SetInternalized(string, forward_to);
return index;
}
String StringForwardingTable::GetForwardString(PtrComprCageBase cage_base,
int index) const {
CHECK_LT(index, size());
uint32_t index_in_block;
const uint32_t block_index = BlockForIndex(index, &index_in_block);
Block* block = blocks_.load(std::memory_order_acquire)
->LoadBlock(block_index, kAcquireLoad);
return block->record(index_in_block)->forward_string(cage_base);
}
// static
Address StringForwardingTable::GetForwardStringAddress(Isolate* isolate,
int index) {
return isolate->string_forwarding_table()
->GetForwardString(isolate, index)
.ptr();
}
uint32_t StringForwardingTable::GetRawHash(PtrComprCageBase cage_base,
int index) const {
CHECK_LT(index, size());
uint32_t index_in_block;
const uint32_t block_index = BlockForIndex(index, &index_in_block);
Block* block = blocks_.load(std::memory_order_acquire)
->LoadBlock(block_index, kAcquireLoad);
return block->record(index_in_block)->raw_hash(cage_base);
}
void StringForwardingTable::Reset() {
isolate_->heap()->safepoint()->AssertActive();
DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC);
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
for (uint32_t block_index = 0; block_index < blocks->size(); ++block_index) {
delete blocks->LoadBlock(block_index);
}
block_vector_storage_.clear();
InitializeBlockVector();
next_free_index_ = 0;
}
void StringForwardingTable::UpdateAfterEvacuation() {
DCHECK(FLAG_always_use_string_forwarding_table);
if (empty()) return;
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
const unsigned int last_block_index =
static_cast<unsigned int>(blocks->size() - 1);
for (unsigned int block_index = 0; block_index < last_block_index;
++block_index) {
Block* block = blocks->LoadBlock(block_index, kAcquireLoad);
block->UpdateAfterEvacuation(isolate_);
}
// Handle last block separately, as it is not filled to capacity.
const int max_index = IndexInBlock(size() - 1, last_block_index) + 1;
blocks->LoadBlock(last_block_index, kAcquireLoad)
->UpdateAfterEvacuation(isolate_, max_index);
}
} // namespace internal
} // namespace v8

87
src/objects/string-forwarding-table.h Normal file
View File

@ -0,0 +1,87 @@
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_H_
#define V8_OBJECTS_STRING_FORWARDING_TABLE_H_
#include "src/objects/string.h"
// Has to be the last include (doesn't have include guards):
#include "src/objects/object-macros.h"
namespace v8 {
namespace internal {
// Mapping from forwarding indices (stored in a string's hash field) to
// internalized strings.
// The table is organised in "blocks". As writes only append new entries, the
// organisation in blocks allows lock-free writes. We need a lock only for
// growing the table (adding more blocks). When the vector holding the blocks
// needs to grow, we keep a copy of the old vector alive to allow concurrent
// reads while the vector is relocated.
class StringForwardingTable {
public:
// Capacity for the first block.
static constexpr int kInitialBlockSize = 16;
static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize));
static constexpr int kInitialBlockSizeHighestBit =
kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1;
// Initial capacity in the block vector.
static constexpr int kInitialBlockVectorCapacity = 4;
static constexpr Smi unused_element() { return Smi::FromInt(0); }
static constexpr Smi deleted_element() { return Smi::FromInt(1); }
explicit StringForwardingTable(Isolate* isolate);
~StringForwardingTable();
inline int size() const;
inline bool empty() const;
// Returns the index of the added record.
int AddForwardString(String string, String forward_to);
String GetForwardString(PtrComprCageBase cage_base, int index) const;
static Address GetForwardStringAddress(Isolate* isolate, int index);
V8_EXPORT_PRIVATE uint32_t GetRawHash(PtrComprCageBase cage_base,
int index) const;
template <typename Func>
V8_INLINE void IterateElements(Isolate* isolate, Func&& callback);
void Reset();
void UpdateAfterEvacuation();
class Record;
private:
class Block;
class BlockVector;
// Returns the block for a given index and sets the index within this block
// as out parameter.
static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out);
static inline uint32_t IndexInBlock(int index, uint32_t block);
static inline uint32_t CapacityForBlock(uint32_t block);
void InitializeBlockVector();
// Ensure that |block| exists in the BlockVector already. If not, a new block
// is created (with capacity double the capacity of the last block) and
// inserted into the BlockVector. The BlockVector itself might grow (to double
// the capacity).
BlockVector* EnsureCapacity(uint32_t block);
Isolate* isolate_;
std::atomic<BlockVector*> blocks_;
// We need a vector of BlockVectors to keep old BlockVectors alive when we
// grow the table, due to concurrent reads that may still hold a pointer to
// them. |block_vector_sotrage_| is only accessed while we grow with the mutex
// held. All regular access go through |block_|, which holds a pointer to the
// current BlockVector.
std::vector<std::unique_ptr<BlockVector>> block_vector_storage_;
std::atomic<int> next_free_index_;
base::Mutex grow_mutex_;
};
} // namespace internal
} // namespace v8
#include "src/objects/object-macros-undef.h"
#endif // V8_OBJECTS_STRING_FORWARDING_TABLE_H_

31
src/objects/string-table-inl.h
View File

@ -24,37 +24,6 @@ uint32_t StringTableKey::hash() const {
return Name::HashBits::decode(raw_hash_field_);
}
int StringForwardingTable::Size() const { return next_free_index_; }
// static
uint32_t StringForwardingTable::BlockForIndex(int index,
uint32_t* index_in_block) {
DCHECK_GE(index, 0);
DCHECK_NOT_NULL(index_in_block);
// The block is the leftmost set bit of the index, corrected by the size
// of the first block.
const uint32_t block = kBitsPerInt -
base::bits::CountLeadingZeros(
static_cast<uint32_t>(index + kInitialBlockSize)) -
kInitialBlockSizeHighestBit - 1;
*index_in_block = IndexInBlock(index, block);
return block;
}
// static
uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block) {
DCHECK_GE(index, 0);
// Clear out the leftmost set bit (the block) to get the index within the
// block.
return static_cast<uint32_t>(index + kInitialBlockSize) &
~(1u << (block + kInitialBlockSizeHighestBit));
}
// static
uint32_t StringForwardingTable::CapacityForBlock(uint32_t block) {
return 1u << (block + kInitialBlockSizeHighestBit);
}
} // namespace internal
} // namespace v8

335
src/objects/string-table.cc
View File

@ -448,7 +448,8 @@ void SetInternalizedReference(Isolate* isolate, String string,
if (Name::IsInternalizedForwardingIndex(field)) return;
const int forwarding_index =
isolate->string_forwarding_table()->Add(isolate, string, internalized);
isolate->string_forwarding_table()->AddForwardString(string,
internalized);
string.set_raw_hash_field(
String::CreateInternalizedForwardingIndex(forwarding_index),
kReleaseStore);
@ -830,337 +831,5 @@ void StringTable::NotifyElementsRemoved(int count) {
data_.load(std::memory_order_relaxed)->ElementsRemoved(count);
}
class StringForwardingTable::Block {
public:
static std::unique_ptr<Block> New(int capacity);
explicit Block(int capacity);
int capacity() const { return capacity_; }
void* operator new(size_t size, int capacity);
void* operator new(size_t size) = delete;
void operator delete(void* data);
void Set(int index, String string, String forward_to) {
DCHECK_LT(index, capacity());
Set(IndexOfOriginalString(index), string);
Set(IndexOfForwardString(index), forward_to);
}
String GetOriginalString(Isolate* isolate, int index) const {
DCHECK_LT(index, capacity());
return String::cast(Get(isolate, IndexOfOriginalString(index)));
}
String GetForwardString(Isolate* isolate, int index) const {
DCHECK_LT(index, capacity());
return String::cast(Get(isolate, IndexOfForwardString(index)));
}
uint32_t GetRawHash(Isolate* isolate, int index) const {
DCHECK_LT(index, capacity());
String internalized = GetForwardString(isolate, index);
uint32_t raw_hash = internalized.raw_hash_field();
DCHECK(Name::IsHashFieldComputed(raw_hash));
return raw_hash;
}
void IterateElements(RootVisitor* visitor, int up_to_index) {
OffHeapObjectSlot first_slot = slot(0);
OffHeapObjectSlot end_slot = slot(IndexOfOriginalString(up_to_index));
visitor->VisitRootPointers(Root::kStringForwardingTable, nullptr,
first_slot, end_slot);
}
void UpdateAfterEvacuation(Isolate* isolate);
void UpdateAfterEvacuation(Isolate* isolate, int up_to_index);
private:
static constexpr int kRecordSize = 2;
static constexpr int kOriginalStringOffset = 0;
static constexpr int kForwardStringOffset = 1;
int IndexOfOriginalString(int index) const {
return index * kRecordSize + kOriginalStringOffset;
}
int IndexOfForwardString(int index) const {
return index * kRecordSize + kForwardStringOffset;
}
OffHeapObjectSlot slot(int index) const {
return OffHeapObjectSlot(&elements_[index]);
}
Object Get(PtrComprCageBase cage_base, int internal_index) const {
return slot(internal_index).Acquire_Load(cage_base);
}
void Set(int internal_index, Object object) {
slot(internal_index).Release_Store(object);
}
const int capacity_;
Tagged_t elements_[1];
};
StringForwardingTable::Block::Block(int capacity) : capacity_(capacity) {}
void* StringForwardingTable::Block::operator new(size_t size, int capacity) {
// Make sure the size given is the size of the Block structure.
DCHECK_EQ(size, sizeof(StringForwardingTable::Block));
// Make sure that the elements_ array is at the end of Block, with no padding,
// so that subsequent elements can be accessed as offsets from elements_.
static_assert(offsetof(StringForwardingTable::Block, elements_) ==
sizeof(StringForwardingTable::Block) - sizeof(Tagged_t) * 1);
// Make sure that elements_ is aligned when StringTable::Block is aligned.
static_assert((alignof(StringForwardingTable::Block) +
offsetof(StringForwardingTable::Block, elements_)) %
kTaggedSize ==
0);
const size_t elements_size = capacity * kRecordSize * sizeof(Tagged_t);
// Storage for the first element is already supplied by elements_, so subtract
// sizeof(Tagged_t).
const size_t new_size = size + elements_size - sizeof(Tagged_t);
DCHECK_LE(alignof(StringForwardingTable::Block), kSystemPointerSize);
return AlignedAlloc(new_size, kSystemPointerSize);
}
void StringForwardingTable::Block::operator delete(void* block) {
AlignedFree(block);
}
std::unique_ptr<StringForwardingTable::Block> StringForwardingTable::Block::New(
int capacity) {
return std::unique_ptr<Block>(new (capacity) Block(capacity));
}
void StringForwardingTable::Block::UpdateAfterEvacuation(Isolate* isolate) {
UpdateAfterEvacuation(isolate, capacity_);
}
void StringForwardingTable::Block::UpdateAfterEvacuation(Isolate* isolate,
int up_to_index) {
// This is only used for Scavenger.
DCHECK(!FLAG_minor_mc);
DCHECK(FLAG_always_use_string_forwarding_table);
for (int index = 0; index < up_to_index; ++index) {
Object original = Get(isolate, IndexOfOriginalString(index));
if (!original.IsHeapObject()) continue;
HeapObject object = HeapObject::cast(original);
if (Heap::InFromPage(object)) {
DCHECK(!object.InSharedWritableHeap());
MapWord map_word = object.map_word(kRelaxedLoad);
if (map_word.IsForwardingAddress()) {
HeapObject forwarded_object = map_word.ToForwardingAddress();
Set(IndexOfOriginalString(index), String::cast(forwarded_object));
} else {
Set(IndexOfOriginalString(index), deleted_element());
}
} else {
DCHECK(!object.map_word(kRelaxedLoad).IsForwardingAddress());
}
}
}
class StringForwardingTable::BlockVector {
public:
using Block = StringForwardingTable::Block;
using Allocator = std::allocator<Block*>;
explicit BlockVector(size_t capacity);
~BlockVector();
size_t capacity() const { return capacity_; }
Block* LoadBlock(size_t index, AcquireLoadTag) {
DCHECK_LT(index, size());
return base::AsAtomicPointer::Acquire_Load(&begin_[index]);
}
Block* LoadBlock(size_t index) {
DCHECK_LT(index, size());
return begin_[index];
}
void AddBlock(std::unique_ptr<Block> block) {
DCHECK_LT(size(), capacity());
base::AsAtomicPointer::Release_Store(&begin_[size_], block.release());
size_++;
}
static std::unique_ptr<BlockVector> Grow(BlockVector* data, size_t capacity,
const base::Mutex& mutex);
size_t size() const { return size_; }
private:
V8_NO_UNIQUE_ADDRESS Allocator allocator_;
const size_t capacity_;
std::atomic<size_t> size_;
Block** begin_;
};
StringForwardingTable::BlockVector::BlockVector(size_t capacity)
: allocator_(Allocator()), capacity_(capacity), size_(0) {
begin_ = allocator_.allocate(capacity);
}
StringForwardingTable::BlockVector::~BlockVector() {
allocator_.deallocate(begin_, capacity());
}
// static
std::unique_ptr<StringForwardingTable::BlockVector>
StringForwardingTable::BlockVector::Grow(
StringForwardingTable::BlockVector* data, size_t capacity,
const base::Mutex& mutex) {
mutex.AssertHeld();
std::unique_ptr<BlockVector> new_data =
std::make_unique<BlockVector>(capacity);
// Copy pointers to blocks from the old to the new vector.
for (size_t i = 0; i < data->size(); i++) {
new_data->begin_[i] = data->LoadBlock(i);
}
new_data->size_ = data->size();
return new_data;
}
StringForwardingTable::StringForwardingTable(Isolate* isolate)
: isolate_(isolate), next_free_index_(0) {
InitializeBlockVector();
}
StringForwardingTable::~StringForwardingTable() {
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
for (uint32_t block = 0; block < blocks->size(); block++) {
delete blocks->LoadBlock(block);
}
}
void StringForwardingTable::InitializeBlockVector() {
BlockVector* blocks = block_vector_storage_
.emplace_back(std::make_unique<BlockVector>(
kInitialBlockVectorCapacity))
.get();
blocks->AddBlock(Block::New(kInitialBlockSize));
blocks_.store(blocks, std::memory_order_relaxed);
}
StringForwardingTable::BlockVector* StringForwardingTable::EnsureCapacity(
uint32_t block) {
BlockVector* blocks = blocks_.load(std::memory_order_acquire);
if V8_UNLIKELY (block >= blocks->size()) {
base::MutexGuard table_grow_guard(&grow_mutex_);
// Reload the vector, as another thread could have grown it.
blocks = blocks_.load(std::memory_order_relaxed);
// Check again if we need to grow under lock.
if (block >= blocks->size()) {
const uint32_t capacity = CapacityForBlock(block);
std::unique_ptr<Block> new_block = Block::New(capacity);
// Grow the vector if the block to insert is greater than the vectors
// capacity.
if (block >= blocks->capacity()) {
std::unique_ptr<BlockVector> new_blocks =
BlockVector::Grow(blocks, blocks->capacity() * 2, grow_mutex_);
block_vector_storage_.push_back(std::move(new_blocks));
blocks = block_vector_storage_.back().get();
blocks_.store(blocks, std::memory_order_release);
}
blocks->AddBlock(std::move(new_block));
}
}
return blocks;
}
int StringForwardingTable::Add(Isolate* isolate, String string,
String forward_to) {
DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
string.InSharedHeap());
DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
forward_to.InSharedHeap());
int index = next_free_index_++;
uint32_t index_in_block;
const uint32_t block = BlockForIndex(index, &index_in_block);
BlockVector* blocks = EnsureCapacity(block);
Block* data = blocks->LoadBlock(block, kAcquireLoad);
data->Set(index_in_block, string, forward_to);
return index;
}
String StringForwardingTable::GetForwardString(Isolate* isolate,
int index) const {
CHECK_LT(index, Size());
uint32_t index_in_block;
const uint32_t block = BlockForIndex(index, &index_in_block);
Block* data =
blocks_.load(std::memory_order_acquire)->LoadBlock(block, kAcquireLoad);
return data->GetForwardString(isolate, index_in_block);
}
// static
Address StringForwardingTable::GetForwardStringAddress(Isolate* isolate,
int index) {
return isolate->string_forwarding_table()
->GetForwardString(isolate, index)
.ptr();
}
uint32_t StringForwardingTable::GetRawHash(Isolate* isolate, int index) const {
CHECK_LT(index, Size());
uint32_t index_in_block;
const uint32_t block = BlockForIndex(index, &index_in_block);
Block* data =
blocks_.load(std::memory_order_acquire)->LoadBlock(block, kAcquireLoad);
return data->GetRawHash(isolate, index_in_block);
}
void StringForwardingTable::IterateElements(RootVisitor* visitor) {
isolate_->heap()->safepoint()->AssertActive();
DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC);
if (next_free_index_ == 0) return; // Early exit if table is empty.
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
const uint32_t last_block = static_cast<uint32_t>(blocks->size() - 1);
for (uint32_t block = 0; block < last_block; ++block) {
Block* data = blocks->LoadBlock(block);
data->IterateElements(visitor, data->capacity());
}
// Handle last block separately, as it is not filled to capacity.
const uint32_t max_index = IndexInBlock(next_free_index_ - 1, last_block) + 1;
Block* data = blocks->LoadBlock(last_block);
data->IterateElements(visitor, max_index);
}
void StringForwardingTable::Reset() {
isolate_->heap()->safepoint()->AssertActive();
DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC);
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
for (uint32_t block = 0; block < blocks->size(); ++block) {
delete blocks->LoadBlock(block);
}
block_vector_storage_.clear();
InitializeBlockVector();
next_free_index_ = 0;
}
void StringForwardingTable::UpdateAfterEvacuation() {
DCHECK(FLAG_always_use_string_forwarding_table);
if (next_free_index_ == 0) return; // Early exit if table is empty.
BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
const unsigned int last_block = static_cast<unsigned int>(blocks->size() - 1);
for (unsigned int block = 0; block < last_block; ++block) {
Block* data = blocks->LoadBlock(block, kAcquireLoad);
data->UpdateAfterEvacuation(isolate_);
}
// Handle last block separately, as it is not filled to capacity.
const int max_index = IndexInBlock(next_free_index_ - 1, last_block) + 1;
blocks->LoadBlock(last_block, kAcquireLoad)
->UpdateAfterEvacuation(isolate_, max_index);
}
} // namespace internal
} // namespace v8

60
src/objects/string-table.h
View File

@ -95,66 +95,6 @@ class V8_EXPORT_PRIVATE StringTable {
Isolate* isolate_;
};
// Mapping from forwarding indices (stored in a string's hash field) to
// internalized strings.
// The table is organised in "blocks". As writes only append new entries, the
// organisation in blocks allows lock-free writes. We need a lock only for
// growing the table (adding more blocks). When the vector holding the blocks
// needs to grow, we keep a copy of the old vector alive to allow concurrent
// reads while the vector is relocated.
class StringForwardingTable {
public:
// Capacity for the first block.
static constexpr int kInitialBlockSize = 16;
static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize));
static constexpr int kInitialBlockSizeHighestBit =
kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1;
// Initial capacity in the block vector.
static constexpr int kInitialBlockVectorCapacity = 4;
static constexpr Smi deleted_element() { return Smi::FromInt(0); }
explicit StringForwardingTable(Isolate* isolate);
~StringForwardingTable();
inline int Size() const;
// Returns the index of the added string pair.
int Add(Isolate* isolate, String string, String forward_to);
String GetForwardString(Isolate* isolate, int index) const;
static Address GetForwardStringAddress(Isolate* isolate, int index);
V8_EXPORT_PRIVATE uint32_t GetRawHash(Isolate* isolate, int index) const;
void IterateElements(RootVisitor* visitor);
void Reset();
void UpdateAfterEvacuation();
private:
class Block;
class BlockVector;
// Returns the block for a given index and sets the index within this block
// as out parameter.
static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out);
static inline uint32_t IndexInBlock(int index, uint32_t block);
static inline uint32_t CapacityForBlock(uint32_t block);
void InitializeBlockVector();
// Ensure that |block| exists in the BlockVector already. If not, a new block
// is created (with capacity double the capacity of the last block) and
// inserted into the BlockVector. The BlockVector itself might grow (to double
// the capacity).
BlockVector* EnsureCapacity(uint32_t block);
Isolate* isolate_;
std::atomic<BlockVector*> blocks_;
// We need a vector of BlockVectors to keep old BlockVectors alive when we
// grow the table, due to concurrent reads that may still hold a pointer to
// them. |block_vector_sotrage_| is only accessed while we grow with the mutex
// held. All regular access go through |block_|, which holds a pointer to the
// current BlockVector.
std::vector<std::unique_ptr<BlockVector>> block_vector_storage_;
std::atomic<int> next_free_index_;
base::Mutex grow_mutex_;
};
} // namespace internal
} // namespace v8

1
src/objects/visitors.h
View File

@ -18,7 +18,6 @@ class CodeDataContainer;
#define ROOT_ID_LIST(V) \
V(kStringTable, "(Internalized strings)") \
V(kStringForwardingTable, "(Forwarded strings)") \
V(kExternalStringsTable, "(External strings)") \
V(kReadOnlyRootList, "(Read-only roots)") \
V(kStrongRootList, "(Strong roots)") \

3
test/cctest/test-shared-strings.cc
View File

@ -13,6 +13,7 @@
#include "src/heap/remembered-set.h"
#include "src/objects/fixed-array.h"
#include "src/objects/objects-inl.h"
#include "src/objects/string-forwarding-table-inl.h"
#include "test/cctest/cctest.h"
#include "test/cctest/heap/heap-utils.h"
@ -901,7 +902,7 @@ UNINITIALIZED_TEST(SharedStringsTransitionDuringGC) {
i_isolate->heap()->CollectSharedGarbage(GarbageCollectionReason::kTesting);
// Check that GC cleared the forwarding table.
CHECK_EQ(i_isolate->string_forwarding_table()->Size(), 0);
CHECK_EQ(i_isolate->string_forwarding_table()->size(), 0);
// Check all strings are transitioned to ThinStrings
for (int i = 0; i < shared_strings->length(); i++) {