[ptr-compr] Store cage bases in globals when cage sharing is enabled

... instead of computing them on the fly. This approach seems to
perform slightly better because it requires less code.

Bug: v8:7703, v8:11460
Change-Id: If31a06fbc748251c491c011e9e3f118665e20159
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4020456
Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/main@{#84413}
This commit is contained in:
Igor Sheludko 2022-11-18 18:50:12 +01:00 committed by V8 LUCI CQ
parent 5e07bb70e5
commit 00c7e383a8
16 changed files with 278 additions and 57 deletions

View File

@ -1245,6 +1245,7 @@ filegroup(
"src/common/message-template.h",
"src/common/operation.h",
"src/common/ptr-compr-inl.h",
"src/common/ptr-compr.cc",
"src/common/ptr-compr.h",
"src/compiler-dispatcher/lazy-compile-dispatcher.cc",
"src/compiler-dispatcher/lazy-compile-dispatcher.h",

View File

@ -4407,6 +4407,7 @@ v8_source_set("v8_base_without_compiler") {
"src/codegen/unoptimized-compilation-info.cc",
"src/common/assert-scope.cc",
"src/common/code-memory-access.cc",
"src/common/ptr-compr.cc",
"src/compiler-dispatcher/lazy-compile-dispatcher.cc",
"src/compiler-dispatcher/optimizing-compile-dispatcher.cc",
"src/date/date.cc",

View File

@ -920,6 +920,7 @@ class CompressedMaybeObjectSlot;
class CompressedMapWordSlot;
class CompressedHeapObjectSlot;
class V8HeapCompressionScheme;
class ExternalCodeCompressionScheme;
template <typename CompressionScheme>
class OffHeapCompressedObjectSlot;
class FullObjectSlot;
@ -951,7 +952,12 @@ struct SlotTraits {
using THeapObjectSlot = CompressedHeapObjectSlot;
using TOffHeapObjectSlot =
OffHeapCompressedObjectSlot<V8HeapCompressionScheme>;
using TCodeObjectSlot = OffHeapCompressedObjectSlot<V8HeapCompressionScheme>;
#ifdef V8_EXTERNAL_CODE_SPACE
using TCodeObjectSlot =
OffHeapCompressedObjectSlot<ExternalCodeCompressionScheme>;
#else
using TCodeObjectSlot = TObjectSlot;
#endif // V8_EXTERNAL_CODE_SPACE
#else
using TObjectSlot = FullObjectSlot;
using TMaybeObjectSlot = FullMaybeObjectSlot;
@ -2082,7 +2088,7 @@ class PtrComprCageBase {
// NOLINTNEXTLINE
inline PtrComprCageBase(const LocalIsolate* isolate);
inline Address address() const;
inline Address address() const { return address_; }
bool operator==(const PtrComprCageBase& other) const {
return address_ == other.address_;

View File

@ -19,13 +19,6 @@ PtrComprCageBase::PtrComprCageBase(const Isolate* isolate)
PtrComprCageBase::PtrComprCageBase(const LocalIsolate* isolate)
: address_(isolate->cage_base()) {}
Address PtrComprCageBase::address() const {
Address ret = address_;
ret = reinterpret_cast<Address>(V8_ASSUME_ALIGNED(
reinterpret_cast<void*>(ret), kPtrComprCageBaseAlignment));
return ret;
}
//
// V8HeapCompressionScheme
//
@ -39,9 +32,27 @@ Address V8HeapCompressionScheme::GetPtrComprCageBaseAddress(
// static
Address V8HeapCompressionScheme::GetPtrComprCageBaseAddress(
PtrComprCageBase cage_base) {
return cage_base.address();
Address base = cage_base.address();
base = reinterpret_cast<Address>(V8_ASSUME_ALIGNED(
reinterpret_cast<void*>(base), kPtrComprCageBaseAlignment));
return base;
}
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// static
void V8HeapCompressionScheme::InitBase(Address base) {
CHECK_EQ(base, GetPtrComprCageBaseAddress(base));
base_ = base;
}
// static
Address V8HeapCompressionScheme::base() {
return reinterpret_cast<Address>(V8_ASSUME_ALIGNED(
reinterpret_cast<void*>(base_), kPtrComprCageBaseAlignment));
}
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// static
Tagged_t V8HeapCompressionScheme::CompressTagged(Address tagged) {
return static_cast<Tagged_t>(static_cast<uint32_t>(tagged));
@ -57,8 +68,13 @@ Address V8HeapCompressionScheme::DecompressTaggedSigned(Tagged_t raw_value) {
template <typename TOnHeapAddress>
Address V8HeapCompressionScheme::DecompressTaggedPointer(
TOnHeapAddress on_heap_addr, Tagged_t raw_value) {
return GetPtrComprCageBaseAddress(on_heap_addr) +
static_cast<Address>(raw_value);
#if defined(V8_COMPRESS_POINTERS_IN_SHARED_CAGE) && \
!defined(V8_COMPRESS_POINTERS_DONT_USE_GLOBAL_BASE)
Address cage_base = base();
#else
Address cage_base = GetPtrComprCageBaseAddress(on_heap_addr);
#endif
return cage_base + static_cast<Address>(raw_value);
}
// static
@ -85,6 +101,76 @@ void V8HeapCompressionScheme::ProcessIntermediatePointers(
callback(decompressed_high);
}
#ifdef V8_EXTERNAL_CODE_SPACE
//
// ExternalCodeCompressionScheme
//
// static
Address ExternalCodeCompressionScheme::PrepareCageBaseAddress(
Address on_heap_addr) {
return RoundDown<kPtrComprCageBaseAlignment>(on_heap_addr);
}
// static
Address ExternalCodeCompressionScheme::GetPtrComprCageBaseAddress(
PtrComprCageBase cage_base) {
Address base = cage_base.address();
base = reinterpret_cast<Address>(V8_ASSUME_ALIGNED(
reinterpret_cast<void*>(base), kPtrComprCageBaseAlignment));
return base;
}
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// static
void ExternalCodeCompressionScheme::InitBase(Address base) {
CHECK_EQ(base, PrepareCageBaseAddress(base));
base_ = base;
}
// static
Address ExternalCodeCompressionScheme::base() {
return reinterpret_cast<Address>(V8_ASSUME_ALIGNED(
reinterpret_cast<void*>(base_), kPtrComprCageBaseAlignment));
}
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// static
Tagged_t ExternalCodeCompressionScheme::CompressTagged(Address tagged) {
return static_cast<Tagged_t>(static_cast<uint32_t>(tagged));
}
// static
Address ExternalCodeCompressionScheme::DecompressTaggedSigned(
Tagged_t raw_value) {
// For runtime code the upper 32-bits of the Smi value do not matter.
return static_cast<Address>(raw_value);
}
// static
template <typename TOnHeapAddress>
Address ExternalCodeCompressionScheme::DecompressTaggedPointer(
TOnHeapAddress on_heap_addr, Tagged_t raw_value) {
#if defined(V8_COMPRESS_POINTERS_IN_SHARED_CAGE) && \
!defined(V8_COMPRESS_POINTERS_DONT_USE_GLOBAL_BASE)
Address cage_base = base();
#else
Address cage_base = GetPtrComprCageBaseAddress(on_heap_addr);
#endif
return cage_base + static_cast<Address>(raw_value);
}
// static
template <typename TOnHeapAddress>
Address ExternalCodeCompressionScheme::DecompressTaggedAny(
TOnHeapAddress on_heap_addr, Tagged_t raw_value) {
return DecompressTaggedPointer(on_heap_addr, raw_value);
}
#endif // V8_EXTERNAL_CODE_SPACE
//
// Misc functions.
//
@ -117,6 +203,7 @@ Address V8HeapCompressionScheme::DecompressTaggedSigned(Tagged_t raw_value) {
UNREACHABLE();
}
// static
template <typename TOnHeapAddress>
Address V8HeapCompressionScheme::DecompressTaggedPointer(
TOnHeapAddress on_heap_addr, Tagged_t raw_value) {

19
src/common/ptr-compr.cc Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/common/ptr-compr.h"
namespace v8::internal {
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
uintptr_t V8HeapCompressionScheme::base_ = kNullAddress;
#ifdef V8_EXTERNAL_CODE_SPACE
uintptr_t ExternalCodeCompressionScheme::base_ = kNullAddress;
#endif // V8_EXTERNAL_CODE_SPACE
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
} // namespace v8::internal

View File

@ -47,12 +47,61 @@ class V8HeapCompressionScheme {
V8_INLINE static void ProcessIntermediatePointers(
PtrComprCageBase cage_base, Address raw_value,
ProcessPointerCallback callback);
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// Process-wide cage base value used for decompression.
V8_INLINE static void InitBase(Address base);
V8_INLINE static Address base();
private:
static V8_EXPORT_PRIVATE uintptr_t base_ V8_CONSTINIT;
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
};
#ifdef V8_EXTERNAL_CODE_SPACE
// Compression scheme used for fields containing Code objects (namely for the
// CodeDataContainer::code field).
using ExternalCodeCompressionScheme = V8HeapCompressionScheme;
// Same as V8HeapCompressionScheme but with a different base value.
class ExternalCodeCompressionScheme {
public:
V8_INLINE static Address PrepareCageBaseAddress(Address on_heap_addr);
// Note that this compression scheme doesn't allow reconstruction of the cage
// base value from any arbitrary value, thus the cage base has to be passed
// explicitly to the decompression functions.
static Address GetPtrComprCageBaseAddress(Address on_heap_addr) = delete;
V8_INLINE static Address GetPtrComprCageBaseAddress(
PtrComprCageBase cage_base);
// Compresses full-pointer representation of a tagged value to on-heap
// representation.
V8_INLINE static Tagged_t CompressTagged(Address tagged);
// Decompresses smi value.
V8_INLINE static Address DecompressTaggedSigned(Tagged_t raw_value);
// Decompresses weak or strong heap object pointer or forwarding pointer,
// preserving both weak- and smi- tags.
template <typename TOnHeapAddress>
V8_INLINE static Address DecompressTaggedPointer(TOnHeapAddress on_heap_addr,
Tagged_t raw_value);
// Decompresses any tagged value, preserving both weak- and smi- tags.
template <typename TOnHeapAddress>
V8_INLINE static Address DecompressTaggedAny(TOnHeapAddress on_heap_addr,
Tagged_t raw_value);
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// Process-wide cage base value used for decompression.
V8_INLINE static void InitBase(Address base);
V8_INLINE static Address base();
private:
static V8_EXPORT_PRIVATE uintptr_t base_ V8_CONSTINIT;
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
};
#endif // V8_EXTERNAL_CODE_SPACE
// Accessors for fields that may be unaligned due to pointer compression.

View File

@ -4109,6 +4109,11 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data,
SnapshotData* read_only_snapshot_data,
SnapshotData* shared_heap_snapshot_data, bool can_rehash) {
TRACE_ISOLATE(init);
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
CHECK_EQ(V8HeapCompressionScheme::base(), cage_base());
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
const bool create_heap_objects = (read_only_snapshot_data == nullptr);
// We either have all or none.
DCHECK_EQ(create_heap_objects, startup_snapshot_data == nullptr);
@ -4287,15 +4292,32 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data,
}
}
#ifdef V8_EXTERNAL_CODE_SPACE
if (heap_.code_range()) {
code_cage_base_ = ExternalCodeCompressionScheme::GetPtrComprCageBaseAddress(
heap_.code_range()->base());
} else {
CHECK(jitless_);
// In jitless mode the code space pages will be allocated in the main
// pointer compression cage.
code_cage_base_ =
ExternalCodeCompressionScheme::GetPtrComprCageBaseAddress(cage_base());
{
VirtualMemoryCage* code_cage;
if (heap_.code_range()) {
code_cage = heap_.code_range();
} else {
CHECK(jitless_);
// In jitless mode the code space pages will be allocated in the main
// pointer compression cage.
code_cage = GetPtrComprCage();
}
code_cage_base_ = ExternalCodeCompressionScheme::PrepareCageBaseAddress(
code_cage->base());
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
CHECK_EQ(ExternalCodeCompressionScheme::base(), code_cage_base_);
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// Ensure that ExternalCodeCompressionScheme is applicable to all objects
// stored in the code cage.
using ComprScheme = ExternalCodeCompressionScheme;
Address base = code_cage->base();
Address last = base + code_cage->size() - 1;
PtrComprCageBase code_cage_base{code_cage_base_};
CHECK_EQ(base, ComprScheme::DecompressTaggedPointer(
code_cage_base, ComprScheme::CompressTagged(base)));
CHECK_EQ(last, ComprScheme::DecompressTaggedPointer(
code_cage_base, ComprScheme::CompressTagged(last)));
}
#endif // V8_EXTERNAL_CODE_SPACE

View File

@ -147,15 +147,6 @@ bool CodeRange::InitReservation(v8::PageAllocator* page_allocator,
if (!VirtualMemoryCage::InitReservation(params)) return false;
#ifdef V8_EXTERNAL_CODE_SPACE
// Ensure that ExternalCodeCompressionScheme is applicable to all objects
// stored in the code range.
Address base = page_allocator_->begin();
Address last = base + page_allocator_->size() - 1;
CHECK_EQ(ExternalCodeCompressionScheme::GetPtrComprCageBaseAddress(base),
ExternalCodeCompressionScheme::GetPtrComprCageBaseAddress(last));
#endif // V8_EXTERNAL_CODE_SPACE
// On some platforms, specifically Win64, we need to reserve some pages at
// the beginning of an executable space. See
// https://cs.chromium.org/chromium/src/components/crash/content/
@ -319,6 +310,16 @@ std::shared_ptr<CodeRange> CodeRange::EnsureProcessWideCodeRange(
nullptr, "Failed to reserve virtual memory for CodeRange");
}
*process_wide_code_range_.Pointer() = code_range;
#ifdef V8_EXTERNAL_CODE_SPACE
#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE
// This might be not the first time we create a code range because previous
// code range instance could have been deleted if it wasn't kept alive by an
// active Isolate. Let the base be initialized from scratch once again.
ExternalCodeCompressionScheme::InitBase(
ExternalCodeCompressionScheme::PrepareCageBaseAddress(
code_range->base()));
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
#endif // V8_EXTERNAL_CODE_SPACE
}
return code_range;
}

View File

@ -3712,6 +3712,14 @@ MaybeObject MakeSlotValue<FullMaybeObjectSlot, HeapObjectReferenceType::STRONG>(
return HeapObjectReference::Strong(heap_object);
}
#ifdef V8_EXTERNAL_CODE_SPACE
template <>
Object MakeSlotValue<CodeObjectSlot, HeapObjectReferenceType::STRONG>(
HeapObject heap_object) {
return heap_object;
}
#endif // V8_EXTERNAL_CODE_SPACE
// The following specialization
// MakeSlotValue<FullMaybeObjectSlot, HeapObjectReferenceType::WEAK>()
// is not used.
@ -3726,9 +3734,10 @@ static inline void UpdateSlot(PtrComprCageBase cage_base, TSlot slot,
std::is_same<TSlot, ObjectSlot>::value ||
std::is_same<TSlot, FullMaybeObjectSlot>::value ||
std::is_same<TSlot, MaybeObjectSlot>::value ||
std::is_same<TSlot, OffHeapObjectSlot>::value,
"Only [Full|OffHeap]ObjectSlot and [Full]MaybeObjectSlot are "
"expected here");
std::is_same<TSlot, OffHeapObjectSlot>::value ||
std::is_same<TSlot, CodeObjectSlot>::value,
"Only [Full|OffHeap]ObjectSlot, [Full]MaybeObjectSlot "
"or CodeObjectSlot are expected here");
MapWord map_word = heap_obj.map_word(cage_base, kRelaxedLoad);
if (map_word.IsForwardingAddress()) {
DCHECK_IMPLIES((!v8_flags.minor_mc && !Heap::InFromPage(heap_obj)),

View File

@ -5,6 +5,7 @@
#include "src/init/isolate-allocator.h"
#include "src/base/bounded-page-allocator.h"
#include "src/common/ptr-compr-inl.h"
#include "src/execution/isolate.h"
#include "src/heap/code-range.h"
#include "src/sandbox/sandbox.h"
@ -94,7 +95,14 @@ void IsolateAllocator::InitializeOncePerProcess() {
"Failed to reserve virtual memory for process-wide V8 "
"pointer compression cage");
}
#endif
V8HeapCompressionScheme::InitBase(GetProcessWidePtrComprCage()->base());
#ifdef V8_EXTERNAL_CODE_SPACE
// Speculatively set the code cage base to the same value in case jitless
// mode will be used. Once the process-wide CodeRange instance is created
// the code cage base will be set accordingly.
ExternalCodeCompressionScheme::InitBase(V8HeapCompressionScheme::base());
#endif // V8_EXTERNAL_CODE_SPACE
#endif // V8_COMPRESS_POINTERS_IN_SHARED_CAGE
}
IsolateAllocator::IsolateAllocator() {

View File

@ -1448,8 +1448,7 @@ Object CodeDataContainer::raw_code() const {
Object CodeDataContainer::raw_code(PtrComprCageBase cage_base) const {
#ifdef V8_EXTERNAL_CODE_SPACE
Object value = ExternalCodeField::load(cage_base, *this);
return value;
return ExternalCodeField<Object>::load(cage_base, *this);
#else
UNREACHABLE();
#endif // V8_EXTERNAL_CODE_SPACE
@ -1457,7 +1456,7 @@ Object CodeDataContainer::raw_code(PtrComprCageBase cage_base) const {
void CodeDataContainer::set_raw_code(Object value, WriteBarrierMode mode) {
#ifdef V8_EXTERNAL_CODE_SPACE
ExternalCodeField::Release_Store(*this, value);
ExternalCodeField<Object>::Release_Store(*this, value);
CONDITIONAL_WRITE_BARRIER(*this, kCodeOffset, value, mode);
#else
UNREACHABLE();
@ -1472,8 +1471,7 @@ Object CodeDataContainer::raw_code(RelaxedLoadTag tag) const {
Object CodeDataContainer::raw_code(PtrComprCageBase cage_base,
RelaxedLoadTag) const {
#ifdef V8_EXTERNAL_CODE_SPACE
Object value = ExternalCodeField::Relaxed_Load(cage_base, *this);
return value;
return ExternalCodeField<Object>::Relaxed_Load(cage_base, *this);
#else
UNREACHABLE();
#endif // V8_EXTERNAL_CODE_SPACE
@ -1487,7 +1485,7 @@ PtrComprCageBase CodeDataContainer::code_cage_base() const {
return PtrComprCageBase(isolate->code_cage_base());
#else
return GetPtrComprCageBase(*this);
#endif
#endif // V8_EXTERNAL_CODE_SPACE
}
Code CodeDataContainer::code() const {
@ -1495,11 +1493,12 @@ Code CodeDataContainer::code() const {
return CodeDataContainer::code(cage_base);
}
Code CodeDataContainer::code(PtrComprCageBase cage_base) const {
CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
#ifdef V8_EXTERNAL_CODE_SPACE
DCHECK(!is_off_heap_trampoline());
#endif
return Code::cast(raw_code(cage_base));
return ExternalCodeField<Code>::load(cage_base, *this);
#else
UNREACHABLE();
#endif // V8_EXTERNAL_CODE_SPACE
}
Code CodeDataContainer::code(RelaxedLoadTag tag) const {
@ -1509,8 +1508,12 @@ Code CodeDataContainer::code(RelaxedLoadTag tag) const {
Code CodeDataContainer::code(PtrComprCageBase cage_base,
RelaxedLoadTag tag) const {
CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
return Code::cast(raw_code(cage_base, tag));
#ifdef V8_EXTERNAL_CODE_SPACE
DCHECK(!is_off_heap_trampoline());
return ExternalCodeField<Code>::Relaxed_Load(cage_base, *this);
#else
UNREACHABLE();
#endif // V8_EXTERNAL_CODE_SPACE
}
DEF_GETTER(CodeDataContainer, code_entry_point, Address) {

View File

@ -261,8 +261,9 @@ class CodeDataContainer : public HeapObject {
#undef CODE_DATA_FIELDS
#ifdef V8_EXTERNAL_CODE_SPACE
template <typename T>
using ExternalCodeField =
TaggedField<Object, kCodeOffset, ExternalCodeCompressionScheme>;
TaggedField<T, kCodeOffset, ExternalCodeCompressionScheme>;
#endif
class BodyDescriptor;

View File

@ -129,17 +129,21 @@ UNINITIALIZED_TEST(SharedPtrComprCageRace) {
// Make a bunch of Isolates concurrently as a smoke test against races during
// initialization and de-initialization.
std::vector<std::unique_ptr<IsolateAllocatingThread>> threads;
constexpr int kThreads = 10;
// Repeat twice to enforce multiple initializations of CodeRange instances.
constexpr int kRepeats = 2;
for (int repeat = 0; repeat < kRepeats; repeat++) {
std::vector<std::unique_ptr<IsolateAllocatingThread>> threads;
constexpr int kThreads = 10;
for (int i = 0; i < kThreads; i++) {
auto thread = std::make_unique<IsolateAllocatingThread>();
CHECK(thread->Start());
threads.push_back(std::move(thread));
}
for (int i = 0; i < kThreads; i++) {
auto thread = std::make_unique<IsolateAllocatingThread>();
CHECK(thread->Start());
threads.push_back(std::move(thread));
}
for (auto& thread : threads) {
thread->Join();
for (auto& thread : threads) {
thread->Join();
}
}
}

View File

@ -151,7 +151,11 @@ TEST_F(HeapTest, HeapLayout) {
EXPECT_TRUE(IsAligned(cage_base, size_t{4} * GB));
Address code_cage_base = i_isolate()->code_cage_base();
EXPECT_TRUE(IsAligned(code_cage_base, size_t{4} * GB));
if (V8_EXTERNAL_CODE_SPACE_BOOL) {
EXPECT_TRUE(IsAligned(code_cage_base, kMinExpectedOSPageSize));
} else {
EXPECT_TRUE(IsAligned(code_cage_base, size_t{4} * GB));
}
#ifdef V8_COMPRESS_POINTERS_IN_ISOLATE_CAGE
Address isolate_root = i_isolate()->isolate_root();

View File

@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Don't bother initializing global cage base value, compute it from any
// on heap address instead.
#define V8_COMPRESS_POINTERS_DONT_USE_GLOBAL_BASE
#include "debug-helper-internal.h"
#include "src/common/ptr-compr-inl.h"
#include "torque-generated/class-debug-readers.h"

View File

@ -16,6 +16,9 @@ out = """
#include <cstdint>
#include <string>
// Don't bother initializing global cage base value, compute it from any
// on heap address instead.
#define V8_COMPRESS_POINTERS_DONT_USE_GLOBAL_BASE
#include "src/common/ptr-compr-inl.h"
#include "tools/debug_helper/debug-helper-internal.h"