cppgc: Speed up pointer decompression

With this CL, the decompression simply becomes:
       movsxd  rax, edi
       add     rax, rax
       and     rax, qword ptr fs:[base@TPOFF]

Bug: chromium:1325007

Change-Id: I931e4e667a9b9697671bccf14575420f8cb705e8
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3629871
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80521}
This commit is contained in:
Anton Bikineev 2022-05-13 14:29:54 +02:00 committed by V8 LUCI CQ
parent 24286b8e24
commit 2c40f3af4f
9 changed files with 115 additions and 23 deletions

View File

@ -3003,6 +3003,7 @@ filegroup(
"src/heap/cppgc/marking-worklists.cc",
"src/heap/cppgc/marking-worklists.h",
"src/heap/cppgc/member.cc",
"src/heap/cppgc/member.h",
"src/heap/cppgc/memory.cc",
"src/heap/cppgc/memory.h",
"src/heap/cppgc/metric-recorder.h",

View File

@ -5726,6 +5726,7 @@ v8_source_set("cppgc_base") {
"src/heap/cppgc/marking-worklists.cc",
"src/heap/cppgc/marking-worklists.h",
"src/heap/cppgc/member.cc",
"src/heap/cppgc/member.h",
"src/heap/cppgc/memory.cc",
"src/heap/cppgc/memory.h",
"src/heap/cppgc/metric-recorder.h",

View File

@ -25,19 +25,34 @@ namespace internal {
class CageBaseGlobal final {
public:
V8_INLINE static void Update(uintptr_t base) {
CPPGC_DCHECK(0u ==
(base & (api_constants::kCagedHeapReservationAlignment - 1)));
g_base_ = base;
V8_INLINE static uintptr_t Get() {
CPPGC_DCHECK(IsBaseConsistent());
return g_base_;
}
V8_INLINE static uintptr_t Get() { return g_base_; }
V8_INLINE static bool IsSet() {
CPPGC_DCHECK(IsBaseConsistent());
return (g_base_ & ~kLowerHalfWordMask) != 0;
}
private:
// We keep the lower halfword as ones to speed up decompression.
static constexpr uintptr_t kLowerHalfWordMask =
(api_constants::kCagedHeapReservationAlignment - 1);
static thread_local V8_EXPORT uintptr_t g_base_
__attribute__((require_constant_initialization));
#if !V8_CC_MSVC
__attribute__((require_constant_initialization))
#endif // !V8_CC_MSVC
;
CageBaseGlobal() = delete;
V8_INLINE static bool IsBaseConsistent() {
return kLowerHalfWordMask == (g_base_ & kLowerHalfWordMask);
}
friend class CageBaseGlobalUpdater;
};
class CompressedPointer final {
@ -64,22 +79,36 @@ class CompressedPointer final {
V8_INLINE void StoreRaw(Storage value) { value_ = value; }
static V8_INLINE Storage Compress(const void* ptr) {
static_assert(
SentinelPointer::kSentinelValue == 0b10,
"The compression scheme relies on the sentinel encoded as 0b10");
static constexpr size_t kGigaCageMask =
~(api_constants::kCagedHeapReservationAlignment - 1);
CPPGC_DCHECK(CageBaseGlobal::IsSet());
const uintptr_t base = CageBaseGlobal::Get();
CPPGC_DCHECK(base);
CPPGC_DCHECK(!ptr || ptr == kSentinelPointer ||
base == (reinterpret_cast<uintptr_t>(ptr) & kGigaCageMask));
return static_cast<Storage>(reinterpret_cast<uintptr_t>(ptr));
(base & kGigaCageMask) ==
(reinterpret_cast<uintptr_t>(ptr) & kGigaCageMask));
const auto uptr = reinterpret_cast<uintptr_t>(ptr);
// Truncate the pointer and shift right by one.
auto compressed = static_cast<Storage>(uptr) >> 1;
// If the pointer is regular, set the most significant bit.
if (V8_LIKELY(compressed > 1)) {
CPPGC_DCHECK((reinterpret_cast<uintptr_t>(ptr) &
(api_constants::kAllocationGranularity - 1)) == 0);
compressed |= 0x80000000;
}
return compressed;
}
static V8_INLINE void* Decompress(Storage ptr) {
CPPGC_DCHECK(CageBaseGlobal::IsSet());
const uintptr_t base = CageBaseGlobal::Get();
CPPGC_DCHECK(base);
// We have to preserve nullptr and kSentinel, Members can't point to
// GigaCage metadata.
if (V8_UNLIKELY(ptr <= 1)) return reinterpret_cast<void*>(ptr);
return reinterpret_cast<void*>(base | static_cast<uintptr_t>(ptr));
// Sign extend the pointer and shift left by one.
const int64_t mask = static_cast<int64_t>(static_cast<int32_t>(ptr)) << 1;
return reinterpret_cast<void*>(mask & base);
}
private:

View File

@ -13,9 +13,9 @@ namespace internal {
// Special tag type used to denote some sentinel member. The semantics of the
// sentinel is defined by the embedder.
struct SentinelPointer {
static constexpr intptr_t kSentinelValue = 0b10;
template <typename T>
operator T*() const {
static constexpr intptr_t kSentinelValue = 1;
return reinterpret_cast<T*>(kSentinelValue);
}
// Hidden friends.

View File

@ -16,6 +16,7 @@
#include "src/base/platform/platform.h"
#include "src/heap/cppgc/caged-heap.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/member.h"
namespace cppgc {
namespace internal {
@ -54,8 +55,9 @@ CagedHeap::CagedHeap(HeapBase& heap_base, PageAllocator& platform_allocator)
#if defined(CPPGC_POINTER_COMPRESSION)
// With pointer compression only single heap per thread is allowed.
CHECK(!CageBaseGlobal::Get());
CageBaseGlobal::Update(reinterpret_cast<uintptr_t>(reserved_area_.address()));
CHECK(!CageBaseGlobal::IsSet());
CageBaseGlobalUpdater::UpdateCageBase(
reinterpret_cast<uintptr_t>(reserved_area_.address()));
#endif // defined(CPPGC_POINTER_COMPRESSION)
const bool is_not_oom = platform_allocator.SetPermissions(
@ -86,8 +88,8 @@ CagedHeap::CagedHeap(HeapBase& heap_base, PageAllocator& platform_allocator)
CagedHeap::~CagedHeap() {
#if defined(CPPGC_POINTER_COMPRESSION)
CHECK_EQ(reinterpret_cast<uintptr_t>(reserved_area_.address()),
CageBaseGlobal::Get());
CageBaseGlobal::Update(0u);
CageBaseGlobalUpdater::GetCageBase());
CageBaseGlobalUpdater::UpdateCageBase(0u);
#endif // defined(CPPGC_POINTER_COMPRESSION)
}

View File

@ -10,6 +10,7 @@
#include "src/heap/cppgc/liveness-broker.h"
#include "src/heap/cppgc/marking-state.h"
#include "src/heap/cppgc/marking-visitor.h"
#include "src/heap/cppgc/member.h"
#include "src/heap/cppgc/stats-collector.h"
namespace cppgc {
@ -63,12 +64,14 @@ namespace {
class PointerCompressionCageScope final {
public:
explicit PointerCompressionCageScope(HeapBase& heap)
: prev_cage_base_(CageBaseGlobal::Get()) {
CageBaseGlobal::Update(
: prev_cage_base_(CageBaseGlobalUpdater::GetCageBase()) {
CageBaseGlobalUpdater::UpdateCageBase(
reinterpret_cast<uintptr_t>(heap.caged_heap().base()));
}
~PointerCompressionCageScope() { CageBaseGlobal::Update(prev_cage_base_); }
~PointerCompressionCageScope() {
CageBaseGlobalUpdater::UpdateCageBase(prev_cage_base_);
}
private:
const uintptr_t prev_cage_base_;

View File

@ -8,7 +8,8 @@ namespace cppgc {
namespace internal {
#if defined(CPPGC_POINTER_COMPRESSION)
thread_local uintptr_t CageBaseGlobal::g_base_ = 0u;
thread_local uintptr_t CageBaseGlobal::g_base_ =
CageBaseGlobal::kLowerHalfWordMask;
#endif // defined(CPPGC_POINTER_COMPRESSION)
} // namespace internal

33
src/heap/cppgc/member.h Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2022 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_MEMBER_H_
#define V8_HEAP_CPPGC_MEMBER_H_
#include "include/cppgc/member.h"
namespace cppgc {
namespace internal {
#if defined(CPPGC_POINTER_COMPRESSION)
class CageBaseGlobalUpdater final {
public:
CageBaseGlobalUpdater() = delete;
static void UpdateCageBase(uintptr_t cage_base) {
CPPGC_DCHECK(CageBaseGlobal::IsBaseConsistent());
CPPGC_DCHECK(0u == (cage_base & CageBaseGlobal::kLowerHalfWordMask));
CageBaseGlobal::g_base_ = cage_base | CageBaseGlobal::kLowerHalfWordMask;
}
static uintptr_t GetCageBase() {
CPPGC_DCHECK(CageBaseGlobal::IsBaseConsistent());
return CageBaseGlobal::g_base_ & ~CageBaseGlobal::kLowerHalfWordMask;
}
};
#endif // defined(CPPGC_POINTER_COMPRESSION)
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_MEMBER_H_

View File

@ -566,6 +566,28 @@ TEST_F(MemberHeapDeathTest, CheckForOnHeapMemberCrashesOnInitialAssignment) {
}
#endif // defined(CPPGC_POINTER_COMPRESSION)
#if defined(CPPGC_POINTER_COMPRESSION)
TEST_F(MemberTest, CompressDecompress) {
CompressedPointer cp;
EXPECT_EQ(nullptr, cp.Load());
Member<GCed> member;
cp.Store(member.Get());
EXPECT_EQ(nullptr, cp.Load());
cp.Store(kSentinelPointer);
EXPECT_EQ(kSentinelPointer, cp.Load());
member = kSentinelPointer;
cp.Store(member.Get());
EXPECT_EQ(kSentinelPointer, cp.Load());
member = MakeGarbageCollected<GCed>(GetAllocationHandle());
cp.Store(member.Get());
EXPECT_EQ(member.Get(), cp.Load());
}
#endif // defined(CPPGC_POINTER_COMPRESSION)
#endif // V8_ENABLE_CHECKS
} // namespace internal