cppgc: Port Persistent

CrossThreadPersistent and friends are the followup.

Bug: chromium:1056170
Change-Id: Ide910062d80952da73b922398c281162b1861f47
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2144957
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67153}

BUILD.gn

@@ -218,6 +218,9 @@ declare_args() {
   # Enable control-flow integrity features, such as pointer authentication for
   # ARM64.
   v8_control_flow_integrity = false
+
+  # Enable object names in cppgc for debug purposes.
+  cppgc_enable_object_names = false
 }
 
 # Derived defaults.
@@ -351,15 +354,19 @@ config("cppgc_base_config") {
   # Assume is_clang = false means GCC or other compilers that are compatible
   # with gas inline assembly on non-Windows builds. Cross builds for
   # Linux->Windows are covered as they use clang.
+  defines = []
   if (is_clang || !is_win) {
     if (target_cpu == "x64" || target_cpu == "x86" || target_cpu == "arm" ||
         target_cpu == "arm64" || target_cpu == "ppc64" ||
         target_cpu == "s390x") {
-      defines = [ "CPPGC_SUPPORTS_CONSERVATIVE_STACK_SCAN" ]
+      defines += [ "CPPGC_SUPPORTS_CONSERVATIVE_STACK_SCAN" ]
     }
   } else if (is_win) {
     # Assume that non-clang builds on Windows use native tools.
-    defines = [ "CPPGC_SUPPORTS_CONSERVATIVE_STACK_SCAN" ]
+    defines += [ "CPPGC_SUPPORTS_CONSERVATIVE_STACK_SCAN" ]
+  }
+  if (cppgc_enable_object_names) {
+    defines += [ "CPPGC_SUPPORTS_OBJECT_NAMES" ]
   }
 }
 
@@ -3955,12 +3962,14 @@ v8_source_set("cppgc_base") {
     "include/cppgc/internal/compiler-specific.h",
     "include/cppgc/internal/finalizer-traits.h",
     "include/cppgc/internal/gc-info.h",
+    "include/cppgc/internal/persistent-node.h",
     "include/cppgc/internal/pointer-policies.h",
     "include/cppgc/internal/prefinalizer-handler.h",
     "include/cppgc/liveness-broker.h",
     "include/cppgc/liveness-broker.h",
     "include/cppgc/macros.h",
     "include/cppgc/member.h",
+    "include/cppgc/persistent.h",
     "include/cppgc/platform.h",
     "include/cppgc/prefinalizer.h",
     "include/cppgc/source-location.h",
@@ -3985,6 +3994,7 @@ v8_source_set("cppgc_base") {
     "src/heap/cppgc/page-memory-inl.h",
     "src/heap/cppgc/page-memory.cc",
     "src/heap/cppgc/page-memory.h",
+    "src/heap/cppgc/persistent-node.cc",
     "src/heap/cppgc/platform.cc",
     "src/heap/cppgc/pointer-policies.cc",
     "src/heap/cppgc/prefinalizer-handler.cc",

include/cppgc/internal/persistent-node.h

@@ -0,0 +1,109 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDE_CPPGC_INTERNAL_PERSISTENT_NODE_H_
+#define INCLUDE_CPPGC_INTERNAL_PERSISTENT_NODE_H_
+
+#include <array>
+#include <memory>
+#include <vector>
+
+#include "include/cppgc/internal/logging.h"
+#include "include/cppgc/trace-trait.h"
+#include "include/v8config.h"
+
+namespace cppgc {
+
+class Visitor;
+
+namespace internal {
+
+// PersistentNode represesents a variant of two states:
+// 1) traceable node with a back pointer to the Persistent object;
+// 2) freelist entry.
+class PersistentNode final {
+ public:
+  PersistentNode() = default;
+
+  PersistentNode(const PersistentNode&) = delete;
+  PersistentNode& operator=(const PersistentNode&) = delete;
+
+  void InitializeAsUsedNode(void* owner, TraceCallback trace) {
+    owner_ = owner;
+    trace_ = trace;
+  }
+
+  void InitializeAsFreeNode(PersistentNode* next) {
+    next_ = next;
+    trace_ = nullptr;
+  }
+
+  void UpdateOwner(void* owner) {
+    CPPGC_DCHECK(IsUsed());
+    owner_ = owner;
+  }
+
+  PersistentNode* FreeListNext() const {
+    CPPGC_DCHECK(!IsUsed());
+    return next_;
+  }
+
+  void Trace(Visitor* visitor) const {
+    CPPGC_DCHECK(IsUsed());
+    trace_(visitor, owner_);
+  }
+
+  bool IsUsed() const { return trace_; }
+
+ private:
+  // PersistentNode acts as a designated union:
+  // If trace_ != nullptr, owner_ points to the corresponding Persistent handle.
+  // Otherwise, next_ points to the next freed PersistentNode.
+  union {
+    void* owner_ = nullptr;
+    PersistentNode* next_;
+  };
+  TraceCallback trace_ = nullptr;
+};
+
+class V8_EXPORT PersistentRegion {
+  using PersistentNodeSlots = std::array<PersistentNode, 256u>;
+
+ public:
+  PersistentRegion() = default;
+
+  PersistentRegion(const PersistentRegion&) = delete;
+  PersistentRegion& operator=(const PersistentRegion&) = delete;
+
+  PersistentNode* AllocateNode(void* owner, TraceCallback trace) {
+    if (!free_list_head_) {
+      EnsureNodeSlots();
+    }
+    PersistentNode* node = free_list_head_;
+    free_list_head_ = free_list_head_->FreeListNext();
+    node->InitializeAsUsedNode(owner, trace);
+    return node;
+  }
+
+  void FreeNode(PersistentNode* node) {
+    node->InitializeAsFreeNode(free_list_head_);
+    free_list_head_ = node;
+  }
+
+  void Trace(Visitor*);
+
+  size_t NodesInUse() const;
+
+ private:
+  void EnsureNodeSlots();
+
+  std::vector<std::unique_ptr<PersistentNodeSlots>> nodes_;
+  PersistentNode* free_list_head_ = nullptr;
+};
+
+}  // namespace internal
+
+}  // namespace cppgc
+
+#endif  // INCLUDE_CPPGC_INTERNAL_PERSISTENT_NODE_H_

include/cppgc/internal/pointer-policies.h

@@ -6,12 +6,16 @@
 #define INCLUDE_CPPGC_INTERNAL_POINTER_POLICIES_H_
 
 #include <cstdint>
+#include <type_traits>
 
+#include "include/cppgc/source-location.h"
 #include "include/v8config.h"
 
 namespace cppgc {
 namespace internal {
 
+class PersistentRegion;
+
 // Tags to distinguish between strong and weak member types.
 class StrongMemberTag;
 class WeakMemberTag;
@@ -52,6 +56,61 @@ using DefaultCheckingPolicy = EnabledCheckingPolicy;
 using DefaultCheckingPolicy = DisabledCheckingPolicy;
 #endif
 
+class KeepLocationPolicy {
+ public:
+  constexpr const SourceLocation& Location() const { return location_; }
+
+ protected:
+  constexpr explicit KeepLocationPolicy(const SourceLocation& location)
+      : location_(location) {}
+
+  // KeepLocationPolicy must not copy underlying source locations.
+  KeepLocationPolicy(const KeepLocationPolicy&) = delete;
+  KeepLocationPolicy& operator=(const KeepLocationPolicy&) = delete;
+
+  // Location of the original moved from object should be preserved.
+  KeepLocationPolicy(KeepLocationPolicy&&) = default;
+  KeepLocationPolicy& operator=(KeepLocationPolicy&&) = default;
+
+ private:
+  SourceLocation location_;
+};
+
+class IgnoreLocationPolicy {
+ public:
+  constexpr SourceLocation Location() const { return {}; }
+
+ protected:
+  constexpr explicit IgnoreLocationPolicy(const SourceLocation&) {}
+};
+
+#if CPPGC_SUPPORTS_OBJECT_NAMES
+using DefaultLocationPolicy = KeepLocationPolicy;
+#else
+using DefaultLocationPolicy = IgnoreLocationPolicy;
+#endif
+
+struct StrongPersistentPolicy {
+  using IsStrongPersistent = std::true_type;
+
+  static V8_EXPORT PersistentRegion& GetPersistentRegion(void* object);
+};
+
+struct WeakPersistentPolicy {
+  using IsStrongPersistent = std::false_type;
+
+  static V8_EXPORT PersistentRegion& GetPersistentRegion(void* object);
+};
+
+// Persistent/Member forward declarations.
+template <typename T, typename WeaknessPolicy,
+          typename LocationPolicy = DefaultLocationPolicy,
+          typename CheckingPolicy = DefaultCheckingPolicy>
+class BasicPersistent;
+template <typename T, typename WeaknessTag, typename WriteBarrierPolicy,
+          typename CheckingPolicy = DefaultCheckingPolicy>
+class BasicMember;
+
 // Special tag type used to denote some sentinel member. The semantics of the
 // sentinel is defined by the embedder.
 struct SentinelPointer {

include/cppgc/member.h

@@ -21,13 +21,15 @@ namespace internal {
 
 // The basic class from which all Member classes are 'generated'.
 template <typename T, typename WeaknessTag, typename WriteBarrierPolicy,
-          typename CheckingPolicy = DefaultCheckingPolicy>
+          typename CheckingPolicy>
 class BasicMember : private CheckingPolicy {
  public:
+  using PointeeType = T;
+
   constexpr BasicMember() = default;
-  constexpr BasicMember(std::nullptr_t) {}  // NOLINT
+  constexpr BasicMember(std::nullptr_t) {}     // NOLINT
   BasicMember(SentinelPointer s) : raw_(s) {}  // NOLINT
-  BasicMember(T* raw) : raw_(raw) {  // NOLINT
+  BasicMember(T* raw) : raw_(raw) {            // NOLINT
     InitializingWriteBarrier();
     this->CheckPointer(raw_);
   }
@@ -41,6 +43,16 @@ class BasicMember : private CheckingPolicy {
       const BasicMember<U, OtherWeaknessTag, OtherBarrierPolicy,
                         OtherCheckingPolicy>& other)
       : BasicMember(other.Get()) {}
+  // Construction from Persistent.
+  template <typename U, typename PersistentWeaknessPolicy,
+            typename PersistentLocationPolicy,
+            typename PersistentCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicMember(  // NOLINT
+      const BasicPersistent<U, PersistentWeaknessPolicy,
+                            PersistentLocationPolicy, PersistentCheckingPolicy>&
+          p)
+      : BasicMember(p.Get()) {}
 
   BasicMember& operator=(const BasicMember& other) {
     return operator=(other.Get());
@@ -54,6 +66,17 @@ class BasicMember : private CheckingPolicy {
                         OtherCheckingPolicy>& other) {
     return operator=(other.Get());
   }
+  // Assignment from Persistent.
+  template <typename U, typename PersistentWeaknessPolicy,
+            typename PersistentLocationPolicy,
+            typename PersistentCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicMember& operator=(
+      const BasicPersistent<U, PersistentWeaknessPolicy,
+                            PersistentLocationPolicy, PersistentCheckingPolicy>&
+          other) {
+    return operator=(other.Get());
+  }
   BasicMember& operator=(T* other) {
     SetRawAtomic(other);
     AssigningWriteBarrier();

include/cppgc/persistent.h

@@ -0,0 +1,304 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDE_CPPGC_PERSISTENT_H_
+#define INCLUDE_CPPGC_PERSISTENT_H_
+
+#include <type_traits>
+
+#include "include/cppgc/internal/persistent-node.h"
+#include "include/cppgc/internal/pointer-policies.h"
+#include "include/cppgc/source-location.h"
+#include "include/cppgc/type-traits.h"
+#include "include/cppgc/visitor.h"
+#include "include/v8config.h"
+
+namespace cppgc {
+namespace internal {
+
+// The basic class from which all Persistent classes are generated.
+template <typename T, typename WeaknessPolicy, typename LocationPolicy,
+          typename CheckingPolicy>
+class BasicPersistent : public LocationPolicy,
+                        private WeaknessPolicy,
+                        private CheckingPolicy {
+ public:
+  using typename WeaknessPolicy::IsStrongPersistent;
+  using PointeeType = T;
+
+  // Null-state/sentinel constructors.
+  BasicPersistent(  // NOLINT
+      const SourceLocation& loc = SourceLocation::Current())
+      : LocationPolicy(loc) {}
+
+  BasicPersistent(std::nullptr_t,  // NOLINT
+                  const SourceLocation& loc = SourceLocation::Current())
+      : LocationPolicy(loc) {}
+
+  BasicPersistent(  // NOLINT
+      SentinelPointer s, const SourceLocation& loc = SourceLocation::Current())
+      : LocationPolicy(loc), raw_(s) {}
+
+  // Raw value contstructors.
+  BasicPersistent(T* raw,  // NOLINT
+                  const SourceLocation& loc = SourceLocation::Current())
+      : LocationPolicy(loc), raw_(raw) {
+    if (!IsValid()) return;
+    node_ = WeaknessPolicy::GetPersistentRegion(raw_).AllocateNode(
+        this, &BasicPersistent::Trace);
+    this->CheckPointer(Get());
+  }
+
+  BasicPersistent(T& raw,  // NOLINT
+                  const SourceLocation& loc = SourceLocation::Current())
+      : BasicPersistent(&raw, loc) {}
+
+  // Copy ctor.
+  BasicPersistent(const BasicPersistent& other,
+                  const SourceLocation& loc = SourceLocation::Current())
+      : BasicPersistent(other.Get(), loc) {}
+
+  // Heterogeneous ctor.
+  template <typename U, typename OtherWeaknessPolicy,
+            typename OtherLocationPolicy, typename OtherCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicPersistent(  // NOLINT
+      const BasicPersistent<U, OtherWeaknessPolicy, OtherLocationPolicy,
+                            OtherCheckingPolicy>& other,
+      const SourceLocation& loc = SourceLocation::Current())
+      : BasicPersistent(other.Get(), loc) {}
+
+  // Move ctor. The heterogeneous move ctor is not supported since e.g.
+  // persistent can't reuse persistent node from weak persistent.
+  BasicPersistent(
+      BasicPersistent&& other,
+      const SourceLocation& loc = SourceLocation::Current()) noexcept
+      : LocationPolicy(std::move(other)),
+        raw_(std::move(other.raw_)),
+        node_(std::move(other.node_)) {
+    if (!IsValid()) return;
+    node_->UpdateOwner(this);
+    other.raw_ = nullptr;
+    other.node_ = nullptr;
+    this->CheckPointer(Get());
+  }
+
+  // Constructor from member.
+  template <typename U, typename MemberBarrierPolicy,
+            typename MemberWeaknessTag, typename MemberCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicPersistent(internal::BasicMember<U, MemberBarrierPolicy,  // NOLINT
+                                        MemberWeaknessTag, MemberCheckingPolicy>
+                      member,
+                  const SourceLocation& loc = SourceLocation::Current())
+      : BasicPersistent(member.Get(), loc) {}
+
+  ~BasicPersistent() { Clear(); }
+
+  // Copy assignment.
+  BasicPersistent& operator=(const BasicPersistent& other) {
+    return operator=(other.Get());
+  }
+
+  template <typename U, typename OtherWeaknessPolicy,
+            typename OtherLocationPolicy, typename OtherCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicPersistent& operator=(
+      const BasicPersistent<U, OtherWeaknessPolicy, OtherLocationPolicy,
+                            OtherCheckingPolicy>& other) {
+    return operator=(other.Get());
+  }
+
+  // Move assignment.
+  BasicPersistent& operator=(BasicPersistent&& other) {
+    if (this == &other) return *this;
+    Clear();
+    LocationPolicy::operator=(std::move(other));
+    raw_ = std::move(other.raw_);
+    node_ = std::move(other.node_);
+    if (!IsValid()) return *this;
+    node_->UpdateOwner(this);
+    other.raw_ = nullptr;
+    other.node_ = nullptr;
+    this->CheckPointer(Get());
+    return *this;
+  }
+
+  // Assignment from member.
+  template <typename U, typename MemberBarrierPolicy,
+            typename MemberWeaknessTag, typename MemberCheckingPolicy,
+            typename = std::enable_if_t<std::is_base_of<T, U>::value>>
+  BasicPersistent& operator=(
+      internal::BasicMember<U, MemberBarrierPolicy, MemberWeaknessTag,
+                            MemberCheckingPolicy>
+          member) {
+    return operator=(member.Get());
+  }
+
+  BasicPersistent& operator=(T* other) {
+    Assign(other);
+    return *this;
+  }
+
+  BasicPersistent& operator=(std::nullptr_t) {
+    Clear();
+    return *this;
+  }
+
+  BasicPersistent& operator=(SentinelPointer s) {
+    Assign(s);
+    return *this;
+  }
+
+  explicit operator bool() const { return Get(); }
+  operator T*() const { return Get(); }
+  T* operator->() const { return Get(); }
+  T& operator*() const { return *Get(); }
+
+  T* Get() const { return raw_; }
+
+  void Clear() { Assign(nullptr); }
+
+  T* Release() {
+    T* result = Get();
+    Clear();
+    return result;
+  }
+
+ private:
+  static void Trace(Visitor* v, const void* ptr) {
+    const auto* persistent = static_cast<const BasicPersistent*>(ptr);
+    v->TraceRoot(*persistent, persistent->Location());
+  }
+
+  bool IsValid() const {
+    // Ideally, handling kSentinelPointer would be done by the embedder. On the
+    // other hand, having Persistent aware of it is beneficial since no node
+    // gets wasted.
+    return raw_ != nullptr && raw_ != kSentinelPointer;
+  }
+
+  void Assign(T* ptr) {
+    if (IsValid()) {
+      if (ptr && ptr != kSentinelPointer) {
+        // Simply assign the pointer reusing the existing node.
+        raw_ = ptr;
+        this->CheckPointer(ptr);
+        return;
+      }
+      WeaknessPolicy::GetPersistentRegion(raw_).FreeNode(node_);
+      node_ = nullptr;
+    }
+    raw_ = ptr;
+    if (!IsValid()) return;
+    node_ = WeaknessPolicy::GetPersistentRegion(raw_).AllocateNode(
+        this, &BasicPersistent::Trace);
+    this->CheckPointer(Get());
+  }
+
+  T* raw_ = nullptr;
+  PersistentNode* node_ = nullptr;
+};
+
+template <typename T1, typename WeaknessPolicy1, typename LocationPolicy1,
+          typename CheckingPolicy1, typename T2, typename WeaknessPolicy2,
+          typename LocationPolicy2, typename CheckingPolicy2>
+bool operator==(const BasicPersistent<T1, WeaknessPolicy1, LocationPolicy1,
+                                      CheckingPolicy1>& p1,
+                const BasicPersistent<T2, WeaknessPolicy2, LocationPolicy2,
+                                      CheckingPolicy2>& p2) {
+  return p1.Get() == p2.Get();
+}
+
+template <typename T1, typename WeaknessPolicy1, typename LocationPolicy1,
+          typename CheckingPolicy1, typename T2, typename WeaknessPolicy2,
+          typename LocationPolicy2, typename CheckingPolicy2>
+bool operator!=(const BasicPersistent<T1, WeaknessPolicy1, LocationPolicy1,
+                                      CheckingPolicy1>& p1,
+                const BasicPersistent<T2, WeaknessPolicy2, LocationPolicy2,
+                                      CheckingPolicy2>& p2) {
+  return !(p1 == p2);
+}
+
+template <typename T1, typename PersistentWeaknessPolicy,
+          typename PersistentLocationPolicy, typename PersistentCheckingPolicy,
+          typename T2, typename MemberWriteBarrierPolicy,
+          typename MemberWeaknessTag, typename MemberCheckingPolicy>
+bool operator==(const BasicPersistent<T1, PersistentWeaknessPolicy,
+                                      PersistentLocationPolicy,
+                                      PersistentCheckingPolicy>& p,
+                BasicMember<T2, MemberWeaknessTag, MemberWriteBarrierPolicy,
+                            MemberCheckingPolicy>
+                    m) {
+  return p.Get() == m.Get();
+}
+
+template <typename T1, typename PersistentWeaknessPolicy,
+          typename PersistentLocationPolicy, typename PersistentCheckingPolicy,
+          typename T2, typename MemberWriteBarrierPolicy,
+          typename MemberWeaknessTag, typename MemberCheckingPolicy>
+bool operator!=(const BasicPersistent<T1, PersistentWeaknessPolicy,
+                                      PersistentLocationPolicy,
+                                      PersistentCheckingPolicy>& p,
+                BasicMember<T2, MemberWeaknessTag, MemberWriteBarrierPolicy,
+                            MemberCheckingPolicy>
+                    m) {
+  return !(p == m);
+}
+
+template <typename T1, typename MemberWriteBarrierPolicy,
+          typename MemberWeaknessTag, typename MemberCheckingPolicy,
+          typename T2, typename PersistentWeaknessPolicy,
+          typename PersistentLocationPolicy, typename PersistentCheckingPolicy>
+bool operator==(BasicMember<T2, MemberWeaknessTag, MemberWriteBarrierPolicy,
+                            MemberCheckingPolicy>
+                    m,
+                const BasicPersistent<T1, PersistentWeaknessPolicy,
+                                      PersistentLocationPolicy,
+                                      PersistentCheckingPolicy>& p) {
+  return m.Get() == p.Get();
+}
+
+template <typename T1, typename MemberWriteBarrierPolicy,
+          typename MemberWeaknessTag, typename MemberCheckingPolicy,
+          typename T2, typename PersistentWeaknessPolicy,
+          typename PersistentLocationPolicy, typename PersistentCheckingPolicy>
+bool operator!=(BasicMember<T2, MemberWeaknessTag, MemberWriteBarrierPolicy,
+                            MemberCheckingPolicy>
+                    m,
+                const BasicPersistent<T1, PersistentWeaknessPolicy,
+                                      PersistentLocationPolicy,
+                                      PersistentCheckingPolicy>& p) {
+  return !(m == p);
+}
+
+template <typename T, typename LocationPolicy, typename CheckingPolicy>
+struct IsWeak<BasicPersistent<T, internal::WeakPersistentPolicy, LocationPolicy,
+                              CheckingPolicy>> : std::true_type {};
+}  // namespace internal
+
+/**
+ * Persistent is a way to create a strong pointer from an off-heap object to
+ * another on-heap object. As long as the Persistent handle is alive the GC will
+ * keep the object pointed to alive. The Persistent handle is always a GC root
+ * from the point of view of the GC. Persistent must be constructed and
+ * destructed in the same thread.
+ */
+template <typename T>
+using Persistent =
+    internal::BasicPersistent<T, internal::StrongPersistentPolicy>;
+
+/**
+ * WeakPersistent is a way to create a weak pointer from an off-heap object to
+ * an on-heap object. The pointer is automatically cleared when the pointee gets
+ * collected. WeakPersistent must be constructed and destructed in the same
+ * thread.
+ */
+template <typename T>
+using WeakPersistent =
+    internal::BasicPersistent<T, internal::WeakPersistentPolicy>;
+
+}  // namespace cppgc
+
+#endif  // INCLUDE_CPPGC_PERSISTENT_H_

include/cppgc/visitor.h

@@ -10,6 +10,7 @@
 #include "include/cppgc/internal/pointer-policies.h"
 #include "include/cppgc/liveness-broker.h"
 #include "include/cppgc/member.h"
+#include "include/cppgc/source-location.h"
 #include "include/cppgc/trace-trait.h"
 
 namespace cppgc {
@@ -17,8 +18,6 @@ namespace internal {
 class VisitorBase;
 }  // namespace internal
 
-class Visitor;
-
 using WeakCallback = void (*)(const LivenessBroker&, const void*);
 
 /**
@@ -50,24 +49,55 @@ class Visitor {
     // TODO(chromium:1056170): DCHECK (or similar) for deleted values as they
     // should come in at a different path.
     VisitWeak(value, TraceTrait<T>::GetTraceDescriptor(value),
-              &HandleWeakMember<T>, &weak_member);
+              &HandleWeak<WeakMember<T>>, &weak_member);
+  }
+
+  template <typename Persistent,
+            std::enable_if_t<Persistent::IsStrongPersistent::value>* = nullptr>
+  void TraceRoot(const Persistent& p, const SourceLocation& loc) {
+    using PointeeType = typename Persistent::PointeeType;
+    static_assert(sizeof(PointeeType),
+                  "Persistent's pointee type must be fully defined");
+    static_assert(internal::IsGarbageCollectedType<PointeeType>::value,
+                  "Persisent's pointee type must be GarabgeCollected or "
+                  "GarbageCollectedMixin");
+    if (!p.Get()) {
+      return;
+    }
+    VisitRoot(p.Get(), TraceTrait<PointeeType>::GetTraceDescriptor(p.Get()),
+              loc);
+  }
+
+  template <typename Persistent,
+            std::enable_if_t<!Persistent::IsStrongPersistent::value>* = nullptr>
+  void TraceRoot(const Persistent& p, const SourceLocation& loc) {
+    using PointeeType = typename Persistent::PointeeType;
+    static_assert(sizeof(PointeeType),
+                  "Persistent's pointee type must be fully defined");
+    static_assert(internal::IsGarbageCollectedType<PointeeType>::value,
+                  "Persisent's pointee type must be GarabgeCollected or "
+                  "GarbageCollectedMixin");
+    VisitWeakRoot(&p, &HandleWeak<Persistent>);
   }
 
  protected:
   virtual void Visit(const void* self, TraceDescriptor) {}
   virtual void VisitWeak(const void* self, TraceDescriptor, WeakCallback,
                          const void* weak_member) {}
+  virtual void VisitRoot(const void*, TraceDescriptor,
+                         const SourceLocation& loc) {}
+  virtual void VisitWeakRoot(const void*, WeakCallback) {}
 
  private:
-  template <typename T>
-  static void HandleWeakMember(const LivenessBroker& info, const void* object) {
-    const WeakMember<T>* weak_member =
-        reinterpret_cast<const WeakMember<T>*>(object);
-    if (!info.IsHeapObjectAlive(*weak_member)) {
+  template <typename PointerType>
+  static void HandleWeak(const LivenessBroker& info, const void* object) {
+    const PointerType* weak = static_cast<const PointerType*>(object);
+    const auto* raw = weak->Get();
+    if (raw && !info.IsHeapObjectAlive(raw)) {
       // Object is passed down through the marker as const. Alternatives are
       // - non-const Trace method;
       // - mutable pointer in MemberBase;
-      const_cast<WeakMember<T>*>(weak_member)->Clear();
+      const_cast<PointerType*>(weak)->Clear();
     }
   }
 

src/heap/cppgc/heap.h

@@ -10,6 +10,7 @@
 
 #include "include/cppgc/heap.h"
 #include "include/cppgc/internal/gc-info.h"
+#include "include/cppgc/internal/persistent-node.h"
 #include "include/cppgc/liveness-broker.h"
 #include "src/heap/cppgc/heap-object-header.h"
 #include "src/heap/cppgc/prefinalizer-handler.h"
@@ -75,6 +76,19 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
     return prefinalizer_handler_.get();
   }
 
+  PersistentRegion& GetStrongPersistentRegion() {
+    return strong_persistent_region_;
+  }
+  const PersistentRegion& GetStrongPersistentRegion() const {
+    return strong_persistent_region_;
+  }
+  PersistentRegion& GetWeakPersistentRegion() {
+    return weak_persistent_region_;
+  }
+  const PersistentRegion& GetWeakPersistentRegion() const {
+    return weak_persistent_region_;
+  }
+
  private:
   // TODO(chromium:1056170): Remove as soon as arenas are available for
   // allocation.
@@ -106,6 +120,9 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
   std::vector<HeapObjectHeader*> objects_;
   std::unique_ptr<PreFinalizerHandler> prefinalizer_handler_;
 
+  PersistentRegion strong_persistent_region_;
+  PersistentRegion weak_persistent_region_;
+
   size_t no_gc_scope_ = 0;
   size_t no_allocation_scope_ = 0;
 };

src/heap/cppgc/persistent-node.cc

@@ -0,0 +1,60 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "include/cppgc/internal/persistent-node.h"
+
+#include <algorithm>
+#include <numeric>
+
+namespace cppgc {
+namespace internal {
+
+size_t PersistentRegion::NodesInUse() const {
+  return std::accumulate(
+      nodes_.cbegin(), nodes_.cend(), 0u, [](size_t acc, const auto& slots) {
+        return acc + std::count_if(slots->cbegin(), slots->cend(),
+                                   [](const PersistentNode& node) {
+                                     return node.IsUsed();
+                                   });
+      });
+}
+
+void PersistentRegion::EnsureNodeSlots() {
+  nodes_.push_back(std::make_unique<PersistentNodeSlots>());
+  for (auto& node : *nodes_.back()) {
+    node.InitializeAsFreeNode(free_list_head_);
+    free_list_head_ = &node;
+  }
+}
+
+void PersistentRegion::Trace(Visitor* visitor) {
+  free_list_head_ = nullptr;
+  for (auto& slots : nodes_) {
+    bool is_empty = true;
+    for (auto& node : *slots) {
+      if (node.IsUsed()) {
+        node.Trace(visitor);
+        is_empty = false;
+      } else {
+        node.InitializeAsFreeNode(free_list_head_);
+        free_list_head_ = &node;
+      }
+    }
+    if (is_empty) {
+      PersistentNode* first_next = (*slots)[0].FreeListNext();
+      // First next was processed first in the loop above, guaranteeing that it
+      // either points to null or into a different node block.
+      CPPGC_DCHECK(!first_next || first_next < &slots->front() ||
+                   first_next > &slots->back());
+      free_list_head_ = first_next;
+      slots.reset();
+    }
+  }
+  nodes_.erase(std::remove_if(nodes_.begin(), nodes_.end(),
+                              [](const auto& ptr) { return !ptr; }),
+               nodes_.end());
+}
+
+}  // namespace internal
+}  // namespace cppgc

src/heap/cppgc/pointer-policies.cc

@@ -3,8 +3,11 @@
 // found in the LICENSE file.
 
 #include "include/cppgc/internal/pointer-policies.h"
+#include "include/cppgc/internal/persistent-node.h"
 
+#include "include/cppgc/internal/accessors.h"
 #include "src/base/macros.h"
+#include "src/heap/cppgc/heap.h"
 
 namespace cppgc {
 namespace internal {
@@ -18,5 +21,15 @@ void EnabledCheckingPolicy::CheckPointer(const void* ptr) {
   // TODO(chromium:1056170): Provide implementation.
 }
 
+PersistentRegion& StrongPersistentPolicy::GetPersistentRegion(void* object) {
+  auto* heap = Heap::From(GetHeapFromPayload(object));
+  return heap->GetStrongPersistentRegion();
+}
+
+PersistentRegion& WeakPersistentPolicy::GetPersistentRegion(void* object) {
+  auto* heap = Heap::From(GetHeapFromPayload(object));
+  return heap->GetWeakPersistentRegion();
+}
+
 }  // namespace internal
 }  // namespace cppgc

test/unittests/BUILD.gn

@@ -53,6 +53,7 @@ v8_source_set("cppgc_unittests_sources") {
     "heap/cppgc/logging_unittest.cc",
     "heap/cppgc/member_unittests.cc",
     "heap/cppgc/page-memory_unittest.cc",
+    "heap/cppgc/persistent_unittests.cc",
     "heap/cppgc/prefinalizer_unittests.cc",
     "heap/cppgc/source-location_unittest.cc",
     "heap/cppgc/stack_unittest.cc",

test/unittests/heap/cppgc/member_unittests.cc

@@ -8,6 +8,7 @@
 #include "include/cppgc/allocation.h"
 #include "include/cppgc/garbage-collected.h"
 #include "include/cppgc/member.h"
+#include "include/cppgc/persistent.h"
 #include "include/cppgc/type-traits.h"
 
 #include "test/unittests/heap/cppgc/tests.h"
@@ -18,8 +19,12 @@ namespace internal {
 
 namespace {
 
-struct GCed : GarbageCollected<GCed> {};
-struct DerivedGCed : GCed {};
+struct GCed : GarbageCollected<GCed> {
+  virtual void Trace(cppgc::Visitor*) {}
+};
+struct DerivedGCed : GCed {
+  void Trace(cppgc::Visitor* v) override { GCed::Trace(v); }
+};
 
 // Compile tests.
 static_assert(!IsWeakV<Member<GCed>>, "Member is always strong.");
@@ -60,15 +65,15 @@ class MemberTest : public testing::TestSupportingAllocationOnly {};
 
 }  // namespace
 
-template <template <typename> class Member>
+template <template <typename> class MemberType>
 void EmptyTest() {
   {
-    Member<GCed> empty;
+    MemberType<GCed> empty;
     EXPECT_EQ(nullptr, empty.Get());
     EXPECT_EQ(nullptr, empty.Release());
   }
   {
-    Member<GCed> empty = nullptr;
+    MemberType<GCed> empty = nullptr;
     EXPECT_EQ(nullptr, empty.Get());
     EXPECT_EQ(nullptr, empty.Release());
   }
@@ -80,9 +85,9 @@ TEST_F(MemberTest, Empty) {
   EmptyTest<UntracedMember>();
 }
 
-template <template <typename> class Member>
+template <template <typename> class MemberType>
 void ClearTest(cppgc::Heap* heap) {
-  Member<GCed> member = MakeGarbageCollected<GCed>(heap);
+  MemberType<GCed> member = MakeGarbageCollected<GCed>(heap);
   EXPECT_NE(nullptr, member.Get());
   member.Clear();
   EXPECT_EQ(nullptr, member.Get());
@@ -95,10 +100,10 @@ TEST_F(MemberTest, Clear) {
   ClearTest<UntracedMember>(heap);
 }
 
-template <template <typename> class Member>
+template <template <typename> class MemberType>
 void ReleaseTest(cppgc::Heap* heap) {
   GCed* gced = MakeGarbageCollected<GCed>(heap);
-  Member<GCed> member = gced;
+  MemberType<GCed> member = gced;
   EXPECT_NE(nullptr, member.Get());
   GCed* raw = member.Release();
   EXPECT_EQ(gced, raw);
@@ -112,12 +117,13 @@ TEST_F(MemberTest, Release) {
   ReleaseTest<UntracedMember>(heap);
 }
 
-template <template <typename> class Member1, template <typename> class Member2>
+template <template <typename> class MemberType1,
+          template <typename> class MemberType2>
 void SwapTest(cppgc::Heap* heap) {
   GCed* gced1 = MakeGarbageCollected<GCed>(heap);
   GCed* gced2 = MakeGarbageCollected<GCed>(heap);
-  Member1<GCed> member1 = gced1;
-  Member2<GCed> member2 = gced2;
+  MemberType1<GCed> member1 = gced1;
+  MemberType2<GCed> member2 = gced2;
   EXPECT_EQ(gced1, member1.Get());
   EXPECT_EQ(gced2, member2.Get());
   member1.Swap(member2);
@@ -138,27 +144,28 @@ TEST_F(MemberTest, Swap) {
   SwapTest<UntracedMember, UntracedMember>(heap);
 }
 
-template <template <typename> class Member1, template <typename> class Member2>
+template <template <typename> class MemberType1,
+          template <typename> class MemberType2>
 void HeterogeneousConversionTest(cppgc::Heap* heap) {
   {
-    Member1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
-    Member2<GCed> member2 = member1;
+    MemberType1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
+    MemberType2<GCed> member2 = member1;
     EXPECT_EQ(member1.Get(), member2.Get());
   }
   {
-    Member1<DerivedGCed> member1 = MakeGarbageCollected<DerivedGCed>(heap);
-    Member2<GCed> member2 = member1;
+    MemberType1<DerivedGCed> member1 = MakeGarbageCollected<DerivedGCed>(heap);
+    MemberType2<GCed> member2 = member1;
     EXPECT_EQ(member1.Get(), member2.Get());
   }
   {
-    Member1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
-    Member2<GCed> member2;
+    MemberType1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
+    MemberType2<GCed> member2;
     member2 = member1;
     EXPECT_EQ(member1.Get(), member2.Get());
   }
   {
-    Member1<DerivedGCed> member1 = MakeGarbageCollected<DerivedGCed>(heap);
-    Member2<GCed> member2;
+    MemberType1<DerivedGCed> member1 = MakeGarbageCollected<DerivedGCed>(heap);
+    MemberType2<GCed> member2;
     member2 = member1;
     EXPECT_EQ(member1.Get(), member2.Get());
   }
@@ -177,12 +184,52 @@ TEST_F(MemberTest, HeterogeneousInterface) {
   HeterogeneousConversionTest<UntracedMember, UntracedMember>(heap);
 }
 
-template <template <typename> class Member1, template <typename> class Member2>
+template <template <typename> class MemberType,
+          template <typename> class PersistentType>
+void PersistentConversionTest(cppgc::Heap* heap) {
+  {
+    PersistentType<GCed> persistent = MakeGarbageCollected<GCed>(heap);
+    MemberType<GCed> member = persistent;
+    EXPECT_EQ(persistent.Get(), member.Get());
+  }
+  {
+    PersistentType<DerivedGCed> persistent =
+        MakeGarbageCollected<DerivedGCed>(heap);
+    MemberType<GCed> member = persistent;
+    EXPECT_EQ(persistent.Get(), member.Get());
+  }
+  {
+    PersistentType<GCed> persistent = MakeGarbageCollected<GCed>(heap);
+    MemberType<GCed> member;
+    member = persistent;
+    EXPECT_EQ(persistent.Get(), member.Get());
+  }
+  {
+    PersistentType<DerivedGCed> persistent =
+        MakeGarbageCollected<DerivedGCed>(heap);
+    MemberType<GCed> member;
+    member = persistent;
+    EXPECT_EQ(persistent.Get(), member.Get());
+  }
+}
+
+TEST_F(MemberTest, PersistentConversion) {
+  cppgc::Heap* heap = GetHeap();
+  PersistentConversionTest<Member, Persistent>(heap);
+  PersistentConversionTest<Member, WeakPersistent>(heap);
+  PersistentConversionTest<WeakMember, Persistent>(heap);
+  PersistentConversionTest<WeakMember, WeakPersistent>(heap);
+  PersistentConversionTest<UntracedMember, Persistent>(heap);
+  PersistentConversionTest<UntracedMember, WeakPersistent>(heap);
+}
+
+template <template <typename> class MemberType1,
+          template <typename> class MemberType2>
 void EqualityTest(cppgc::Heap* heap) {
   {
     GCed* gced = MakeGarbageCollected<GCed>(heap);
-    Member1<GCed> member1 = gced;
-    Member2<GCed> member2 = gced;
+    MemberType1<GCed> member1 = gced;
+    MemberType2<GCed> member2 = gced;
     EXPECT_TRUE(member1 == member2);
     EXPECT_FALSE(member1 != member2);
     member2 = member1;
@@ -190,8 +237,8 @@ void EqualityTest(cppgc::Heap* heap) {
     EXPECT_FALSE(member1 != member2);
   }
   {
-    Member1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
-    Member2<GCed> member2 = MakeGarbageCollected<GCed>(heap);
+    MemberType1<GCed> member1 = MakeGarbageCollected<GCed>(heap);
+    MemberType2<GCed> member2 = MakeGarbageCollected<GCed>(heap);
     EXPECT_TRUE(member1 != member2);
     EXPECT_FALSE(member1 == member2);
   }
@@ -229,6 +276,7 @@ TEST_F(MemberTest, WriteBarrierTriggered) {
   EXPECT_EQ(1u, CustomWriteBarrierPolicy::AssigningWriteBarriersTriggered);
   member2 = kSentinelPointer;
   EXPECT_EQ(kSentinelPointer, member2.Get());
+  EXPECT_EQ(kSentinelPointer, member2);
   // No initializing barriers for pointer sentinel.
   EXPECT_EQ(1u, CustomWriteBarrierPolicy::InitializingWriteBarriersTriggered);
   EXPECT_EQ(1u, CustomWriteBarrierPolicy::AssigningWriteBarriersTriggered);

test/unittests/heap/cppgc/persistent_unittests.cc

@@ -0,0 +1,658 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <vector>
+
+#include "include/cppgc/allocation.h"
+#include "include/cppgc/garbage-collected.h"
+#include "include/cppgc/member.h"
+#include "include/cppgc/persistent.h"
+#include "include/cppgc/type-traits.h"
+#include "src/heap/cppgc/heap.h"
+#include "src/heap/cppgc/visitor.h"
+
+#include "test/unittests/heap/cppgc/tests.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+
+struct GCed : GarbageCollected<GCed> {
+  static size_t trace_call_count;
+  virtual void Trace(cppgc::Visitor*) { ++trace_call_count; }
+};
+size_t GCed::trace_call_count = 0;
+
+struct DerivedGCed : GCed {
+  void Trace(cppgc::Visitor* v) override { GCed::Trace(v); }
+};
+
+template <template <typename> class PersistentType>
+PersistentRegion& GetRegion(cppgc::Heap* heap) {
+  auto* heap_impl = internal::Heap::From(heap);
+  return IsWeak<PersistentType<GCed>>::value
+             ? heap_impl->GetWeakPersistentRegion()
+             : heap_impl->GetStrongPersistentRegion();
+}
+
+template <typename T>
+using LocalizedPersistent =
+    internal::BasicPersistent<T, internal::StrongPersistentPolicy,
+                              internal::KeepLocationPolicy,
+                              internal::DefaultCheckingPolicy>;
+
+class RootVisitor final : public VisitorBase {
+ public:
+  RootVisitor() = default;
+
+  const auto& WeakCallbacks() const { return weak_callbacks_; }
+
+  void ProcessWeakCallbacks() {
+    const auto info = LivenessBrokerFactory::Create();
+    for (const auto& cb : weak_callbacks_) {
+      cb.first(info, cb.second);
+    }
+    weak_callbacks_.clear();
+  }
+
+ protected:
+  void VisitRoot(const void* t, TraceDescriptor desc,
+                 const SourceLocation&) final {
+    desc.callback(this, desc.base_object_payload);
+  }
+  void VisitWeakRoot(const void* object, WeakCallback callback) final {
+    weak_callbacks_.emplace_back(callback, object);
+  }
+
+ private:
+  std::vector<std::pair<WeakCallback, const void*>> weak_callbacks_;
+};
+class PersistentTest : public testing::TestSupportingAllocationOnly {};
+
+}  // namespace
+
+template <template <typename> class PersistentType>
+void NullStateCtor(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  {
+    PersistentType<GCed> empty;
+    EXPECT_EQ(nullptr, empty.Get());
+    EXPECT_EQ(nullptr, empty.Release());
+    EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  {
+    PersistentType<GCed> empty = nullptr;
+    EXPECT_EQ(nullptr, empty.Get());
+    EXPECT_EQ(nullptr, empty.Release());
+    EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  {
+    PersistentType<GCed> empty = kSentinelPointer;
+    EXPECT_EQ(kSentinelPointer, empty);
+    EXPECT_EQ(kSentinelPointer, empty.Release());
+    EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  {
+    // Runtime null must not allocated associated node.
+    PersistentType<GCed> empty = static_cast<GCed*>(0);
+    EXPECT_EQ(nullptr, empty.Get());
+    EXPECT_EQ(nullptr, empty.Release());
+    EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, NullStateCtor) {
+  auto* heap = GetHeap();
+  NullStateCtor<Persistent>(heap);
+  NullStateCtor<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void RawCtor(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  GCed* gced = MakeGarbageCollected<GCed>(heap);
+  {
+    PersistentType<GCed> p = gced;
+    EXPECT_EQ(gced, p.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p = *gced;
+    EXPECT_EQ(gced, p.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, RawCtor) {
+  auto* heap = GetHeap();
+  RawCtor<Persistent>(heap);
+  RawCtor<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void CopyCtor(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1 = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    PersistentType<GCed> p2 = p1;
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1;
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    PersistentType<GCed> p2 = p1;
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(nullptr, p1.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<DerivedGCed> p1 = MakeGarbageCollected<DerivedGCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    PersistentType<GCed> p2 = p1;
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    static constexpr size_t kSlots = 512u;
+    const PersistentType<GCed> prototype = MakeGarbageCollected<GCed>(heap);
+    std::vector<PersistentType<GCed>> vector;
+    vector.reserve(kSlots);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    for (size_t i = 0; i < kSlots; ++i) {
+      vector.emplace_back(prototype);
+      EXPECT_EQ(i + 2, GetRegion<PersistentType>(heap).NodesInUse());
+    }
+    vector.clear();
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, CopyCtor) {
+  auto* heap = GetHeap();
+  CopyCtor<Persistent>(heap);
+  CopyCtor<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void MoveCtor(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    GCed* gced = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p1 = gced;
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    PersistentType<GCed> p2 = std::move(p1);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(gced, p2.Get());
+    // Moved-from-object is in the valid specified (nullptr) state.
+    EXPECT_EQ(nullptr, p1.Get());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<DerivedGCed> p1 = MakeGarbageCollected<DerivedGCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    // Move ctor is not heterogeneous - fall back to copy ctor.
+    PersistentType<GCed> p2 = std::move(p1);
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1;
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    PersistentType<GCed> p2 = std::move(p1);
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(nullptr, p1.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, MoveCtor) {
+  auto* heap = GetHeap();
+  MoveCtor<Persistent>(heap);
+  MoveCtor<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType,
+          template <typename> class MemberType>
+void MemberCtor(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    MemberType<GCed> m = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p = m;
+    EXPECT_EQ(m.Get(), p.Get());
+    EXPECT_EQ(m, p);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, MemberCtor) {
+  auto* heap = GetHeap();
+  MemberCtor<Persistent, Member>(heap);
+  MemberCtor<Persistent, WeakMember>(heap);
+  MemberCtor<Persistent, UntracedMember>(heap);
+  MemberCtor<WeakPersistent, Member>(heap);
+  MemberCtor<WeakPersistent, WeakMember>(heap);
+  MemberCtor<WeakPersistent, UntracedMember>(heap);
+}
+
+template <template <typename> class PersistentType>
+void NullStateAssignemnt(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p = nullptr;
+    EXPECT_EQ(nullptr, p.Get());
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  {
+    PersistentType<GCed> p = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p = kSentinelPointer;
+    EXPECT_EQ(kSentinelPointer, p);
+    EXPECT_EQ(kSentinelPointer, p.Get());
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  {
+    PersistentType<GCed> p = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p = static_cast<GCed*>(0);
+    EXPECT_EQ(nullptr, p.Get());
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+}
+
+TEST_F(PersistentTest, NullStateAssignemnt) {
+  auto* heap = GetHeap();
+  NullStateAssignemnt<Persistent>(heap);
+  NullStateAssignemnt<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void RawAssignment(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  GCed* gced = MakeGarbageCollected<GCed>(heap);
+  {
+    PersistentType<GCed> p;
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    p = gced;
+    EXPECT_EQ(gced, p.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p;
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    p = *gced;
+    EXPECT_EQ(gced, p.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, RawAssignment) {
+  auto* heap = GetHeap();
+  RawAssignment<Persistent>(heap);
+  RawAssignment<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void CopyAssignment(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1 = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p2;
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = p1;
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1 = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p2 = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = p1;
+    // The old node from p2 must be dropped.
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<DerivedGCed> p1 = MakeGarbageCollected<DerivedGCed>(heap);
+    PersistentType<GCed> p2;
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = p1;
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    static constexpr size_t kSlots = 512u;
+    const PersistentType<GCed> prototype = MakeGarbageCollected<GCed>(heap);
+    std::vector<PersistentType<GCed>> vector(kSlots);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    size_t i = 0;
+    for (auto& p : vector) {
+      p = prototype;
+      EXPECT_EQ(i + 2, GetRegion<PersistentType>(heap).NodesInUse());
+      ++i;
+    }
+    vector.clear();
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, CopyAssignment) {
+  auto* heap = GetHeap();
+  CopyAssignment<Persistent>(heap);
+  CopyAssignment<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void MoveAssignment(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    GCed* gced = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p1 = gced;
+    PersistentType<GCed> p2;
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = std::move(p1);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(gced, p2.Get());
+    // Moved-from-object is in the valid specified (nullptr) state.
+    EXPECT_EQ(nullptr, p1.Get());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<GCed> p1;
+    PersistentType<GCed> p2 = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = std::move(p1);
+    EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+    // Moved-from-object is in the valid specified (nullptr) state.
+    EXPECT_EQ(nullptr, p2.Get());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    GCed* gced = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p1 = gced;
+    PersistentType<GCed> p2 = MakeGarbageCollected<GCed>(heap);
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    p2 = std::move(p1);
+    // The old node from p2 must be dropped.
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(gced, p2.Get());
+    // Moved-from-object is in the valid specified (nullptr) state.
+    EXPECT_EQ(nullptr, p1.Get());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    PersistentType<DerivedGCed> p1 = MakeGarbageCollected<DerivedGCed>(heap);
+    PersistentType<GCed> p2;
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+    // Move ctor is not heterogeneous - fall back to copy assignment.
+    p2 = std::move(p1);
+    EXPECT_EQ(2u, GetRegion<PersistentType>(heap).NodesInUse());
+    EXPECT_EQ(p1.Get(), p2.Get());
+    EXPECT_EQ(p1, p2);
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, MoveAssignment) {
+  auto* heap = GetHeap();
+  MoveAssignment<Persistent>(heap);
+  MoveAssignment<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType,
+          template <typename> class MemberType>
+void MemberAssignment(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  {
+    MemberType<GCed> m = MakeGarbageCollected<GCed>(heap);
+    PersistentType<GCed> p;
+    p = m;
+    EXPECT_EQ(m.Get(), p.Get());
+    EXPECT_EQ(m, p);
+    EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, MemberAssignment) {
+  auto* heap = GetHeap();
+  MemberAssignment<Persistent, Member>(heap);
+  MemberAssignment<Persistent, WeakMember>(heap);
+  MemberAssignment<Persistent, UntracedMember>(heap);
+  MemberAssignment<WeakPersistent, Member>(heap);
+  MemberAssignment<WeakPersistent, WeakMember>(heap);
+  MemberAssignment<WeakPersistent, UntracedMember>(heap);
+}
+
+template <template <typename> class PersistentType>
+void ClearTest(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  PersistentType<GCed> p = MakeGarbageCollected<GCed>(heap);
+  EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  EXPECT_NE(nullptr, p.Get());
+  p.Clear();
+  EXPECT_EQ(nullptr, p.Get());
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, Clear) {
+  auto* heap = GetHeap();
+  ClearTest<Persistent>(heap);
+  ClearTest<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType>
+void ReleaseTest(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+  GCed* gced = MakeGarbageCollected<GCed>(heap);
+  PersistentType<GCed> p = gced;
+  EXPECT_EQ(1u, GetRegion<PersistentType>(heap).NodesInUse());
+  EXPECT_NE(nullptr, p.Get());
+  GCed* raw = p.Release();
+  EXPECT_EQ(gced, raw);
+  EXPECT_EQ(nullptr, p.Get());
+  EXPECT_EQ(0u, GetRegion<PersistentType>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, Release) {
+  auto* heap = GetHeap();
+  ReleaseTest<Persistent>(heap);
+  ReleaseTest<WeakPersistent>(heap);
+}
+
+template <template <typename> class PersistentType1,
+          template <typename> class PersistentType2>
+void HeterogeneousConversion(cppgc::Heap* heap) {
+  EXPECT_EQ(0u, GetRegion<PersistentType1>(heap).NodesInUse());
+  EXPECT_EQ(0u, GetRegion<PersistentType2>(heap).NodesInUse());
+  {
+    PersistentType1<GCed> persistent1 = MakeGarbageCollected<GCed>(heap);
+    PersistentType2<GCed> persistent2 = persistent1;
+    EXPECT_EQ(persistent1.Get(), persistent2.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType1>(heap).NodesInUse());
+    EXPECT_EQ(1u, GetRegion<PersistentType2>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType1>(heap).NodesInUse());
+  EXPECT_EQ(0u, GetRegion<PersistentType2>(heap).NodesInUse());
+  {
+    PersistentType1<DerivedGCed> persistent1 =
+        MakeGarbageCollected<DerivedGCed>(heap);
+    PersistentType2<GCed> persistent2 = persistent1;
+    EXPECT_EQ(persistent1.Get(), persistent2.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType1>(heap).NodesInUse());
+    EXPECT_EQ(1u, GetRegion<PersistentType2>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType1>(heap).NodesInUse());
+  EXPECT_EQ(0u, GetRegion<PersistentType2>(heap).NodesInUse());
+  {
+    PersistentType1<GCed> persistent1 = MakeGarbageCollected<GCed>(heap);
+    PersistentType2<GCed> persistent2;
+    persistent2 = persistent1;
+    EXPECT_EQ(persistent1.Get(), persistent2.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType1>(heap).NodesInUse());
+    EXPECT_EQ(1u, GetRegion<PersistentType2>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType1>(heap).NodesInUse());
+  EXPECT_EQ(0u, GetRegion<PersistentType2>(heap).NodesInUse());
+  {
+    PersistentType1<DerivedGCed> persistent1 =
+        MakeGarbageCollected<DerivedGCed>(heap);
+    PersistentType2<GCed> persistent2;
+    persistent2 = persistent1;
+    EXPECT_EQ(persistent1.Get(), persistent2.Get());
+    EXPECT_EQ(1u, GetRegion<PersistentType1>(heap).NodesInUse());
+    EXPECT_EQ(1u, GetRegion<PersistentType2>(heap).NodesInUse());
+  }
+  EXPECT_EQ(0u, GetRegion<PersistentType1>(heap).NodesInUse());
+  EXPECT_EQ(0u, GetRegion<PersistentType2>(heap).NodesInUse());
+}
+
+TEST_F(PersistentTest, HeterogeneousConversion) {
+  auto* heap = GetHeap();
+  HeterogeneousConversion<Persistent, WeakPersistent>(heap);
+  HeterogeneousConversion<WeakPersistent, Persistent>(heap);
+}
+
+TEST_F(PersistentTest, TraceStrong) {
+  auto* heap = GetHeap();
+  static constexpr size_t kItems = 512;
+  std::vector<Persistent<GCed>> vec(kItems);
+  for (auto& p : vec) {
+    p = MakeGarbageCollected<GCed>(heap);
+  }
+  {
+    GCed::trace_call_count = 0;
+    RootVisitor v;
+    GetRegion<Persistent>(heap).Trace(&v);
+    EXPECT_EQ(kItems, GCed::trace_call_count);
+    EXPECT_EQ(kItems, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  {
+    GCed::trace_call_count = 0;
+    vec[0].Clear();
+    vec[kItems / 2].Clear();
+    vec[kItems / 4].Clear();
+    vec[kItems - 1].Clear();
+    RootVisitor v;
+    GetRegion<Persistent>(heap).Trace(&v);
+    EXPECT_EQ(kItems - 4, GCed::trace_call_count);
+    EXPECT_EQ(kItems - 4, GetRegion<Persistent>(heap).NodesInUse());
+  }
+  {
+    GCed::trace_call_count = 0;
+    vec.clear();
+    RootVisitor v;
+    GetRegion<Persistent>(heap).Trace(&v);
+    EXPECT_EQ(0u, GCed::trace_call_count);
+    EXPECT_EQ(0u, GetRegion<Persistent>(heap).NodesInUse());
+  }
+}
+
+TEST_F(PersistentTest, TraceWeak) {
+  auto* heap = GetHeap();
+  static constexpr size_t kItems = 512;
+  std::vector<WeakPersistent<GCed>> vec(kItems);
+  for (auto& p : vec) {
+    p = MakeGarbageCollected<GCed>(heap);
+  }
+  GCed::trace_call_count = 0;
+  RootVisitor v;
+  GetRegion<WeakPersistent>(heap).Trace(&v);
+  const auto& callbacks = v.WeakCallbacks();
+  EXPECT_EQ(kItems, callbacks.size());
+  EXPECT_EQ(kItems, GetRegion<WeakPersistent>(heap).NodesInUse());
+
+  v.ProcessWeakCallbacks();
+  for (const auto& p : vec) {
+    EXPECT_EQ(nullptr, p.Get());
+  }
+  EXPECT_EQ(0u, GetRegion<WeakPersistent>(heap).NodesInUse());
+}
+
+#if CPPGC_SUPPORTS_SOURCE_LOCATION
+TEST_F(PersistentTest, LocalizedPersistent) {
+  GCed* gced = MakeGarbageCollected<GCed>(GetHeap());
+  {
+    const auto expected_loc = SourceLocation::Current();
+    LocalizedPersistent<GCed> p = gced;
+    const auto actual_loc = p.Location();
+    EXPECT_STREQ(expected_loc.Function(), actual_loc.Function());
+    EXPECT_STREQ(expected_loc.FileName(), actual_loc.FileName());
+    EXPECT_EQ(expected_loc.Line() + 1, actual_loc.Line());
+  }
+  {
+    // Copy ctor doesn't copy source location.
+    LocalizedPersistent<GCed> p1 = gced;
+    LocalizedPersistent<GCed> p2 = p1;
+    EXPECT_STREQ(p1.Location().Function(), p2.Location().Function());
+    EXPECT_STREQ(p1.Location().FileName(), p2.Location().FileName());
+    EXPECT_EQ(p1.Location().Line() + 1, p2.Location().Line());
+  }
+  {
+    // Copy assignment doesn't copy source location.
+    LocalizedPersistent<GCed> p1 = gced;
+    LocalizedPersistent<GCed> p2;
+    p2 = p1;
+    EXPECT_STREQ(p1.Location().Function(), p2.Location().Function());
+    EXPECT_STREQ(p1.Location().FileName(), p2.Location().FileName());
+    EXPECT_EQ(p1.Location().Line() + 1, p2.Location().Line());
+  }
+  {
+    // Clearing doesn't clear source location.
+    LocalizedPersistent<GCed> p1 = gced;
+    LocalizedPersistent<GCed> p2 = gced;
+    p2.Clear();
+    EXPECT_STREQ(p1.Location().Function(), p2.Location().Function());
+    EXPECT_STREQ(p1.Location().FileName(), p2.Location().FileName());
+    EXPECT_EQ(p1.Location().Line() + 1, p2.Location().Line());
+  }
+  {
+    LocalizedPersistent<GCed> p1 = gced;
+    const auto expected_loc = p1.Location();
+    LocalizedPersistent<GCed> p2 = std::move(p1);
+    EXPECT_STREQ(expected_loc.Function(), p2.Location().Function());
+    EXPECT_STREQ(expected_loc.FileName(), p2.Location().FileName());
+    EXPECT_EQ(expected_loc.Line(), p2.Location().Line());
+  }
+  {
+    LocalizedPersistent<GCed> p1 = gced;
+    const auto expected_loc = p1.Location();
+    LocalizedPersistent<GCed> p2;
+    p2 = std::move(p1);
+    EXPECT_STREQ(expected_loc.Function(), p2.Location().Function());
+    EXPECT_STREQ(expected_loc.FileName(), p2.Location().FileName());
+    EXPECT_EQ(expected_loc.Line(), p2.Location().Line());
+  }
+}
+#endif
+
+}  // namespace internal
+}  // namespace cppgc