cppgc-js: Report C++ memory to V8's heap growing

Add reporting of C++ memory to V8's heap growing strategy via
existing EmbedderHeapTracer interface.

In addition, introduce API-level NoGarbageCollectionScope which
allows to temporarily avoid scheduling GC finalizations. Replace
internal NoGCScope with NoGarbageCollectionScope and remove
NoGCScope.

Bug: chromium:1056170
Change-Id: I0ad3dfd67eb81f09f48e2ab87f9bbece7491ed71
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2650210
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72345}

BUILD.gn

@@ -4634,6 +4634,7 @@ v8_source_set("cppgc_base") {
     "src/heap/cppgc/gc-invoker.h",
     "src/heap/cppgc/heap-base.cc",
     "src/heap/cppgc/heap-base.h",
+    "src/heap/cppgc/heap-consistency.cc",
     "src/heap/cppgc/heap-growing.cc",
     "src/heap/cppgc/heap-growing.h",
     "src/heap/cppgc/heap-object-header.cc",

include/cppgc/heap-consistency.h

@@ -8,6 +8,7 @@
 #include <cstddef>
 
 #include "cppgc/internal/write-barrier.h"
+#include "cppgc/macros.h"
 #include "cppgc/trace-trait.h"
 #include "v8config.h"  // NOLINT(build/include_directory)
 
@@ -131,6 +132,49 @@ class HeapConsistency final {
   HeapConsistency() = delete;
 };
 
+/**
+ * Avoids invoking garbage collection finalizations. Already running garbage
+ * collection phase are unaffected by this scope.
+ *
+ * Should only be used temporarily as the scope has an impact on memory usage
+ * and follow up garbage collections.
+ */
+class V8_EXPORT V8_NODISCARD NoGarbageCollectionScope final {
+  CPPGC_STACK_ALLOCATED();
+
+ public:
+  /**
+   * Enters a no garbage collection scope. Must be paired with `Leave()`. Prefer
+   * a scope instance of `NoGarbageCollectionScope`.
+   *
+   * \param heap_handle The corresponding heap.
+   */
+  static void Enter(HeapHandle& heap_handle);
+
+  /**
+   * Leaves a no garbage collection scope. Must be paired with `Enter()`. Prefer
+   * a scope instance of `NoGarbageCollectionScope`.
+   *
+   * \param heap_handle The corresponding heap.
+   */
+  static void Leave(HeapHandle& heap_handle);
+
+  /**
+   * Constructs a scoped object that automatically enters and leaves a no
+   * garbage collection scope based on its lifetime.
+   *
+   * \param heap_handle The corresponding heap.
+   */
+  explicit NoGarbageCollectionScope(HeapHandle& heap_handle);
+  ~NoGarbageCollectionScope();
+
+  NoGarbageCollectionScope(const NoGarbageCollectionScope&) = delete;
+  NoGarbageCollectionScope& operator=(const NoGarbageCollectionScope&) = delete;
+
+ private:
+  HeapHandle& heap_handle_;
+};
+
 }  // namespace subtle
 }  // namespace cppgc
 

src/heap/cppgc-js/cpp-heap.cc

@@ -4,6 +4,7 @@
 
 #include "src/heap/cppgc-js/cpp-heap.h"
 
+#include "include/cppgc/heap-consistency.h"
 #include "include/cppgc/platform.h"
 #include "include/v8-platform.h"
 #include "include/v8.h"
@@ -181,6 +182,7 @@ CppHeap::CppHeap(
     isolate_.heap_profiler()->AddBuildEmbedderGraphCallback(
         &CppGraphBuilder::Run, this);
   }
+  stats_collector()->RegisterObserver(this);
 }
 
 CppHeap::~CppHeap() {
@@ -290,7 +292,7 @@ void CppHeap::TraceEpilogue(TraceSummary* trace_summary) {
 #endif
 
   {
-    NoGCScope no_gc(*this);
+    cppgc::subtle::NoGarbageCollectionScope no_gc(*this);
     cppgc::internal::Sweeper::SweepingConfig::CompactableSpaceHandling
         compactable_space_handling = compactor_.CompactSpacesIfEnabled();
     const cppgc::internal::Sweeper::SweepingConfig sweeping_config{
@@ -302,5 +304,32 @@ void CppHeap::TraceEpilogue(TraceSummary* trace_summary) {
   sweeper().NotifyDoneIfNeeded();
 }
 
+void CppHeap::AllocatedObjectSizeIncreased(size_t bytes) {
+  buffered_allocated_bytes_ += static_cast<int64_t>(bytes);
+  ReportBufferedAllocationSizeIfPossible();
+}
+
+void CppHeap::AllocatedObjectSizeDecreased(size_t bytes) {
+  buffered_allocated_bytes_ -= static_cast<int64_t>(bytes);
+  ReportBufferedAllocationSizeIfPossible();
+}
+
+void CppHeap::ReportBufferedAllocationSizeIfPossible() {
+  // Avoid reporting to V8 in the following conditions as that may trigger GC
+  // finalizations where not allowed.
+  // - Recursive sweeping.
+  // - GC forbidden scope.
+  if (sweeper().IsSweepingOnMutatorThread() || in_no_gc_scope()) {
+    return;
+  }
+
+  if (buffered_allocated_bytes_ < 0) {
+    DecreaseAllocatedSize(static_cast<size_t>(-buffered_allocated_bytes_));
+  } else {
+    IncreaseAllocatedSize(static_cast<size_t>(buffered_allocated_bytes_));
+  }
+  buffered_allocated_bytes_ = 0;
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/cppgc-js/cpp-heap.h

@@ -9,6 +9,7 @@
 #include "include/v8.h"
 #include "src/base/macros.h"
 #include "src/heap/cppgc/heap-base.h"
+#include "src/heap/cppgc/stats-collector.h"
 
 namespace v8 {
 
@@ -17,9 +18,11 @@ class Isolate;
 namespace internal {
 
 // A C++ heap implementation used with V8 to implement unified heap.
-class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
+class V8_EXPORT_PRIVATE CppHeap final
-                                        public v8::CppHeap,
+    : public cppgc::internal::HeapBase,
-                                        public v8::EmbedderHeapTracer {
+      public v8::CppHeap,
+      public v8::EmbedderHeapTracer,
+      public cppgc::internal::StatsCollector::AllocationObserver {
  public:
   static CppHeap* From(v8::CppHeap* heap) {
     return static_cast<CppHeap*>(heap);
@@ -41,6 +44,9 @@ class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
   HeapBase& AsBase() { return *this; }
   const HeapBase& AsBase() const { return *this; }
 
+  void Terminate();
+
+  // v8::EmbedderHeapTracer interface.
   void RegisterV8References(
       const std::vector<std::pair<void*, void*> >& embedder_fields) final;
   void TracePrologue(TraceFlags flags) final;
@@ -49,7 +55,10 @@ class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
   void TraceEpilogue(TraceSummary* trace_summary) final;
   void EnterFinalPause(EmbedderStackState stack_state) final;
 
-  void Terminate();
+  // StatsCollector::AllocationObserver interface.
+  void AllocatedObjectSizeIncreased(size_t) final;
+  void AllocatedObjectSizeDecreased(size_t) final;
+  void ResetAllocatedObjectSize(size_t) final {}
 
  private:
   void FinalizeIncrementalGarbageCollectionIfNeeded(
@@ -58,9 +67,16 @@ class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
     // finalization is not needed) thus this method is left empty.
   }
 
+  void ReportBufferedAllocationSizeIfPossible();
+
   Isolate& isolate_;
   bool marking_done_ = false;
   bool is_in_final_pause_ = false;
+
+  // Buffered allocated bytes. Reporting allocated bytes to V8 can trigger a GC
+  // atomic pause. Allocated bytes are buffer in case this is temporarily
+  // prohibited.
+  int64_t buffered_allocated_bytes_ = 0;
 };
 
 }  // namespace internal

src/heap/cppgc/heap-base.cc

@@ -85,12 +85,6 @@ size_t HeapBase::ObjectPayloadSize() const {
   return ObjectSizeCounter().GetSize(const_cast<RawHeap*>(&raw_heap()));
 }
 
-HeapBase::NoGCScope::NoGCScope(HeapBase& heap) : heap_(heap) {
-  heap_.no_gc_scope_++;
-}
-
-HeapBase::NoGCScope::~NoGCScope() { heap_.no_gc_scope_--; }
-
 void HeapBase::AdvanceIncrementalGarbageCollectionOnAllocationIfNeeded() {
   if (marker_) marker_->AdvanceMarkingOnAllocation();
 }

src/heap/cppgc/heap-base.h

@@ -31,6 +31,9 @@ class Stack;
 }  // namespace heap
 
 namespace cppgc {
+namespace subtle {
+class NoGarbageCollectionScope;
+}  // namespace subtle
 
 class Platform;
 
@@ -55,22 +58,6 @@ class V8_EXPORT_PRIVATE HeapBase : public cppgc::HeapHandle {
  public:
   using StackSupport = cppgc::Heap::StackSupport;
 
-  // NoGCScope allows going over limits and avoids triggering garbage
-  // collection triggered through allocations or even explicitly.
-  class V8_EXPORT_PRIVATE V8_NODISCARD NoGCScope final {
-    CPPGC_STACK_ALLOCATED();
-
-   public:
-    explicit NoGCScope(HeapBase& heap);
-    ~NoGCScope();
-
-    NoGCScope(const NoGCScope&) = delete;
-    NoGCScope& operator=(const NoGCScope&) = delete;
-
-   private:
-    HeapBase& heap_;
-  };
-
   static HeapBase& From(cppgc::HeapHandle& heap_handle) {
     return static_cast<HeapBase&>(heap_handle);
   }
@@ -199,6 +186,7 @@ class V8_EXPORT_PRIVATE HeapBase : public cppgc::HeapHandle {
 
   friend class MarkerBase::IncrementalMarkingTask;
   friend class testing::TestWithHeap;
+  friend class cppgc::subtle::NoGarbageCollectionScope;
 };
 
 }  // namespace internal

src/heap/cppgc/heap-consistency.cc

@@ -0,0 +1,36 @@
+// Copyright 2021 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "include/cppgc/heap-consistency.h"
+
+#include "include/cppgc/heap.h"
+#include "src/base/logging.h"
+#include "src/heap/cppgc/heap-base.h"
+
+namespace cppgc {
+namespace subtle {
+
+// static
+void NoGarbageCollectionScope::Enter(cppgc::HeapHandle& heap_handle) {
+  auto& heap_base = internal::HeapBase::From(heap_handle);
+  heap_base.no_gc_scope_++;
+}
+
+// static
+void NoGarbageCollectionScope::Leave(cppgc::HeapHandle& heap_handle) {
+  auto& heap_base = internal::HeapBase::From(heap_handle);
+  DCHECK_GT(heap_base.no_gc_scope_, 0);
+  heap_base.no_gc_scope_--;
+}
+
+NoGarbageCollectionScope::NoGarbageCollectionScope(
+    cppgc::HeapHandle& heap_handle)
+    : heap_handle_(heap_handle) {
+  Enter(heap_handle);
+}
+
+NoGarbageCollectionScope::~NoGarbageCollectionScope() { Leave(heap_handle_); }
+
+}  // namespace subtle
+}  // namespace cppgc

src/heap/cppgc/heap.cc

@@ -4,6 +4,7 @@
 
 #include "src/heap/cppgc/heap.h"
 
+#include "include/cppgc/heap-consistency.h"
 #include "src/heap/base/stack.h"
 #include "src/heap/cppgc/garbage-collector.h"
 #include "src/heap/cppgc/gc-invoker.h"
@@ -101,7 +102,7 @@ Heap::Heap(std::shared_ptr<cppgc::Platform> platform,
 }
 
 Heap::~Heap() {
-  NoGCScope no_gc(*this);
+  subtle::NoGarbageCollectionScope no_gc(*this);
   // Finish already running GC if any, but don't finalize live objects.
   sweeper_.FinishIfRunning();
 }
@@ -189,7 +190,7 @@ void Heap::FinalizeGarbageCollection(Config::StackState stack_state) {
   verifier.Run(stack_state);
 #endif
 
-  NoGCScope no_gc(*this);
+  subtle::NoGarbageCollectionScope no_gc(*this);
   const Sweeper::SweepingConfig sweeping_config{
       config_.sweeping_type,
       Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};

src/heap/cppgc/sweeper.cc

@@ -583,6 +583,10 @@ class Sweeper::SweeperImpl final {
     if (concurrent_sweeper_handle_) concurrent_sweeper_handle_->Join();
   }
 
+  bool IsSweepingOnMutatorThread() const {
+    return is_sweeping_on_mutator_thread_;
+  }
+
  private:
   class MutatorThreadSweepingScope final {
    public:
@@ -713,6 +717,9 @@ void Sweeper::WaitForConcurrentSweepingForTesting() {
   impl_->WaitForConcurrentSweepingForTesting();
 }
 void Sweeper::NotifyDoneIfNeeded() { impl_->NotifyDoneIfNeeded(); }
+bool Sweeper::IsSweepingOnMutatorThread() const {
+  return impl_->IsSweepingOnMutatorThread();
+}
 
 }  // namespace internal
 }  // namespace cppgc

src/heap/cppgc/sweeper.h

@@ -42,6 +42,8 @@ class V8_EXPORT_PRIVATE Sweeper final {
   void FinishIfRunning();
   void NotifyDoneIfNeeded();
 
+  bool IsSweepingOnMutatorThread() const;
+
  private:
   void WaitForConcurrentSweepingForTesting();
 

test/benchmarks/cpp/cppgc/allocation_perf.cc

@@ -4,6 +4,7 @@
 
 #include "include/cppgc/allocation.h"
 #include "include/cppgc/garbage-collected.h"
+#include "include/cppgc/heap-consistency.h"
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap.h"
 #include "test/benchmarks/cpp/cppgc/utils.h"
@@ -21,7 +22,7 @@ class TinyObject final : public cppgc::GarbageCollected<TinyObject> {
 };
 
 BENCHMARK_F(Allocate, Tiny)(benchmark::State& st) {
-  Heap::NoGCScope no_gc(*Heap::From(&heap()));
+  subtle::NoGarbageCollectionScope no_gc(*Heap::From(&heap()));
   for (auto _ : st) {
     benchmark::DoNotOptimize(
         cppgc::MakeGarbageCollected<TinyObject>(heap().GetAllocationHandle()));
@@ -36,7 +37,7 @@ class LargeObject final : public GarbageCollected<LargeObject> {
 };
 
 BENCHMARK_F(Allocate, Large)(benchmark::State& st) {
-  Heap::NoGCScope no_gc(*Heap::From(&heap()));
+  subtle::NoGarbageCollectionScope no_gc(*Heap::From(&heap()));
   for (auto _ : st) {
     benchmark::DoNotOptimize(
         cppgc::MakeGarbageCollected<LargeObject>(heap().GetAllocationHandle()));

test/unittests/heap/cppgc/heap-unittest.cc

@@ -9,6 +9,7 @@
 #include <numeric>
 
 #include "include/cppgc/allocation.h"
+#include "include/cppgc/heap-consistency.h"
 #include "include/cppgc/persistent.h"
 #include "src/heap/cppgc/globals.h"
 #include "test/unittests/heap/cppgc/tests.h"
@@ -95,7 +96,7 @@ TEST_F(GCHeapTest, ObjectPayloadSize) {
   Heap::From(GetHeap())->CollectGarbage(
       GarbageCollector::Config::ConservativeAtomicConfig());
 
-  Heap::NoGCScope no_gc_scope(*Heap::From(GetHeap()));
+  subtle::NoGarbageCollectionScope no_gc(*Heap::From(GetHeap()));
 
   for (size_t k = 0; k < kNumberOfObjectsPerArena; ++k) {
     MakeGarbageCollected<GCed<kObjectSizes[0]>>(GetAllocationHandle());
@@ -156,6 +157,23 @@ TEST_F(GCHeapTest, AllocatedSizeDependOnAdditionalBytes) {
             HeapObjectHeader::FromPayload(object_with_more_bytes).GetSize());
 }
 
+TEST_F(GCHeapTest, Epoch) {
+  const size_t epoch_before = internal::Heap::From(GetHeap())->epoch();
+  PreciseGC();
+  const size_t epoch_after_gc = internal::Heap::From(GetHeap())->epoch();
+  EXPECT_EQ(epoch_after_gc, epoch_before + 1);
+}
+
+TEST_F(GCHeapTest, NoGarbageCollectionScope) {
+  const size_t epoch_before = internal::Heap::From(GetHeap())->epoch();
+  {
+    subtle::NoGarbageCollectionScope scope(GetHeap()->GetHeapHandle());
+    PreciseGC();
+  }
+  const size_t epoch_after_gc = internal::Heap::From(GetHeap())->epoch();
+  EXPECT_EQ(epoch_after_gc, epoch_before);
+}
+
 TEST_F(GCHeapTest, TerminateEmptyHeap) { Heap::From(GetHeap())->Terminate(); }
 
 TEST_F(GCHeapTest, TerminateClearsPersistent) {

test/unittests/heap/cppgc/minor-gc-unittest.cc

@@ -89,7 +89,7 @@ TYPED_TEST(MinorGCTestForType, MinorCollection) {
   EXPECT_EQ(1u, TestFixture::DestructedObjects());
 
   {
-    Heap::NoGCScope no_gc_scope_(*Heap::From(this->GetHeap()));
+    subtle::NoGarbageCollectionScope no_gc_scope(*Heap::From(this->GetHeap()));
 
     Type* prev = nullptr;
     for (size_t i = 0; i < 64; ++i) {
@@ -144,7 +144,7 @@ void InterGenerationalPointerTest(MinorGCTest* test, cppgc::Heap* heap) {
   Type2* young = nullptr;
 
   {
-    Heap::NoGCScope no_gc_scope_(*Heap::From(heap));
+    subtle::NoGarbageCollectionScope no_gc_scope(*Heap::From(heap));
 
     // Allocate young objects.
     for (size_t i = 0; i < 64; ++i) {

test/unittests/heap/cppgc/tests.cc

@@ -38,7 +38,7 @@ void TestWithHeap::ResetLinearAllocationBuffers() {
 }
 
 TestSupportingAllocationOnly::TestSupportingAllocationOnly()
-    : no_gc_scope_(*internal::Heap::From(GetHeap())) {}
+    : no_gc_scope_(GetHeap()->GetHeapHandle()) {}
 
 }  // namespace testing
 }  // namespace internal

test/unittests/heap/cppgc/tests.h

@@ -5,6 +5,7 @@
 #ifndef V8_UNITTESTS_HEAP_CPPGC_TESTS_H_
 #define V8_UNITTESTS_HEAP_CPPGC_TESTS_H_
 
+#include "include/cppgc/heap-consistency.h"
 #include "include/cppgc/heap.h"
 #include "include/cppgc/platform.h"
 #include "src/heap/cppgc/heap.h"
@@ -101,7 +102,7 @@ class TestSupportingAllocationOnly : public TestWithHeap {
   TestSupportingAllocationOnly();
 
  private:
-  Heap::NoGCScope no_gc_scope_;
+  subtle::NoGarbageCollectionScope no_gc_scope_;
 };
 
 }  // namespace testing