diff --git a/BUILD.gn b/BUILD.gn index 0c983ad026..ad4f63655d 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -4697,7 +4697,6 @@ v8_source_set("cppgc_base") { "include/cppgc/internal/persistent-node.h", "include/cppgc/internal/pointer-policies.h", "include/cppgc/internal/prefinalizer-handler.h", - "include/cppgc/internal/process-heap.h", "include/cppgc/internal/write-barrier.h", "include/cppgc/liveness-broker.h", "include/cppgc/macros.h", @@ -4707,6 +4706,7 @@ v8_source_set("cppgc_base") { "include/cppgc/persistent.h", "include/cppgc/platform.h", "include/cppgc/prefinalizer.h", + "include/cppgc/process-heap-statistics.h", "include/cppgc/sentinel-pointer.h", "include/cppgc/source-location.h", "include/cppgc/trace-trait.h", @@ -4773,6 +4773,8 @@ v8_source_set("cppgc_base") { "src/heap/cppgc/pointer-policies.cc", "src/heap/cppgc/prefinalizer-handler.cc", "src/heap/cppgc/prefinalizer-handler.h", + "src/heap/cppgc/process-heap-statistics.cc", + "src/heap/cppgc/process-heap-statistics.h", "src/heap/cppgc/process-heap.cc", "src/heap/cppgc/process-heap.h", "src/heap/cppgc/raw-heap.cc", @@ -4790,6 +4792,7 @@ v8_source_set("cppgc_base") { "src/heap/cppgc/virtual-memory.h", "src/heap/cppgc/visitor.cc", "src/heap/cppgc/write-barrier.cc", + "src/heap/cppgc/write-barrier.h", ] if (cppgc_is_standalone) { diff --git a/include/DEPS b/include/DEPS index fd150b7143..21ce3d9645 100644 --- a/include/DEPS +++ b/include/DEPS @@ -5,7 +5,6 @@ include_rules = [ # Used by v8-cppgc.h to bridge to cppgc. "+cppgc/custom-space.h", "+cppgc/heap-statistics.h", - "+cppgc/internal/process-heap.h", "+cppgc/internal/write-barrier.h", "+cppgc/visitor.h", ] diff --git a/include/cppgc/internal/process-heap.h b/include/cppgc/internal/process-heap.h deleted file mode 100644 index 0f742a50a9..0000000000 --- a/include/cppgc/internal/process-heap.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 the V8 project authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef INCLUDE_CPPGC_INTERNAL_PROCESS_HEAP_H_ -#define INCLUDE_CPPGC_INTERNAL_PROCESS_HEAP_H_ - -#include "cppgc/internal/atomic-entry-flag.h" -#include "v8config.h" // NOLINT(build/include_directory) - -namespace cppgc { -namespace internal { - -class V8_EXPORT ProcessHeap final { - public: - static void EnterIncrementalOrConcurrentMarking() { - concurrent_marking_flag_.Enter(); - } - static void ExitIncrementalOrConcurrentMarking() { - concurrent_marking_flag_.Exit(); - } - - static bool IsAnyIncrementalOrConcurrentMarking() { - return concurrent_marking_flag_.MightBeEntered(); - } - - private: - static AtomicEntryFlag concurrent_marking_flag_; -}; - -} // namespace internal -} // namespace cppgc - -#endif // INCLUDE_CPPGC_INTERNAL_PROCESS_HEAP_H_ diff --git a/include/cppgc/internal/write-barrier.h b/include/cppgc/internal/write-barrier.h index 000b2488ce..f3aaedb1b8 100644 --- a/include/cppgc/internal/write-barrier.h +++ b/include/cppgc/internal/write-barrier.h @@ -7,7 +7,7 @@ #include "cppgc/heap-state.h" #include "cppgc/internal/api-constants.h" -#include "cppgc/internal/process-heap.h" +#include "cppgc/internal/atomic-entry-flag.h" #include "cppgc/sentinel-pointer.h" #include "cppgc/trace-trait.h" #include "v8config.h" // NOLINT(build/include_directory) @@ -86,6 +86,13 @@ class V8_EXPORT WriteBarrier final { static void CheckParams(Type expected_type, const Params& params) {} #endif // !V8_ENABLE_CHECKS + // The IncrementalOrConcurrentUpdater class allows cppgc internal to update + // |incremental_or_concurrent_marking_flag_|. + class IncrementalOrConcurrentMarkingFlagUpdater; + static bool IsAnyIncrementalOrConcurrentMarking() { + return incremental_or_concurrent_marking_flag_.MightBeEntered(); + } + private: WriteBarrier() = delete; @@ -111,6 +118,8 @@ class V8_EXPORT WriteBarrier final { const AgeTable& ageTable, const void* slot, uintptr_t value_offset); #endif // CPPGC_YOUNG_GENERATION + + static AtomicEntryFlag incremental_or_concurrent_marking_flag_; }; template @@ -218,7 +227,7 @@ struct WriteBarrierTypeForCagedHeapPolicy::ValueModeDispatch< return SetAndReturnType(params); } #else // !CPPGC_YOUNG_GENERATION - if (V8_LIKELY(!ProcessHeap::IsAnyIncrementalOrConcurrentMarking())) { + if (V8_LIKELY(!WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) { return SetAndReturnType(params); } HeapHandle& handle = callback(); @@ -288,7 +297,7 @@ struct WriteBarrierTypeForNonCagedHeapPolicy::ValueModeDispatch< static V8_INLINE WriteBarrier::Type Get(const void*, const void*, WriteBarrier::Params& params, HeapHandleCallback callback) { - if (V8_UNLIKELY(ProcessHeap::IsAnyIncrementalOrConcurrentMarking())) { + if (V8_UNLIKELY(WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) { HeapHandle& handle = callback(); if (IsMarking(handle)) { params.heap = &handle; diff --git a/include/cppgc/process-heap-statistics.h b/include/cppgc/process-heap-statistics.h new file mode 100644 index 0000000000..774cc92f46 --- /dev/null +++ b/include/cppgc/process-heap-statistics.h @@ -0,0 +1,36 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef INCLUDE_CPPGC_PROCESS_HEAP_STATISTICS_H_ +#define INCLUDE_CPPGC_PROCESS_HEAP_STATISTICS_H_ + +#include +#include + +#include "v8config.h" // NOLINT(build/include_directory) + +namespace cppgc { +namespace internal { +class ProcessHeapStatisticsUpdater; +} // namespace internal + +class V8_EXPORT ProcessHeapStatistics final { + public: + static size_t TotalAllocatedObjectSize() { + return total_allocated_object_size_.load(std::memory_order_relaxed); + } + static size_t TotalAllocatedSpace() { + return total_allocated_space_.load(std::memory_order_relaxed); + } + + private: + static std::atomic_size_t total_allocated_space_; + static std::atomic_size_t total_allocated_object_size_; + + friend class internal::ProcessHeapStatisticsUpdater; +}; + +} // namespace cppgc + +#endif // INCLUDE_CPPGC_PROCESS_HEAP_STATISTICS_H_ diff --git a/include/v8-cppgc.h b/include/v8-cppgc.h index 27e38778d2..414cee5d93 100644 --- a/include/v8-cppgc.h +++ b/include/v8-cppgc.h @@ -11,7 +11,6 @@ #include "cppgc/custom-space.h" #include "cppgc/heap-statistics.h" -#include "cppgc/internal/process-heap.h" #include "cppgc/internal/write-barrier.h" #include "cppgc/visitor.h" #include "v8-internal.h" // NOLINT(build/include_directory) @@ -168,7 +167,7 @@ class V8_EXPORT JSHeapConsistency final { WriteBarrierParams& params, HeapHandleCallback callback) { if (ref.IsEmpty()) return WriteBarrierType::kNone; - if (V8_LIKELY(!cppgc::internal::ProcessHeap:: + if (V8_LIKELY(!cppgc::internal::WriteBarrier:: IsAnyIncrementalOrConcurrentMarking())) { return cppgc::internal::WriteBarrier::Type::kNone; } diff --git a/src/heap/cppgc/heap-base.cc b/src/heap/cppgc/heap-base.cc index 59a2a5dec0..c3c32846b8 100644 --- a/src/heap/cppgc/heap-base.cc +++ b/src/heap/cppgc/heap-base.cc @@ -78,6 +78,8 @@ HeapBase::HeapBase( stats_collector_.get()), sweeper_(&raw_heap_, platform_.get(), stats_collector_.get()), stack_support_(stack_support) { + stats_collector_->RegisterObserver( + &allocation_observer_for_PROCESS_HEAP_STATISTICS_); } HeapBase::~HeapBase() = default; diff --git a/src/heap/cppgc/heap-base.h b/src/heap/cppgc/heap-base.h index 9e533c3d78..83077b1f64 100644 --- a/src/heap/cppgc/heap-base.h +++ b/src/heap/cppgc/heap-base.h @@ -17,6 +17,7 @@ #include "src/heap/cppgc/marker.h" #include "src/heap/cppgc/metric-recorder.h" #include "src/heap/cppgc/object-allocator.h" +#include "src/heap/cppgc/process-heap-statistics.h" #include "src/heap/cppgc/raw-heap.h" #include "src/heap/cppgc/sweeper.h" #include "v8config.h" // NOLINT(build/include_directory) @@ -189,6 +190,8 @@ class V8_EXPORT_PRIVATE HeapBase : public cppgc::HeapHandle { PersistentRegion strong_cross_thread_persistent_region_; PersistentRegion weak_cross_thread_persistent_region_; + ProcessHeapStatisticsUpdater::AllocationObserverImpl + allocation_observer_for_PROCESS_HEAP_STATISTICS_; #if defined(CPPGC_YOUNG_GENERATION) std::set remembered_slots_; #endif diff --git a/src/heap/cppgc/marker.cc b/src/heap/cppgc/marker.cc index 71e0a6bd7a..b5c512496f 100644 --- a/src/heap/cppgc/marker.cc +++ b/src/heap/cppgc/marker.cc @@ -8,7 +8,6 @@ #include #include "include/cppgc/heap-consistency.h" -#include "include/cppgc/internal/process-heap.h" #include "include/cppgc/platform.h" #include "src/base/platform/time.h" #include "src/heap/cppgc/heap-object-header.h" @@ -20,6 +19,7 @@ #include "src/heap/cppgc/marking-visitor.h" #include "src/heap/cppgc/process-heap.h" #include "src/heap/cppgc/stats-collector.h" +#include "src/heap/cppgc/write-barrier.h" #if defined(CPPGC_CAGED_HEAP) #include "include/cppgc/internal/caged-heap-local-data.h" @@ -35,7 +35,7 @@ bool EnterIncrementalMarkingIfNeeded(Marker::MarkingConfig config, if (config.marking_type == Marker::MarkingConfig::MarkingType::kIncremental || config.marking_type == Marker::MarkingConfig::MarkingType::kIncrementalAndConcurrent) { - ProcessHeap::EnterIncrementalOrConcurrentMarking(); + WriteBarrier::IncrementalOrConcurrentMarkingFlagUpdater::Enter(); #if defined(CPPGC_CAGED_HEAP) heap.caged_heap().local_data().is_incremental_marking_in_progress = true; #endif @@ -49,7 +49,7 @@ bool ExitIncrementalMarkingIfNeeded(Marker::MarkingConfig config, if (config.marking_type == Marker::MarkingConfig::MarkingType::kIncremental || config.marking_type == Marker::MarkingConfig::MarkingType::kIncrementalAndConcurrent) { - ProcessHeap::ExitIncrementalOrConcurrentMarking(); + WriteBarrier::IncrementalOrConcurrentMarkingFlagUpdater::Exit(); #if defined(CPPGC_CAGED_HEAP) heap.caged_heap().local_data().is_incremental_marking_in_progress = false; #endif diff --git a/src/heap/cppgc/object-start-bitmap.h b/src/heap/cppgc/object-start-bitmap.h index 38ba5ca886..da5df3932e 100644 --- a/src/heap/cppgc/object-start-bitmap.h +++ b/src/heap/cppgc/object-start-bitmap.h @@ -10,7 +10,7 @@ #include -#include "include/cppgc/internal/process-heap.h" +#include "include/cppgc/internal/write-barrier.h" #include "src/base/atomic-utils.h" #include "src/base/bits.h" #include "src/base/macros.h" @@ -217,7 +217,7 @@ bool PlatformAwareObjectStartBitmap::ShouldForceNonAtomic() { #if defined(V8_TARGET_ARCH_ARM) // Use non-atomic accesses on ARMv7 when marking is not active. if (mode == AccessMode::kAtomic) { - if (V8_LIKELY(!ProcessHeap::IsAnyIncrementalOrConcurrentMarking())) + if (V8_LIKELY(!WriteBarrier::IsAnyIncrementalOrConcurrentMarking())) return true; } #endif // defined(V8_TARGET_ARCH_ARM) diff --git a/src/heap/cppgc/process-heap-statistics.cc b/src/heap/cppgc/process-heap-statistics.cc new file mode 100644 index 0000000000..9d38d694b9 --- /dev/null +++ b/src/heap/cppgc/process-heap-statistics.cc @@ -0,0 +1,12 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/heap/cppgc/process-heap-statistics.h" + +namespace cppgc { + +std::atomic_size_t ProcessHeapStatistics::total_allocated_space_{0}; +std::atomic_size_t ProcessHeapStatistics::total_allocated_object_size_{0}; + +} // namespace cppgc diff --git a/src/heap/cppgc/process-heap-statistics.h b/src/heap/cppgc/process-heap-statistics.h new file mode 100644 index 0000000000..2d7bfa117f --- /dev/null +++ b/src/heap/cppgc/process-heap-statistics.h @@ -0,0 +1,73 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_HEAP_CPPGC_PROCESS_HEAP_STATISTICS_H_ +#define V8_HEAP_CPPGC_PROCESS_HEAP_STATISTICS_H_ + +#include "include/cppgc/process-heap-statistics.h" +#include "src/heap/cppgc/stats-collector.h" + +namespace cppgc { +namespace internal { + +class ProcessHeapStatisticsUpdater { + public: + // Allocation observer implementation for heaps should register to contribute + // to ProcessHeapStatistics. The heap is responsible for allocating and + // registering the obsrever impl with its stats collector. + class AllocationObserverImpl final + : public StatsCollector::AllocationObserver { + public: + void AllocatedObjectSizeIncreased(size_t bytes) final { + ProcessHeapStatisticsUpdater::IncreaseTotalAllocatedObjectSize(bytes); + object_size_changes_since_last_reset_ += bytes; + } + + void AllocatedObjectSizeDecreased(size_t bytes) final { + ProcessHeapStatisticsUpdater::DecreaseTotalAllocatedObjectSize(bytes); + object_size_changes_since_last_reset_ -= bytes; + } + + void ResetAllocatedObjectSize(size_t bytes) final { + ProcessHeapStatisticsUpdater::DecreaseTotalAllocatedObjectSize( + object_size_changes_since_last_reset_); + ProcessHeapStatisticsUpdater::IncreaseTotalAllocatedObjectSize(bytes); + object_size_changes_since_last_reset_ = bytes; + } + + void AllocatedSizeIncreased(size_t bytes) final { + ProcessHeapStatisticsUpdater::IncreaseTotalAllocatedSpace(bytes); + } + + void AllocatedSizeDecreased(size_t bytes) final { + ProcessHeapStatisticsUpdater::DecreaseTotalAllocatedSpace(bytes); + } + + private: + size_t object_size_changes_since_last_reset_ = 0; + }; + + // For cppgc::ProcessHeapStatistics + static void IncreaseTotalAllocatedObjectSize(size_t delta) { + ::cppgc::ProcessHeapStatistics::total_allocated_object_size_.fetch_add( + delta, std::memory_order_relaxed); + } + static void DecreaseTotalAllocatedObjectSize(size_t delta) { + ::cppgc::ProcessHeapStatistics::total_allocated_object_size_.fetch_sub( + delta, std::memory_order_relaxed); + } + static void IncreaseTotalAllocatedSpace(size_t delta) { + ::cppgc::ProcessHeapStatistics::total_allocated_space_.fetch_add( + delta, std::memory_order_relaxed); + } + static void DecreaseTotalAllocatedSpace(size_t delta) { + ::cppgc::ProcessHeapStatistics::total_allocated_space_.fetch_sub( + delta, std::memory_order_relaxed); + } +}; + +} // namespace internal +} // namespace cppgc + +#endif // V8_HEAP_CPPGC_PROCESS_HEAP_STATISTICS_H_ diff --git a/src/heap/cppgc/process-heap.cc b/src/heap/cppgc/process-heap.cc index 76a4a5dff5..e084ea1264 100644 --- a/src/heap/cppgc/process-heap.cc +++ b/src/heap/cppgc/process-heap.cc @@ -4,13 +4,9 @@ #include "src/heap/cppgc/process-heap.h" -#include "include/cppgc/internal/process-heap.h" - namespace cppgc { namespace internal { -AtomicEntryFlag ProcessHeap::concurrent_marking_flag_; - v8::base::LazyMutex g_process_mutex = LAZY_MUTEX_INITIALIZER; } // namespace internal diff --git a/src/heap/cppgc/stats-collector.cc b/src/heap/cppgc/stats-collector.cc index 42c5a72649..677216f772 100644 --- a/src/heap/cppgc/stats-collector.cc +++ b/src/heap/cppgc/stats-collector.cc @@ -236,10 +236,16 @@ size_t StatsCollector::allocated_object_size() const { void StatsCollector::NotifyAllocatedMemory(int64_t size) { memory_allocated_bytes_ += size; + ForAllAllocationObservers([size](AllocationObserver* observer) { + observer->AllocatedSizeIncreased(static_cast(size)); + }); } void StatsCollector::NotifyFreedMemory(int64_t size) { memory_freed_bytes_since_end_of_marking_ += size; + ForAllAllocationObservers([size](AllocationObserver* observer) { + observer->AllocatedSizeDecreased(static_cast(size)); + }); } void StatsCollector::RecordHistogramSample(ScopeId scope_id_, diff --git a/src/heap/cppgc/stats-collector.h b/src/heap/cppgc/stats-collector.h index 0d7627ce5e..3f8b952aed 100644 --- a/src/heap/cppgc/stats-collector.h +++ b/src/heap/cppgc/stats-collector.h @@ -225,14 +225,21 @@ class V8_EXPORT_PRIVATE StatsCollector final { // the deltas is interesting. // // May trigger GC. - virtual void AllocatedObjectSizeIncreased(size_t) = 0; - virtual void AllocatedObjectSizeDecreased(size_t) = 0; + virtual void AllocatedObjectSizeIncreased(size_t) {} + virtual void AllocatedObjectSizeDecreased(size_t) {} // Called when the exact size of allocated object size is known. In // practice, this is after marking when marked bytes == allocated bytes. // // Must not trigger GC synchronously. - virtual void ResetAllocatedObjectSize(size_t) = 0; + virtual void ResetAllocatedObjectSize(size_t) {} + + // Called upon allocating/releasing chunks of memory (e.g. pages) that can + // contain objects. + // + // Must not trigger GC. + virtual void AllocatedSizeIncreased(size_t) {} + virtual void AllocatedSizeDecreased(size_t) {} }; // Observers are implemented using virtual calls. Avoid notifications below diff --git a/src/heap/cppgc/write-barrier.cc b/src/heap/cppgc/write-barrier.cc index 009f9f96f4..75ff5ef626 100644 --- a/src/heap/cppgc/write-barrier.cc +++ b/src/heap/cppgc/write-barrier.cc @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "include/cppgc/internal/write-barrier.h" +#include "src/heap/cppgc/write-barrier.h" #include "include/cppgc/heap-consistency.h" #include "include/cppgc/internal/pointer-policies.h" @@ -20,6 +20,9 @@ namespace cppgc { namespace internal { +// static +AtomicEntryFlag WriteBarrier::incremental_or_concurrent_marking_flag_; + namespace { void ProcessMarkValue(HeapObjectHeader& header, MarkerBase* marker, diff --git a/src/heap/cppgc/write-barrier.h b/src/heap/cppgc/write-barrier.h new file mode 100644 index 0000000000..0758b01655 --- /dev/null +++ b/src/heap/cppgc/write-barrier.h @@ -0,0 +1,22 @@ +// Copyright 2021 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_HEAP_CPPGC_WRITE_BARRIER_H_ +#define V8_HEAP_CPPGC_WRITE_BARRIER_H_ + +#include "include/cppgc/internal/write-barrier.h" + +namespace cppgc { +namespace internal { + +class WriteBarrier::IncrementalOrConcurrentMarkingFlagUpdater { + public: + static void Enter() { incremental_or_concurrent_marking_flag_.Enter(); } + static void Exit() { incremental_or_concurrent_marking_flag_.Exit(); } +}; + +} // namespace internal +} // namespace cppgc + +#endif // V8_HEAP_CPPGC_WRITE_BARRIER_H_ diff --git a/test/unittests/heap/cppgc/stats-collector-unittest.cc b/test/unittests/heap/cppgc/stats-collector-unittest.cc index 0af25eebc9..5c8044db7e 100644 --- a/test/unittests/heap/cppgc/stats-collector-unittest.cc +++ b/test/unittests/heap/cppgc/stats-collector-unittest.cc @@ -117,6 +117,8 @@ class MockAllocationObserver : public StatsCollector::AllocationObserver { MOCK_METHOD(void, AllocatedObjectSizeIncreased, (size_t), (override)); MOCK_METHOD(void, AllocatedObjectSizeDecreased, (size_t), (override)); MOCK_METHOD(void, ResetAllocatedObjectSize, (size_t), (override)); + MOCK_METHOD(void, AllocatedSizeIncreased, (size_t), (override)); + MOCK_METHOD(void, AllocatedSizeDecreased, (size_t), (override)); }; TEST_F(StatsCollectorTest, RegisterUnregisterObserver) { @@ -156,6 +158,18 @@ TEST_F(StatsCollectorTest, ObserveResetAllocatedObjectSize) { stats.UnregisterObserver(&observer); } +TEST_F(StatsCollectorTest, ObserveAllocatedMemoryIncreaseAndDecrease) { + MockAllocationObserver observer; + stats.RegisterObserver(&observer); + static constexpr size_t kAllocatedMemorySize = 4096; + EXPECT_CALL(observer, AllocatedSizeIncreased(kAllocatedMemorySize)); + stats.NotifyAllocatedMemory(kAllocatedMemorySize); + static constexpr size_t kFreedMemorySize = 2048; + EXPECT_CALL(observer, AllocatedSizeDecreased(kFreedMemorySize)); + stats.NotifyFreedMemory(kFreedMemorySize); + stats.UnregisterObserver(&observer); +} + namespace { class AllocationObserverTriggeringGC final diff --git a/test/unittests/heap/cppgc/write-barrier-unittest.cc b/test/unittests/heap/cppgc/write-barrier-unittest.cc index d7701dc3e2..ba340743ea 100644 --- a/test/unittests/heap/cppgc/write-barrier-unittest.cc +++ b/test/unittests/heap/cppgc/write-barrier-unittest.cc @@ -175,7 +175,7 @@ class NoWriteBarrierTest : public testing::TestWithHeap {}; TEST_F(WriteBarrierTest, EnableDisableIncrementalMarking) { { IncrementalMarkingScope scope(marker()); - EXPECT_TRUE(ProcessHeap::IsAnyIncrementalOrConcurrentMarking()); + EXPECT_TRUE(WriteBarrier::IsAnyIncrementalOrConcurrentMarking()); } }