cppgc: Port ObjectStartBitmap

This ports ObjectStartBitmap from Blink.

Bug: chromium:1056170
Change-Id: Ib959d9ac1c5e1e34ffa6418f77956e993c570ffc
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2181331
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67735}

BUILD.gn

@@ -4079,6 +4079,8 @@ v8_source_set("cppgc_base") {
     "src/heap/cppgc/object-allocator-inl.h",
     "src/heap/cppgc/object-allocator.cc",
     "src/heap/cppgc/object-allocator.h",
+    "src/heap/cppgc/object-start-bitmap-inl.h",
+    "src/heap/cppgc/object-start-bitmap.h",
     "src/heap/cppgc/page-memory-inl.h",
     "src/heap/cppgc/page-memory.cc",
     "src/heap/cppgc/page-memory.h",

src/heap/cppgc/heap-inl.h

@@ -5,9 +5,8 @@
 #ifndef V8_HEAP_CPPGC_HEAP_INL_H_
 #define V8_HEAP_CPPGC_HEAP_INL_H_
 
-#include "src/heap/cppgc/heap.h"
-
 #include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap.h"
 #include "src/heap/cppgc/object-allocator-inl.h"
 
 namespace cppgc {

src/heap/cppgc/heap-object-header.h

@@ -6,6 +6,7 @@
 #define V8_HEAP_CPPGC_HEAP_OBJECT_HEADER_H_
 
 #include <stdint.h>
+
 #include <atomic>
 
 #include "include/cppgc/internal/gc-info.h"

src/heap/cppgc/heap-page.cc

@@ -2,15 +2,18 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include <algorithm>
-
 #include "src/heap/cppgc/heap-page.h"
 
+#include <algorithm>
+
 #include "include/cppgc/internal/api-constants.h"
 #include "src/base/logging.h"
 #include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
 #include "src/heap/cppgc/heap-space.h"
 #include "src/heap/cppgc/heap.h"
+#include "src/heap/cppgc/object-start-bitmap-inl.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
 #include "src/heap/cppgc/page-memory.h"
 #include "src/heap/cppgc/raw-heap.h"
 
@@ -42,6 +45,27 @@ const BasePage* BasePage::FromPayload(const void* payload) {
       kGuardPageSize);
 }
 
+HeapObjectHeader* BasePage::ObjectHeaderFromInnerAddress(void* address) {
+  return const_cast<HeapObjectHeader*>(
+      ObjectHeaderFromInnerAddress(const_cast<const void*>(address)));
+}
+
+const HeapObjectHeader* BasePage::ObjectHeaderFromInnerAddress(
+    const void* address) {
+  if (is_large()) {
+    return LargePage::From(this)->ObjectHeader();
+  }
+  ObjectStartBitmap& bitmap = NormalPage::From(this)->object_start_bitmap();
+  HeapObjectHeader* header =
+      bitmap.FindHeader(static_cast<ConstAddress>(address));
+  DCHECK_LT(address,
+            reinterpret_cast<ConstAddress>(header) +
+                header->GetSize<HeapObjectHeader::AccessMode::kAtomic>());
+  DCHECK_NE(kFreeListGCInfoIndex,
+            header->GetGCInfoIndex<HeapObjectHeader::AccessMode::kAtomic>());
+  return header;
+}
+
 BasePage::BasePage(Heap* heap, BaseSpace* space, PageType type)
     : heap_(heap), space_(space), type_(type) {
   DCHECK_EQ(0u, (reinterpret_cast<uintptr_t>(this) - kGuardPageSize) &
@@ -59,8 +83,7 @@ NormalPage* NormalPage::Create(NormalPageSpace* space) {
   void* memory = heap->page_backend()->AllocateNormalPageMemory(space->index());
   auto* normal_page = new (memory) NormalPage(heap, space);
   space->AddPage(normal_page);
-  space->free_list().Add(
-      {normal_page->PayloadStart(), normal_page->PayloadSize()});
+  space->AddToFreeList(normal_page->PayloadStart(), normal_page->PayloadSize());
   return normal_page;
 }
 
@@ -76,7 +99,8 @@ void NormalPage::Destroy(NormalPage* page) {
 }
 
 NormalPage::NormalPage(Heap* heap, BaseSpace* space)
-    : BasePage(heap, space, PageType::kNormal) {
+    : BasePage(heap, space, PageType::kNormal),
+      object_start_bitmap_(PayloadStart()) {
   DCHECK_LT(kLargeObjectSizeThreshold,
             static_cast<size_t>(PayloadEnd() - PayloadStart()));
 }

src/heap/cppgc/heap-page.h

@@ -9,6 +9,7 @@
 #include "src/base/macros.h"
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
 
 namespace cppgc {
 namespace internal {
@@ -36,6 +37,10 @@ class V8_EXPORT_PRIVATE BasePage {
 
   bool is_large() const { return type_ == PageType::kLarge; }
 
+  // |address| must refer to real object.
+  HeapObjectHeader* ObjectHeaderFromInnerAddress(void* address);
+  const HeapObjectHeader* ObjectHeaderFromInnerAddress(const void* address);
+
  protected:
   enum class PageType { kNormal, kLarge };
   BasePage(Heap*, BaseSpace*, PageType);
@@ -125,9 +130,16 @@ class V8_EXPORT_PRIVATE NormalPage final : public BasePage {
 
   static size_t PayloadSize();
 
+  ObjectStartBitmap& object_start_bitmap() { return object_start_bitmap_; }
+  const ObjectStartBitmap& object_start_bitmap() const {
+    return object_start_bitmap_;
+  }
+
  private:
   NormalPage(Heap* heap, BaseSpace* space);
   ~NormalPage();
+
+  ObjectStartBitmap object_start_bitmap_;
 };
 
 class V8_EXPORT_PRIVATE LargePage final : public BasePage {

src/heap/cppgc/heap-space.cc

@@ -8,6 +8,7 @@
 
 #include "src/base/logging.h"
 #include "src/heap/cppgc/heap-page.h"
+#include "src/heap/cppgc/object-start-bitmap-inl.h"
 
 namespace cppgc {
 namespace internal {
@@ -35,10 +36,17 @@ BaseSpace::Pages BaseSpace::RemoveAllPages() {
 NormalPageSpace::NormalPageSpace(RawHeap* heap, size_t index)
     : BaseSpace(heap, index, PageType::kNormal) {}
 
+void NormalPageSpace::AddToFreeList(void* address, size_t size) {
+  free_list_.Add({address, size});
+  NormalPage::From(BasePage::FromPayload(address))
+      ->object_start_bitmap()
+      .SetBit(static_cast<Address>(address));
+}
+
 void NormalPageSpace::ResetLinearAllocationBuffer() {
   if (current_lab_.size()) {
     DCHECK_NOT_NULL(current_lab_.start());
-    free_list_.Add({current_lab_.start(), current_lab_.size()});
+    AddToFreeList(current_lab_.start(), current_lab_.size());
     current_lab_.Set(nullptr, 0);
   }
 }

src/heap/cppgc/heap-space.h

@@ -92,6 +92,7 @@ class V8_EXPORT_PRIVATE NormalPageSpace final : public BaseSpace {
 
   NormalPageSpace(RawHeap* heap, size_t index);
 
+  void AddToFreeList(void*, size_t);
   void ResetLinearAllocationBuffer();
 
   LinearAllocationBuffer& linear_allocation_buffer() { return current_lab_; }

src/heap/cppgc/heap.cc

@@ -45,7 +45,7 @@ namespace internal {
 
 namespace {
 
-class ObjectSizeCounter : public HeapVisitor<ObjectSizeCounter> {
+class ObjectSizeCounter : private HeapVisitor<ObjectSizeCounter> {
   friend class HeapVisitor<ObjectSizeCounter>;
 
  public:

src/heap/cppgc/heap.h

@@ -53,6 +53,7 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
   // allocations during GC.
   class V8_EXPORT_PRIVATE NoAllocationScope final {
     CPPGC_STACK_ALLOCATED();
+
    public:
     explicit NoAllocationScope(Heap* heap);
     ~NoAllocationScope();

src/heap/cppgc/object-allocator-inl.h

@@ -10,7 +10,10 @@
 #include "src/base/logging.h"
 #include "src/heap/cppgc/heap-object-header-inl.h"
 #include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/heap-page.h"
 #include "src/heap/cppgc/object-allocator.h"
+#include "src/heap/cppgc/object-start-bitmap-inl.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
 #include "src/heap/cppgc/sanitizers.h"
 
 namespace cppgc {
@@ -58,6 +61,11 @@ void* ObjectAllocator::AllocateObjectOnSpace(NormalPageSpace* space,
   void* raw = current_lab.Allocate(size);
   SET_MEMORY_ACCESIBLE(raw, size);
   auto* header = new (raw) HeapObjectHeader(size, gcinfo);
+
+  NormalPage::From(BasePage::FromPayload(header))
+      ->object_start_bitmap()
+      .SetBit(reinterpret_cast<ConstAddress>(header));
+
   return header->Payload();
 }
 }  // namespace internal

src/heap/cppgc/object-allocator.cc

@@ -11,6 +11,7 @@
 #include "src/heap/cppgc/heap-space.h"
 #include "src/heap/cppgc/heap.h"
 #include "src/heap/cppgc/object-allocator-inl.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
 #include "src/heap/cppgc/page-memory.h"
 #include "src/heap/cppgc/sweeper.h"
 
@@ -72,10 +73,13 @@ void* ObjectAllocator::AllocateFromFreeList(NormalPageSpace* space, size_t size,
 
   auto& current_lab = space->linear_allocation_buffer();
   if (current_lab.size()) {
-    space->free_list().Add({current_lab.start(), current_lab.size()});
+    space->AddToFreeList(current_lab.start(), current_lab.size());
   }
 
   current_lab.Set(static_cast<Address>(entry.address), entry.size);
+  NormalPage::From(BasePage::FromPayload(current_lab.start()))
+      ->object_start_bitmap()
+      .ClearBit(current_lab.start());
   return AllocateObjectOnSpace(space, size, gcinfo);
 }
 

src/heap/cppgc/object-start-bitmap-inl.h

@@ -0,0 +1,95 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_
+#define V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_
+
+#include <algorithm>
+
+#include "src/base/bits.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
+
+namespace cppgc {
+namespace internal {
+
+ObjectStartBitmap::ObjectStartBitmap(Address offset) : offset_(offset) {
+  Clear();
+}
+
+HeapObjectHeader* ObjectStartBitmap::FindHeader(
+    ConstAddress address_maybe_pointing_to_the_middle_of_object) const {
+  size_t object_offset =
+      address_maybe_pointing_to_the_middle_of_object - offset_;
+  size_t object_start_number = object_offset / kAllocationGranularity;
+  size_t cell_index = object_start_number / kBitsPerCell;
+  DCHECK_GT(object_start_bit_map_.size(), cell_index);
+  const size_t bit = object_start_number & kCellMask;
+  uint8_t byte = object_start_bit_map_[cell_index] & ((1 << (bit + 1)) - 1);
+  while (!byte && cell_index) {
+    DCHECK_LT(0u, cell_index);
+    byte = object_start_bit_map_[--cell_index];
+  }
+  const int leading_zeroes = v8::base::bits::CountLeadingZeros(byte);
+  object_start_number =
+      (cell_index * kBitsPerCell) + (kBitsPerCell - 1) - leading_zeroes;
+  object_offset = object_start_number * kAllocationGranularity;
+  return reinterpret_cast<HeapObjectHeader*>(object_offset + offset_);
+}
+
+void ObjectStartBitmap::SetBit(ConstAddress header_address) {
+  size_t cell_index, object_bit;
+  ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
+  object_start_bit_map_[cell_index] |= (1 << object_bit);
+}
+
+void ObjectStartBitmap::ClearBit(ConstAddress header_address) {
+  size_t cell_index, object_bit;
+  ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
+  object_start_bit_map_[cell_index] &= ~(1 << object_bit);
+}
+
+bool ObjectStartBitmap::CheckBit(ConstAddress header_address) const {
+  size_t cell_index, object_bit;
+  ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
+  return object_start_bit_map_[cell_index] & (1 << object_bit);
+}
+
+void ObjectStartBitmap::ObjectStartIndexAndBit(ConstAddress header_address,
+                                               size_t* cell_index,
+                                               size_t* bit) const {
+  const size_t object_offset = header_address - offset_;
+  DCHECK(!(object_offset & kAllocationMask));
+  const size_t object_start_number = object_offset / kAllocationGranularity;
+  *cell_index = object_start_number / kBitsPerCell;
+  DCHECK_GT(kBitmapSize, *cell_index);
+  *bit = object_start_number & kCellMask;
+}
+
+template <typename Callback>
+inline void ObjectStartBitmap::Iterate(Callback callback) const {
+  for (size_t cell_index = 0; cell_index < kReservedForBitmap; cell_index++) {
+    if (!object_start_bit_map_[cell_index]) continue;
+
+    uint8_t value = object_start_bit_map_[cell_index];
+    while (value) {
+      const int trailing_zeroes = v8::base::bits::CountTrailingZeros(value);
+      const size_t object_start_number =
+          (cell_index * kBitsPerCell) + trailing_zeroes;
+      const Address object_address =
+          offset_ + (kAllocationGranularity * object_start_number);
+      callback(object_address);
+      // Clear current object bit in temporary value to advance iteration.
+      value &= ~(1 << (object_start_number & kCellMask));
+    }
+  }
+}
+
+void ObjectStartBitmap::Clear() {
+  std::fill(object_start_bit_map_.begin(), object_start_bit_map_.end(), 0);
+}
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_

src/heap/cppgc/object-start-bitmap.h

@@ -0,0 +1,80 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_
+#define V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_
+
+#include <limits.h>
+#include <stdint.h>
+
+#include <array>
+
+#include "src/base/macros.h"
+#include "src/heap/cppgc/globals.h"
+
+namespace cppgc {
+namespace internal {
+
+class HeapObjectHeader;
+
+// A bitmap for recording object starts. Objects have to be allocated at
+// minimum granularity of kGranularity.
+//
+// Depends on internals such as:
+// - kBlinkPageSize
+// - kAllocationGranularity
+class V8_EXPORT_PRIVATE ObjectStartBitmap {
+ public:
+  // Granularity of addresses added to the bitmap.
+  static constexpr size_t Granularity() { return kAllocationGranularity; }
+
+  // Maximum number of entries in the bitmap.
+  static constexpr size_t MaxEntries() {
+    return kReservedForBitmap * kBitsPerCell;
+  }
+
+  explicit inline ObjectStartBitmap(Address offset);
+
+  // Finds an object header based on a
+  // address_maybe_pointing_to_the_middle_of_object. Will search for an object
+  // start in decreasing address order.
+  inline HeapObjectHeader* FindHeader(
+      ConstAddress address_maybe_pointing_to_the_middle_of_object) const;
+
+  inline void SetBit(ConstAddress);
+  inline void ClearBit(ConstAddress);
+  inline bool CheckBit(ConstAddress) const;
+
+  // Iterates all object starts recorded in the bitmap.
+  //
+  // The callback is of type
+  //   void(Address)
+  // and is passed the object start address as parameter.
+  template <typename Callback>
+  inline void Iterate(Callback) const;
+
+  // Clear the object start bitmap.
+  inline void Clear();
+
+ private:
+  static constexpr size_t kBitsPerCell = sizeof(uint8_t) * CHAR_BIT;
+  static constexpr size_t kCellMask = kBitsPerCell - 1;
+  static constexpr size_t kBitmapSize =
+      (kPageSize + ((kBitsPerCell * kAllocationGranularity) - 1)) /
+      (kBitsPerCell * kAllocationGranularity);
+  static constexpr size_t kReservedForBitmap =
+      ((kBitmapSize + kAllocationMask) & ~kAllocationMask);
+
+  inline void ObjectStartIndexAndBit(ConstAddress, size_t*, size_t*) const;
+
+  Address offset_;
+  // The bitmap contains a bit for every kGranularity aligned address on a
+  // a NormalPage, i.e., for a page of size kBlinkPageSize.
+  std::array<uint8_t, kReservedForBitmap> object_start_bit_map_;
+};
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_OBJECT_START_BITMAP_H_

src/heap/cppgc/sweeper.cc

@@ -13,6 +13,8 @@
 #include "src/heap/cppgc/heap-page.h"
 #include "src/heap/cppgc/heap-space.h"
 #include "src/heap/cppgc/heap-visitor.h"
+#include "src/heap/cppgc/object-start-bitmap-inl.h"
+#include "src/heap/cppgc/object-start-bitmap.h"
 #include "src/heap/cppgc/raw-heap.h"
 #include "src/heap/cppgc/sanitizers.h"
 
@@ -21,6 +23,37 @@ namespace internal {
 
 namespace {
 
+class ObjectStartBitmapVerifier
+    : private HeapVisitor<ObjectStartBitmapVerifier> {
+  friend class HeapVisitor<ObjectStartBitmapVerifier>;
+
+ public:
+  void Verify(RawHeap* heap) { Traverse(heap); }
+
+ private:
+  bool VisitNormalPage(NormalPage* page) {
+    // Remember bitmap and reset previous pointer.
+    bitmap_ = &page->object_start_bitmap();
+    prev_ = nullptr;
+    return false;
+  }
+
+  bool VisitHeapObjectHeader(HeapObjectHeader* header) {
+    if (header->IsLargeObject()) return true;
+
+    auto* raw_header = reinterpret_cast<ConstAddress>(header);
+    CHECK(bitmap_->CheckBit(raw_header));
+    if (prev_) {
+      CHECK_EQ(prev_, bitmap_->FindHeader(raw_header - 1));
+    }
+    prev_ = header;
+    return true;
+  }
+
+  ObjectStartBitmap* bitmap_ = nullptr;
+  HeapObjectHeader* prev_ = nullptr;
+};
+
 struct SpaceState {
   BaseSpace::Pages unswept_pages;
 };
@@ -30,6 +63,9 @@ bool SweepNormalPage(NormalPage* page) {
   constexpr auto kAtomicAccess = HeapObjectHeader::AccessMode::kAtomic;
 
   auto* space = NormalPageSpace::From(page->space());
+  ObjectStartBitmap& bitmap = page->object_start_bitmap();
+  bitmap.Clear();
+
   Address start_of_gap = page->PayloadStart();
   for (Address begin = page->PayloadStart(), end = page->PayloadEnd();
        begin != end;) {
@@ -37,7 +73,7 @@ bool SweepNormalPage(NormalPage* page) {
     const size_t size = header->GetSize();
     // Check if this is a free list entry.
     if (header->IsFree<kAtomicAccess>()) {
-      SET_MEMORY_INACCESIBLE(header, kFreeListEntrySize);
+      SET_MEMORY_INACCESIBLE(header, std::min(kFreeListEntrySize, size));
       begin += size;
       continue;
     }
@@ -51,18 +87,19 @@ bool SweepNormalPage(NormalPage* page) {
     // The object is alive.
     const Address header_address = reinterpret_cast<Address>(header);
     if (start_of_gap != header_address) {
-      space->free_list().Add(
-          {start_of_gap, static_cast<size_t>(header_address - start_of_gap)});
+      space->AddToFreeList(start_of_gap,
+                           static_cast<size_t>(header_address - start_of_gap));
     }
     header->Unmark<kAtomicAccess>();
+    bitmap.SetBit(begin);
     begin += size;
     start_of_gap = begin;
   }
 
   if (start_of_gap != page->PayloadStart() &&
       start_of_gap != page->PayloadEnd()) {
-    space->free_list().Add(
-        {start_of_gap, static_cast<size_t>(page->PayloadEnd() - start_of_gap)});
+    space->AddToFreeList(
+        start_of_gap, static_cast<size_t>(page->PayloadEnd() - start_of_gap));
   }
 
   const bool is_empty = (start_of_gap == page->PayloadStart());
@@ -140,6 +177,9 @@ class Sweeper::SweeperImpl final {
 
   void Start(Config config) {
     is_in_progress_ = true;
+#if DEBUG
+    ObjectStartBitmapVerifier().Verify(heap_);
+#endif
     PrepareForSweepVisitor(&space_states_).Traverse(heap_);
     if (config == Config::kAtomic) {
       Finish();

test/unittests/BUILD.gn

@@ -56,6 +56,7 @@ v8_source_set("cppgc_unittests_sources") {
     "heap/cppgc/marker-unittest.cc",
     "heap/cppgc/marking-visitor-unittest.cc",
     "heap/cppgc/member-unittest.cc",
+    "heap/cppgc/object-start-bitmap-unittest.cc",
     "heap/cppgc/page-memory-unittest.cc",
     "heap/cppgc/persistent-unittest.cc",
     "heap/cppgc/prefinalizer-unittest.cc",

test/unittests/heap/cppgc/heap-page-unittest.cc

@@ -245,5 +245,30 @@ TEST_F(PageTest, UnsweptPageDestruction) {
 }
 #endif
 
+TEST_F(PageTest, ObjectHeaderFromInnerAddress) {
+  {
+    auto* object = MakeGarbageCollected<GCed<64>>(GetHeap());
+    const HeapObjectHeader& expected = HeapObjectHeader::FromPayload(object);
+
+    for (auto* inner_ptr = reinterpret_cast<ConstAddress>(object);
+         inner_ptr < reinterpret_cast<ConstAddress>(object + 1); ++inner_ptr) {
+      const HeapObjectHeader* hoh =
+          BasePage::FromPayload(object)->ObjectHeaderFromInnerAddress(
+              inner_ptr);
+      EXPECT_EQ(&expected, hoh);
+    }
+  }
+  {
+    auto* object =
+        MakeGarbageCollected<GCed<2 * kLargeObjectSizeThreshold>>(GetHeap());
+    const HeapObjectHeader& expected = HeapObjectHeader::FromPayload(object);
+
+    const HeapObjectHeader* hoh =
+        BasePage::FromPayload(object)->ObjectHeaderFromInnerAddress(
+            reinterpret_cast<ConstAddress>(object) + kLargeObjectSizeThreshold);
+    EXPECT_EQ(&expected, hoh);
+  }
+}
+
 }  // namespace internal
 }  // namespace cppgc

test/unittests/heap/cppgc/object-start-bitmap-unittest.cc

@@ -0,0 +1,184 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/object-start-bitmap.h"
+
+#include "include/cppgc/allocation.h"
+#include "src/base/macros.h"
+#include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
+#include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/object-start-bitmap-inl.h"
+#include "src/heap/cppgc/page-memory-inl.h"
+#include "src/heap/cppgc/raw-heap.h"
+#include "test/unittests/heap/cppgc/tests.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+
+bool IsEmpty(const ObjectStartBitmap& bitmap) {
+  size_t count = 0;
+  bitmap.Iterate([&count](Address) { count++; });
+  return count == 0;
+}
+
+// Abstraction for objects that hides ObjectStartBitmap::kGranularity and
+// the base address as getting either of it wrong will result in failed DCHECKs.
+class Object {
+ public:
+  static Address kBaseOffset;
+
+  explicit Object(size_t number) : number_(number) {
+    const size_t max_entries = ObjectStartBitmap::MaxEntries();
+    EXPECT_GE(max_entries, number_);
+  }
+
+  Address address() const {
+    return kBaseOffset + ObjectStartBitmap::Granularity() * number_;
+  }
+
+  HeapObjectHeader* header() const {
+    return reinterpret_cast<HeapObjectHeader*>(address());
+  }
+
+  // Allow implicitly converting Object to Address.
+  operator Address() const { return address(); }  // NOLINT
+
+ private:
+  const size_t number_;
+};
+
+Address Object::kBaseOffset = reinterpret_cast<Address>(0x4000);
+
+}  // namespace
+
+TEST(ObjectStartBitmapTest, MoreThanZeroEntriesPossible) {
+  const size_t max_entries = ObjectStartBitmap::MaxEntries();
+  EXPECT_LT(0u, max_entries);
+}
+
+TEST(ObjectStartBitmapTest, InitialEmpty) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  EXPECT_TRUE(IsEmpty(bitmap));
+}
+
+TEST(ObjectStartBitmapTest, SetBitImpliesNonEmpty) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  bitmap.SetBit(Object(0));
+  EXPECT_FALSE(IsEmpty(bitmap));
+}
+
+TEST(ObjectStartBitmapTest, SetBitCheckBit) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object(7);
+  bitmap.SetBit(object);
+  EXPECT_TRUE(bitmap.CheckBit(object));
+}
+
+TEST(ObjectStartBitmapTest, SetBitClearbitCheckBit) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object(77);
+  bitmap.SetBit(object);
+  bitmap.ClearBit(object);
+  EXPECT_FALSE(bitmap.CheckBit(object));
+}
+
+TEST(ObjectStartBitmapTest, SetBitClearBitImpliesEmpty) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object(123);
+  bitmap.SetBit(object);
+  bitmap.ClearBit(object);
+  EXPECT_TRUE(IsEmpty(bitmap));
+}
+
+TEST(ObjectStartBitmapTest, AdjacentObjectsAtBegin) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object0(0);
+  Object object1(1);
+  bitmap.SetBit(object0);
+  bitmap.SetBit(object1);
+  EXPECT_FALSE(bitmap.CheckBit(Object(3)));
+  size_t count = 0;
+  bitmap.Iterate([&count, object0, object1](Address current) {
+    if (count == 0) {
+      EXPECT_EQ(object0.address(), current);
+    } else if (count == 1) {
+      EXPECT_EQ(object1.address(), current);
+    }
+    count++;
+  });
+  EXPECT_EQ(2u, count);
+}
+
+TEST(ObjectStartBitmapTest, AdjacentObjectsAtEnd) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  const size_t last_entry_index = ObjectStartBitmap::MaxEntries() - 1;
+  Object object0(last_entry_index - 1);
+  Object object1(last_entry_index);
+  bitmap.SetBit(object0);
+  bitmap.SetBit(object1);
+  EXPECT_FALSE(bitmap.CheckBit(Object(last_entry_index - 2)));
+  size_t count = 0;
+  bitmap.Iterate([&count, object0, object1](Address current) {
+    if (count == 0) {
+      EXPECT_EQ(object0.address(), current);
+    } else if (count == 1) {
+      EXPECT_EQ(object1.address(), current);
+    }
+    count++;
+  });
+  EXPECT_EQ(2u, count);
+}
+
+TEST(ObjectStartBitmapTest, FindHeaderExact) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object(654);
+  bitmap.SetBit(object);
+  EXPECT_EQ(object.header(), bitmap.FindHeader(object.address()));
+}
+
+TEST(ObjectStartBitmapTest, FindHeaderApproximate) {
+  static const size_t kInternalDelta = 37;
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object(654);
+  bitmap.SetBit(object);
+  EXPECT_EQ(object.header(),
+            bitmap.FindHeader(object.address() + kInternalDelta));
+}
+
+TEST(ObjectStartBitmapTest, FindHeaderIteratingWholeBitmap) {
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object_to_find(Object(0));
+  Address hint_index = Object(ObjectStartBitmap::MaxEntries() - 1);
+  bitmap.SetBit(object_to_find);
+  EXPECT_EQ(object_to_find.header(), bitmap.FindHeader(hint_index));
+}
+
+TEST(ObjectStartBitmapTest, FindHeaderNextCell) {
+  // This white box test makes use of the fact that cells are of type uint8_t.
+  const size_t kCellSize = sizeof(uint8_t);
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object_to_find(Object(kCellSize - 1));
+  Address hint = Object(kCellSize);
+  bitmap.SetBit(Object(0));
+  bitmap.SetBit(object_to_find);
+  EXPECT_EQ(object_to_find.header(), bitmap.FindHeader(hint));
+}
+
+TEST(ObjectStartBitmapTest, FindHeaderSameCell) {
+  // This white box test makes use of the fact that cells are of type uint8_t.
+  const size_t kCellSize = sizeof(uint8_t);
+  ObjectStartBitmap bitmap(Object::kBaseOffset);
+  Object object_to_find(Object(kCellSize - 1));
+  bitmap.SetBit(Object(0));
+  bitmap.SetBit(object_to_find);
+  EXPECT_EQ(object_to_find.header(),
+            bitmap.FindHeader(object_to_find.address()));
+}
+
+}  // namespace internal
+}  // namespace cppgc