[heap] Make maximum regular code object size a runtime value.

Executable V8 pages include 3 reserved OS pages: one for the writable
header and two as guards. On systems with 64k OS pages, the amount of
allocatable space left for objects can then be quite smaller than the
page size, only 64k for each 256k page.

This means regular code objects cannot be larger than 64k, while the
maximum regular object size is fixed to 128k, half of the page size. As
a result code object never reach this limit and we can end up filling
regular pages with few large code objects.

To fix this, we change the maximum code object size to be runtime value,
set to half of the allocatable space per page. On systems with 64k OS
pages, the limit will be 32k.

Alternatively, we could increase the V8 page size to 512k on Arm64 linux
so we wouldn't waste code space. However, systems with 4k OS pages are
more common, and those with 64k pages tend to have more memory available
so we should be able to live with it.

Bug: v8:10808
Change-Id: I5d807e7a3df89f1e9c648899e9ba2f8e2648264c
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2460809
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Pierre Langlois <pierre.langlois@arm.com>
Cr-Commit-Position: refs/heads/master@{#70569}

src/base/build_config.h

@@ -207,6 +207,10 @@ constexpr int kReturnAddressStackSlotCount =
 // PPC has large (64KB) physical pages.
 const int kPageSizeBits = 19;
 #else
+// Arm64 supports up to 64k OS pages on Linux, however 4k pages are more common
+// so we keep the V8 page size at 256k. Nonetheless, we need to make sure we
+// don't decrease it further in the future due to reserving 3 OS pages for every
+// executable V8 page.
 const int kPageSizeBits = 18;
 #endif
 

src/compiler/allocation-builder.h

@@ -27,7 +27,7 @@ class AllocationBuilder final {
   // Primitive allocation of static size.
   void Allocate(int size, AllocationType allocation = AllocationType::kYoung,
                 Type type = Type::Any()) {
-    DCHECK_LE(size, kMaxRegularHeapObjectSize);
+    DCHECK_LE(size, Heap::MaxRegularHeapObjectSize(allocation));
     effect_ = graph()->NewNode(
         common()->BeginRegion(RegionObservability::kNotObservable), effect_);
     allocation_ =

src/compiler/memory-lowering.cc

@@ -98,6 +98,10 @@ Reduction MemoryLowering::ReduceAllocateRaw(
   DCHECK_EQ(IrOpcode::kAllocateRaw, node->opcode());
   DCHECK_IMPLIES(allocation_folding_ == AllocationFolding::kDoAllocationFolding,
                  state_ptr != nullptr);
+  // Code objects may have a maximum size smaller than kMaxHeapObjectSize due to
+  // guard pages. If we need to support allocating code here we would need to
+  // call MemoryChunkLayout::MaxRegularCodeObjectSize() at runtime.
+  DCHECK_NE(allocation_type, AllocationType::kCode);
   Node* value;
   Node* size = node->InputAt(0);
   Node* effect = node->InputAt(1);

src/diagnostics/objects-debug.cc

@@ -969,7 +969,8 @@ void Code::CodeVerify(Isolate* isolate) {
   // CHECK_EQ(ReadOnlyHeap::Contains(*this), !IsExecutable());
   relocation_info().ObjectVerify(isolate);
   CHECK(V8_ENABLE_THIRD_PARTY_HEAP_BOOL ||
-        Code::SizeFor(body_size()) <= kMaxRegularHeapObjectSize ||
+        Code::SizeFor(body_size()) <=
+            MemoryChunkLayout::MaxRegularCodeObjectSize() ||
         isolate->heap()->InSpace(*this, CODE_LO_SPACE));
   Address last_gc_pc = kNullAddress;
 

src/heap/factory-base.cc

@@ -778,7 +778,8 @@ HeapObject FactoryBase<Impl>::AllocateRawArray(int size,
                                                AllocationType allocation) {
   HeapObject result = AllocateRaw(size, allocation);
   if (!V8_ENABLE_THIRD_PARTY_HEAP_BOOL &&
-      size > kMaxRegularHeapObjectSize && FLAG_use_marking_progress_bar) {
+      (size > Heap::MaxRegularHeapObjectSize(allocation)) &&
+      FLAG_use_marking_progress_bar) {
     BasicMemoryChunk* chunk = BasicMemoryChunk::FromHeapObject(result);
     chunk->SetFlag<AccessMode::ATOMIC>(MemoryChunk::HAS_PROGRESS_BAR);
   }

src/heap/factory.cc

@@ -393,7 +393,8 @@ MaybeHandle<FixedArray> Factory::TryNewFixedArray(
   AllocationResult allocation = heap->AllocateRaw(size, allocation_type);
   HeapObject result;
   if (!allocation.To(&result)) return MaybeHandle<FixedArray>();
-  if (size > kMaxRegularHeapObjectSize && FLAG_use_marking_progress_bar) {
+  if ((size > Heap::MaxRegularHeapObjectSize(allocation_type)) &&
+      FLAG_use_marking_progress_bar) {
     BasicMemoryChunk* chunk = BasicMemoryChunk::FromHeapObject(result);
     chunk->SetFlag<AccessMode::ATOMIC>(MemoryChunk::HAS_PROGRESS_BAR);
   }

src/heap/heap-inl.h

@@ -185,11 +185,7 @@ AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type,
   IncrementObjectCounters();
 #endif
 
-  size_t large_object_threshold =
-      !V8_ENABLE_THIRD_PARTY_HEAP_BOOL &&
-      AllocationType::kCode == type
-          ? std::min(kMaxRegularHeapObjectSize, code_space()->AreaSize())
-          : kMaxRegularHeapObjectSize;
+  size_t large_object_threshold = MaxRegularHeapObjectSize(type);
   bool large_object =
       static_cast<size_t>(size_in_bytes) > large_object_threshold;
 
@@ -281,7 +277,7 @@ HeapObject Heap::AllocateRawWith(int size, AllocationType allocation,
   if (!V8_ENABLE_THIRD_PARTY_HEAP_BOOL &&
       allocation == AllocationType::kYoung &&
       alignment == AllocationAlignment::kWordAligned &&
-      size <= kMaxRegularHeapObjectSize) {
+      size <= MaxRegularHeapObjectSize(allocation)) {
     Address* top = heap->NewSpaceAllocationTopAddress();
     Address* limit = heap->NewSpaceAllocationLimitAddress();
     if ((*limit - *top >= static_cast<unsigned>(size)) &&

src/heap/heap.cc

@@ -4757,6 +4757,15 @@ bool Heap::AllocationLimitOvershotByLargeMargin() {
   return v8_overshoot >= v8_margin || global_overshoot >= global_margin;
 }
 
+// static
+int Heap::MaxRegularHeapObjectSize(AllocationType allocation) {
+  if (!V8_ENABLE_THIRD_PARTY_HEAP_BOOL &&
+      (allocation == AllocationType::kCode)) {
+    return MemoryChunkLayout::MaxRegularCodeObjectSize();
+  }
+  return kMaxRegularHeapObjectSize;
+}
+
 bool Heap::ShouldOptimizeForLoadTime() {
   return isolate()->rail_mode() == PERFORMANCE_LOAD &&
          !AllocationLimitOvershotByLargeMargin() &&

src/heap/heap.h

@@ -544,7 +544,7 @@ class Heap {
 
   bool IsImmovable(HeapObject object);
 
-  static bool IsLargeObject(HeapObject object);
+  V8_EXPORT_PRIVATE static bool IsLargeObject(HeapObject object);
 
   // This method supports the deserialization allocator.  All allocations
   // are word-aligned.  The method should never fail to allocate since the
@@ -1366,6 +1366,14 @@ class Heap {
   // more eager to finalize incremental marking.
   bool AllocationLimitOvershotByLargeMargin();
 
+  // Return the maximum size objects can be before having to allocate them as
+  // large objects. This takes into account allocating in the code space for
+  // which the size of the allocatable space per V8 page may depend on the OS
+  // page size at runtime. You may use kMaxRegularHeapObjectSize as a constant
+  // instead if you know the allocation isn't in the code spaces.
+  V8_EXPORT_PRIVATE static int MaxRegularHeapObjectSize(
+      AllocationType allocation);
+
   // ===========================================================================
   // Prologue/epilogue callback methods.========================================
   // ===========================================================================

src/heap/local-heap-inl.h

@@ -26,7 +26,7 @@ AllocationResult LocalHeap::AllocateRaw(int size_in_bytes, AllocationType type,
   DCHECK(state == Heap::TEAR_DOWN || state == Heap::NOT_IN_GC);
 #endif
 
-  bool large_object = size_in_bytes > kMaxRegularHeapObjectSize;
+  bool large_object = size_in_bytes > Heap::MaxRegularHeapObjectSize(type);
   CHECK_EQ(type, AllocationType::kOld);
 
   if (large_object)

src/heap/memory-chunk-layout.cc

@@ -37,7 +37,6 @@ intptr_t MemoryChunkLayout::ObjectEndOffsetInCodePage() {
 
 size_t MemoryChunkLayout::AllocatableMemoryInCodePage() {
   size_t memory = ObjectEndOffsetInCodePage() - ObjectStartOffsetInCodePage();
-  DCHECK_LE(kMaxRegularHeapObjectSize, memory);
   return memory;
 }
 
@@ -67,5 +66,11 @@ size_t MemoryChunkLayout::AllocatableMemoryInMemoryChunk(
   return AllocatableMemoryInDataPage();
 }
 
+int MemoryChunkLayout::MaxRegularCodeObjectSize() {
+  int size = static_cast<int>(AllocatableMemoryInCodePage() / 2);
+  DCHECK_LE(size, kMaxRegularHeapObjectSize);
+  return size;
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/memory-chunk-layout.h

@@ -83,6 +83,8 @@ class V8_EXPORT_PRIVATE MemoryChunkLayout {
   static size_t AllocatableMemoryInDataPage();
   static size_t ObjectStartOffsetInMemoryChunk(AllocationSpace space);
   static size_t AllocatableMemoryInMemoryChunk(AllocationSpace space);
+
+  static int MaxRegularCodeObjectSize();
 };
 
 }  // namespace internal

src/heap/spaces.h

@@ -101,8 +101,10 @@ class SemiSpace;
 #define DCHECK_OBJECT_SIZE(size) \
   DCHECK((0 < size) && (size <= kMaxRegularHeapObjectSize))
 
-#define DCHECK_CODEOBJECT_SIZE(size, code_space) \
-  DCHECK((0 < size) && (size <= code_space->AreaSize()))
+#define DCHECK_CODEOBJECT_SIZE(size, code_space)                          \
+  DCHECK((0 < size) &&                                                    \
+         (size <= std::min(MemoryChunkLayout::MaxRegularCodeObjectSize(), \
+                           code_space->AreaSize())))
 
 // ----------------------------------------------------------------------------
 // Space is the abstract superclass for all allocation spaces that are not

test/cctest/heap/heap-tester.h

@@ -12,6 +12,7 @@
 // Those tests need to be defined using HEAP_TEST(Name) { ... }.
 #define HEAP_TEST_METHODS(V)                                \
   V(CodeLargeObjectSpace)                                   \
+  V(CodeLargeObjectSpace64k)                                \
   V(CompactionFullAbortedPage)                              \
   V(CompactionPartiallyAbortedPage)                         \
   V(CompactionPartiallyAbortedPageIntraAbortedPointers)     \

test/cctest/heap/test-heap.cc

@@ -6573,7 +6573,7 @@ HEAP_TEST(Regress5831) {
 
   // Generate the code.
   Handle<Code> code = GenerateDummyImmovableCode(isolate);
-  CHECK_GE(i::kMaxRegularHeapObjectSize, code->Size());
+  CHECK_GE(MemoryChunkLayout::MaxRegularCodeObjectSize(), code->Size());
   CHECK(!heap->code_space()->first_page()->Contains(code->address()));
 
   // Ensure it's not in large object space.
@@ -7061,7 +7061,7 @@ TEST(CodeObjectRegistry) {
   {
     // Ensure that both code objects end up on the same page.
     CHECK(HeapTester::CodeEnsureLinearAllocationArea(
-        heap, kMaxRegularHeapObjectSize));
+        heap, MemoryChunkLayout::MaxRegularCodeObjectSize()));
     code1 = DummyOptimizedCode(isolate);
     Handle<Code> code2 = DummyOptimizedCode(isolate);
     code2_address = code2->address();
@@ -7231,19 +7231,85 @@ class TestAllocationTracker : public HeapObjectAllocationTracker {
 
 HEAP_TEST(CodeLargeObjectSpace) {
   Heap* heap = CcTest::heap();
-  int size_in_bytes = kMaxRegularHeapObjectSize + kSystemPointerSize;
+  int size_in_bytes =
+      MemoryChunkLayout::MaxRegularCodeObjectSize() + kTaggedSize;
   TestAllocationTracker allocation_tracker{size_in_bytes};
   heap->AddHeapObjectAllocationTracker(&allocation_tracker);
 
-  AllocationResult allocation = heap->AllocateRaw(
-      size_in_bytes, AllocationType::kCode, AllocationOrigin::kGeneratedCode);
+  HeapObject obj;
+  {
+    AllocationResult allocation = heap->AllocateRaw(
+        size_in_bytes, AllocationType::kCode, AllocationOrigin::kRuntime);
+    CHECK(allocation.To(&obj));
+    CHECK_EQ(allocation.ToAddress(), allocation_tracker.address());
 
-  CHECK(allocation.ToAddress() == allocation_tracker.address());
-  heap->CreateFillerObjectAt(allocation.ToAddress(), size_in_bytes,
-                             ClearRecordedSlots::kNo);
+    heap->CreateFillerObjectAt(obj.address(), size_in_bytes,
+                               ClearRecordedSlots::kNo);
+  }
+
+  CHECK(Heap::IsLargeObject(obj));
   heap->RemoveHeapObjectAllocationTracker(&allocation_tracker);
 }
 
+UNINITIALIZED_HEAP_TEST(CodeLargeObjectSpace64k) {
+  // Simulate having a system with 64k OS pages.
+  i::FLAG_v8_os_page_size = 64;
+
+  // Initialize the isolate manually to make sure --v8-os-page-size is taken
+  // into account.
+  v8::Isolate::CreateParams create_params;
+  create_params.array_buffer_allocator = CcTest::array_buffer_allocator();
+  v8::Isolate* isolate = v8::Isolate::New(create_params);
+
+  Heap* heap = reinterpret_cast<Isolate*>(isolate)->heap();
+
+  // Allocate a regular code object.
+  {
+    int size_in_bytes =
+        MemoryChunkLayout::MaxRegularCodeObjectSize() - kTaggedSize;
+    TestAllocationTracker allocation_tracker{size_in_bytes};
+    heap->AddHeapObjectAllocationTracker(&allocation_tracker);
+
+    HeapObject obj;
+    {
+      AllocationResult allocation = heap->AllocateRaw(
+          size_in_bytes, AllocationType::kCode, AllocationOrigin::kRuntime);
+      CHECK(allocation.To(&obj));
+      CHECK_EQ(allocation.ToAddress(), allocation_tracker.address());
+
+      heap->CreateFillerObjectAt(obj.address(), size_in_bytes,
+                                 ClearRecordedSlots::kNo);
+    }
+
+    CHECK(!Heap::IsLargeObject(obj));
+    heap->RemoveHeapObjectAllocationTracker(&allocation_tracker);
+  }
+
+  // Allocate a large code object.
+  {
+    int size_in_bytes =
+        MemoryChunkLayout::MaxRegularCodeObjectSize() + kTaggedSize;
+    TestAllocationTracker allocation_tracker{size_in_bytes};
+    heap->AddHeapObjectAllocationTracker(&allocation_tracker);
+
+    HeapObject obj;
+    {
+      AllocationResult allocation = heap->AllocateRaw(
+          size_in_bytes, AllocationType::kCode, AllocationOrigin::kRuntime);
+      CHECK(allocation.To(&obj));
+      CHECK_EQ(allocation.ToAddress(), allocation_tracker.address());
+
+      heap->CreateFillerObjectAt(obj.address(), size_in_bytes,
+                                 ClearRecordedSlots::kNo);
+    }
+
+    CHECK(Heap::IsLargeObject(obj));
+    heap->RemoveHeapObjectAllocationTracker(&allocation_tracker);
+  }
+
+  isolate->Dispose();
+}
+
 TEST(Regress10900) {
   FLAG_always_compact = true;
   CcTest::InitializeVM();

test/cctest/test-code-pages.cc

@@ -264,7 +264,7 @@ TEST(LargeCodeObject) {
 
   // Create a big function that ends up in CODE_LO_SPACE.
   const int instruction_size = Page::kPageSize + 1;
-  STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize);
+  CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize());
   std::unique_ptr<byte[]> instructions(new byte[instruction_size]);
 
   CodeDesc desc;
@@ -380,7 +380,7 @@ TEST(LargeCodeObjectWithSignalHandler) {
 
   // Create a big function that ends up in CODE_LO_SPACE.
   const int instruction_size = Page::kPageSize + 1;
-  STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize);
+  CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize());
   std::unique_ptr<byte[]> instructions(new byte[instruction_size]);
 
   CodeDesc desc;
@@ -456,7 +456,7 @@ TEST(Sorted) {
 
   // Create a big function that ends up in CODE_LO_SPACE.
   const int instruction_size = Page::kPageSize + 1;
-  STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize);
+  CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize());
   std::unique_ptr<byte[]> instructions(new byte[instruction_size]);
 
   CodeDesc desc;

test/cctest/test-factory.cc

@@ -60,7 +60,8 @@ TEST(Factory_CodeBuilder) {
   HandleScope scope(isolate);
 
   // Create a big function that ends up in CODE_LO_SPACE.
-  const int instruction_size = kMaxRegularHeapObjectSize + 1;
+  const int instruction_size =
+      MemoryChunkLayout::MaxRegularCodeObjectSize() + 1;
   std::unique_ptr<byte[]> instructions(new byte[instruction_size]);
 
   CodeDesc desc;

test/cctest/test-unwinder-code-pages.cc

@@ -635,7 +635,7 @@ TEST(PCIsInV8_LargeCodeObject_CodePagesAPI) {
 
   // Create a big function that ends up in CODE_LO_SPACE.
   const int instruction_size = Page::kPageSize + 1;
-  STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize);
+  CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize());
   std::unique_ptr<byte[]> instructions(new byte[instruction_size]);
 
   CodeDesc desc;