diff --git a/src/heap/memory-allocator.cc b/src/heap/memory-allocator.cc index 79938ad192..9bef95a734 100644 --- a/src/heap/memory-allocator.cc +++ b/src/heap/memory-allocator.cc @@ -6,9 +6,12 @@ #include +#include "src/base/address-region.h" #include "src/execution/isolate.h" +#include "src/flags/flags.h" #include "src/heap/gc-tracer.h" #include "src/heap/heap-inl.h" +#include "src/heap/memory-chunk.h" #include "src/logging/log.h" namespace v8 { @@ -369,5 +372,363 @@ Address MemoryAllocator::AllocateAlignedMemory( return base; } +MemoryChunk* MemoryAllocator::AllocateChunk(size_t reserve_area_size, + size_t commit_area_size, + Executability executable, + Space* owner) { + DCHECK_LE(commit_area_size, reserve_area_size); + + size_t chunk_size; + Heap* heap = isolate_->heap(); + Address base = kNullAddress; + VirtualMemory reservation; + Address area_start = kNullAddress; + Address area_end = kNullAddress; + void* address_hint = + AlignedAddress(heap->GetRandomMmapAddr(), MemoryChunk::kAlignment); + + // + // MemoryChunk layout: + // + // Executable + // +----------------------------+<- base aligned with MemoryChunk::kAlignment + // | Header | + // +----------------------------+<- base + CodePageGuardStartOffset + // | Guard | + // +----------------------------+<- area_start_ + // | Area | + // +----------------------------+<- area_end_ (area_start + commit_area_size) + // | Committed but not used | + // +----------------------------+<- aligned at OS page boundary + // | Reserved but not committed | + // +----------------------------+<- aligned at OS page boundary + // | Guard | + // +----------------------------+<- base + chunk_size + // + // Non-executable + // +----------------------------+<- base aligned with MemoryChunk::kAlignment + // | Header | + // +----------------------------+<- area_start_ (base + area_start_) + // | Area | + // +----------------------------+<- area_end_ (area_start + commit_area_size) + // | Committed but not used | + // +----------------------------+<- aligned at OS page boundary + // | Reserved but not committed | + // +----------------------------+<- base + chunk_size + // + + if (executable == EXECUTABLE) { + chunk_size = ::RoundUp(MemoryChunkLayout::ObjectStartOffsetInCodePage() + + reserve_area_size + + MemoryChunkLayout::CodePageGuardSize(), + GetCommitPageSize()); + + // Size of header (not executable) plus area (executable). + size_t commit_size = ::RoundUp( + MemoryChunkLayout::CodePageGuardStartOffset() + commit_area_size, + GetCommitPageSize()); + base = + AllocateAlignedMemory(chunk_size, commit_size, MemoryChunk::kAlignment, + executable, address_hint, &reservation); + if (base == kNullAddress) return nullptr; + // Update executable memory size. + size_executable_ += reservation.size(); + + if (Heap::ShouldZapGarbage()) { + ZapBlock(base, MemoryChunkLayout::CodePageGuardStartOffset(), kZapValue); + ZapBlock(base + MemoryChunkLayout::ObjectStartOffsetInCodePage(), + commit_area_size, kZapValue); + } + + area_start = base + MemoryChunkLayout::ObjectStartOffsetInCodePage(); + area_end = area_start + commit_area_size; + } else { + chunk_size = ::RoundUp( + MemoryChunkLayout::ObjectStartOffsetInDataPage() + reserve_area_size, + GetCommitPageSize()); + size_t commit_size = ::RoundUp( + MemoryChunkLayout::ObjectStartOffsetInDataPage() + commit_area_size, + GetCommitPageSize()); + base = + AllocateAlignedMemory(chunk_size, commit_size, MemoryChunk::kAlignment, + executable, address_hint, &reservation); + + if (base == kNullAddress) return nullptr; + + if (Heap::ShouldZapGarbage()) { + ZapBlock( + base, + MemoryChunkLayout::ObjectStartOffsetInDataPage() + commit_area_size, + kZapValue); + } + + area_start = base + MemoryChunkLayout::ObjectStartOffsetInDataPage(); + area_end = area_start + commit_area_size; + } + + // Use chunk_size for statistics because we assume that treat reserved but + // not-yet committed memory regions of chunks as allocated. + LOG(isolate_, + NewEvent("MemoryChunk", reinterpret_cast(base), chunk_size)); + + // We cannot use the last chunk in the address space because we would + // overflow when comparing top and limit if this chunk is used for a + // linear allocation area. + if ((base + chunk_size) == 0u) { + CHECK(!last_chunk_.IsReserved()); + last_chunk_ = std::move(reservation); + UncommitMemory(&last_chunk_); + size_ -= chunk_size; + if (executable == EXECUTABLE) { + size_executable_ -= chunk_size; + } + CHECK(last_chunk_.IsReserved()); + return AllocateChunk(reserve_area_size, commit_area_size, executable, + owner); + } + + MemoryChunk* chunk = + MemoryChunk::Initialize(heap, base, chunk_size, area_start, area_end, + executable, owner, std::move(reservation)); + + if (chunk->executable()) RegisterExecutableMemoryChunk(chunk); + return chunk; +} + +void MemoryAllocator::PartialFreeMemory(MemoryChunk* chunk, Address start_free, + size_t bytes_to_free, + Address new_area_end) { + VirtualMemory* reservation = chunk->reserved_memory(); + DCHECK(reservation->IsReserved()); + chunk->set_size(chunk->size() - bytes_to_free); + chunk->set_area_end(new_area_end); + if (chunk->IsFlagSet(MemoryChunk::IS_EXECUTABLE)) { + // Add guard page at the end. + size_t page_size = GetCommitPageSize(); + DCHECK_EQ(0, chunk->area_end() % static_cast
(page_size)); + DCHECK_EQ(chunk->address() + chunk->size(), + chunk->area_end() + MemoryChunkLayout::CodePageGuardSize()); + reservation->SetPermissions(chunk->area_end(), page_size, + PageAllocator::kNoAccess); + } + // On e.g. Windows, a reservation may be larger than a page and releasing + // partially starting at |start_free| will also release the potentially + // unused part behind the current page. + const size_t released_bytes = reservation->Release(start_free); + DCHECK_GE(size_, released_bytes); + size_ -= released_bytes; +} + +void MemoryAllocator::UnregisterMemory(MemoryChunk* chunk) { + DCHECK(!chunk->IsFlagSet(MemoryChunk::UNREGISTERED)); + VirtualMemory* reservation = chunk->reserved_memory(); + const size_t size = + reservation->IsReserved() ? reservation->size() : chunk->size(); + DCHECK_GE(size_, static_cast(size)); + size_ -= size; + if (chunk->executable() == EXECUTABLE) { + DCHECK_GE(size_executable_, size); + size_executable_ -= size; + } + + if (chunk->executable()) UnregisterExecutableMemoryChunk(chunk); + chunk->SetFlag(MemoryChunk::UNREGISTERED); +} + +void MemoryAllocator::PreFreeMemory(MemoryChunk* chunk) { + DCHECK(!chunk->IsFlagSet(MemoryChunk::PRE_FREED)); + LOG(isolate_, DeleteEvent("MemoryChunk", chunk)); + UnregisterMemory(chunk); + isolate_->heap()->RememberUnmappedPage(reinterpret_cast
(chunk), + chunk->IsEvacuationCandidate()); + chunk->SetFlag(MemoryChunk::PRE_FREED); +} + +void MemoryAllocator::PerformFreeMemory(MemoryChunk* chunk) { + DCHECK(chunk->IsFlagSet(MemoryChunk::UNREGISTERED)); + DCHECK(chunk->IsFlagSet(MemoryChunk::PRE_FREED)); + chunk->ReleaseAllAllocatedMemory(); + + VirtualMemory* reservation = chunk->reserved_memory(); + if (chunk->IsFlagSet(MemoryChunk::POOLED)) { + UncommitMemory(reservation); + } else { + if (reservation->IsReserved()) { + reservation->Free(); + } else { + // Only read-only pages can have non-initialized reservation object. + DCHECK_EQ(RO_SPACE, chunk->owner_identity()); + FreeMemory(page_allocator(chunk->executable()), chunk->address(), + chunk->size()); + } + } +} + +template +void MemoryAllocator::Free(MemoryChunk* chunk) { + switch (mode) { + case kFull: + PreFreeMemory(chunk); + PerformFreeMemory(chunk); + break; + case kAlreadyPooled: + // Pooled pages cannot be touched anymore as their memory is uncommitted. + // Pooled pages are not-executable. + FreeMemory(data_page_allocator(), chunk->address(), + static_cast(MemoryChunk::kPageSize)); + break; + case kPooledAndQueue: + DCHECK_EQ(chunk->size(), static_cast(MemoryChunk::kPageSize)); + DCHECK_EQ(chunk->executable(), NOT_EXECUTABLE); + chunk->SetFlag(MemoryChunk::POOLED); + V8_FALLTHROUGH; + case kPreFreeAndQueue: + PreFreeMemory(chunk); + // The chunks added to this queue will be freed by a concurrent thread. + unmapper()->AddMemoryChunkSafe(chunk); + break; + } +} + +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< + MemoryAllocator::kFull>(MemoryChunk* chunk); + +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< + MemoryAllocator::kAlreadyPooled>(MemoryChunk* chunk); + +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< + MemoryAllocator::kPreFreeAndQueue>(MemoryChunk* chunk); + +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< + MemoryAllocator::kPooledAndQueue>(MemoryChunk* chunk); + +template +Page* MemoryAllocator::AllocatePage(size_t size, SpaceType* owner, + Executability executable) { + MemoryChunk* chunk = nullptr; + if (alloc_mode == kPooled) { + DCHECK_EQ(size, static_cast( + MemoryChunkLayout::AllocatableMemoryInMemoryChunk( + owner->identity()))); + DCHECK_EQ(executable, NOT_EXECUTABLE); + chunk = AllocatePagePooled(owner); + } + if (chunk == nullptr) { + chunk = AllocateChunk(size, size, executable, owner); + } + if (chunk == nullptr) return nullptr; + return owner->InitializePage(chunk); +} + +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) + Page* MemoryAllocator::AllocatePage( + size_t size, PagedSpace* owner, Executability executable); +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) + Page* MemoryAllocator::AllocatePage( + size_t size, SemiSpace* owner, Executability executable); +template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) + Page* MemoryAllocator::AllocatePage( + size_t size, SemiSpace* owner, Executability executable); + +LargePage* MemoryAllocator::AllocateLargePage(size_t size, + LargeObjectSpace* owner, + Executability executable) { + MemoryChunk* chunk = AllocateChunk(size, size, executable, owner); + if (chunk == nullptr) return nullptr; + return LargePage::Initialize(isolate_->heap(), chunk, executable); +} + +template +MemoryChunk* MemoryAllocator::AllocatePagePooled(SpaceType* owner) { + MemoryChunk* chunk = unmapper()->TryGetPooledMemoryChunkSafe(); + if (chunk == nullptr) return nullptr; + const int size = MemoryChunk::kPageSize; + const Address start = reinterpret_cast
(chunk); + const Address area_start = + start + + MemoryChunkLayout::ObjectStartOffsetInMemoryChunk(owner->identity()); + const Address area_end = start + size; + // Pooled pages are always regular data pages. + DCHECK_NE(CODE_SPACE, owner->identity()); + VirtualMemory reservation(data_page_allocator(), start, size); + if (!CommitMemory(&reservation)) return nullptr; + if (Heap::ShouldZapGarbage()) { + ZapBlock(start, size, kZapValue); + } + MemoryChunk::Initialize(isolate_->heap(), start, size, area_start, area_end, + NOT_EXECUTABLE, owner, std::move(reservation)); + size_ += size; + return chunk; +} + +void MemoryAllocator::ZapBlock(Address start, size_t size, + uintptr_t zap_value) { + DCHECK(IsAligned(start, kTaggedSize)); + DCHECK(IsAligned(size, kTaggedSize)); + MemsetTagged(ObjectSlot(start), Object(static_cast
(zap_value)), + size >> kTaggedSizeLog2); +} + +intptr_t MemoryAllocator::GetCommitPageSize() { + if (FLAG_v8_os_page_size != 0) { + DCHECK(base::bits::IsPowerOfTwo(FLAG_v8_os_page_size)); + return FLAG_v8_os_page_size * KB; + } else { + return CommitPageSize(); + } +} + +base::AddressRegion MemoryAllocator::ComputeDiscardMemoryArea(Address addr, + size_t size) { + size_t page_size = MemoryAllocator::GetCommitPageSize(); + if (size < page_size + FreeSpace::kSize) { + return base::AddressRegion(0, 0); + } + Address discardable_start = RoundUp(addr + FreeSpace::kSize, page_size); + Address discardable_end = RoundDown(addr + size, page_size); + if (discardable_start >= discardable_end) return base::AddressRegion(0, 0); + return base::AddressRegion(discardable_start, + discardable_end - discardable_start); +} + +bool MemoryAllocator::CommitExecutableMemory(VirtualMemory* vm, Address start, + size_t commit_size, + size_t reserved_size) { + const size_t page_size = GetCommitPageSize(); + // All addresses and sizes must be aligned to the commit page size. + DCHECK(IsAligned(start, page_size)); + DCHECK_EQ(0, commit_size % page_size); + DCHECK_EQ(0, reserved_size % page_size); + const size_t guard_size = MemoryChunkLayout::CodePageGuardSize(); + const size_t pre_guard_offset = MemoryChunkLayout::CodePageGuardStartOffset(); + const size_t code_area_offset = + MemoryChunkLayout::ObjectStartOffsetInCodePage(); + // reserved_size includes two guard regions, commit_size does not. + DCHECK_LE(commit_size, reserved_size - 2 * guard_size); + const Address pre_guard_page = start + pre_guard_offset; + const Address code_area = start + code_area_offset; + const Address post_guard_page = start + reserved_size - guard_size; + // Commit the non-executable header, from start to pre-code guard page. + if (vm->SetPermissions(start, pre_guard_offset, PageAllocator::kReadWrite)) { + // Create the pre-code guard page, following the header. + if (vm->SetPermissions(pre_guard_page, page_size, + PageAllocator::kNoAccess)) { + // Commit the executable code body. + if (vm->SetPermissions(code_area, commit_size - pre_guard_offset, + PageAllocator::kReadWrite)) { + // Create the post-code guard page. + if (vm->SetPermissions(post_guard_page, page_size, + PageAllocator::kNoAccess)) { + UpdateAllocatedSpaceLimits(start, code_area + commit_size); + return true; + } + vm->SetPermissions(code_area, commit_size, PageAllocator::kNoAccess); + } + } + vm->SetPermissions(start, pre_guard_offset, PageAllocator::kNoAccess); + } + return false; +} + } // namespace internal } // namespace v8 diff --git a/src/heap/spaces.cc b/src/heap/spaces.cc index 231e3093f1..8e338659e8 100644 --- a/src/heap/spaces.cc +++ b/src/heap/spaces.cc @@ -11,11 +11,7 @@ #include "src/base/bits.h" #include "src/base/bounded-page-allocator.h" #include "src/base/macros.h" -#include "src/base/optional.h" -#include "src/base/platform/semaphore.h" #include "src/common/globals.h" -#include "src/execution/vm-state-inl.h" -#include "src/heap/array-buffer-sweeper.h" #include "src/heap/array-buffer-tracker-inl.h" #include "src/heap/combined-heap.h" #include "src/heap/concurrent-marking.h" @@ -29,7 +25,6 @@ #include "src/heap/read-only-heap.h" #include "src/heap/remembered-set-inl.h" #include "src/heap/slot-set.h" -#include "src/heap/sweeper.h" #include "src/init/v8.h" #include "src/logging/counters.h" #include "src/objects/free-space-inl.h" @@ -135,129 +130,6 @@ void Page::MergeOldToNewRememberedSets() { sweeping_slot_set_ = nullptr; } -MemoryChunk* MemoryAllocator::AllocateChunk(size_t reserve_area_size, - size_t commit_area_size, - Executability executable, - Space* owner) { - DCHECK_LE(commit_area_size, reserve_area_size); - - size_t chunk_size; - Heap* heap = isolate_->heap(); - Address base = kNullAddress; - VirtualMemory reservation; - Address area_start = kNullAddress; - Address area_end = kNullAddress; - void* address_hint = - AlignedAddress(heap->GetRandomMmapAddr(), MemoryChunk::kAlignment); - - // - // MemoryChunk layout: - // - // Executable - // +----------------------------+<- base aligned with MemoryChunk::kAlignment - // | Header | - // +----------------------------+<- base + CodePageGuardStartOffset - // | Guard | - // +----------------------------+<- area_start_ - // | Area | - // +----------------------------+<- area_end_ (area_start + commit_area_size) - // | Committed but not used | - // +----------------------------+<- aligned at OS page boundary - // | Reserved but not committed | - // +----------------------------+<- aligned at OS page boundary - // | Guard | - // +----------------------------+<- base + chunk_size - // - // Non-executable - // +----------------------------+<- base aligned with MemoryChunk::kAlignment - // | Header | - // +----------------------------+<- area_start_ (base + area_start_) - // | Area | - // +----------------------------+<- area_end_ (area_start + commit_area_size) - // | Committed but not used | - // +----------------------------+<- aligned at OS page boundary - // | Reserved but not committed | - // +----------------------------+<- base + chunk_size - // - - if (executable == EXECUTABLE) { - chunk_size = ::RoundUp(MemoryChunkLayout::ObjectStartOffsetInCodePage() + - reserve_area_size + - MemoryChunkLayout::CodePageGuardSize(), - GetCommitPageSize()); - - // Size of header (not executable) plus area (executable). - size_t commit_size = ::RoundUp( - MemoryChunkLayout::CodePageGuardStartOffset() + commit_area_size, - GetCommitPageSize()); - base = - AllocateAlignedMemory(chunk_size, commit_size, MemoryChunk::kAlignment, - executable, address_hint, &reservation); - if (base == kNullAddress) return nullptr; - // Update executable memory size. - size_executable_ += reservation.size(); - - if (Heap::ShouldZapGarbage()) { - ZapBlock(base, MemoryChunkLayout::CodePageGuardStartOffset(), kZapValue); - ZapBlock(base + MemoryChunkLayout::ObjectStartOffsetInCodePage(), - commit_area_size, kZapValue); - } - - area_start = base + MemoryChunkLayout::ObjectStartOffsetInCodePage(); - area_end = area_start + commit_area_size; - } else { - chunk_size = ::RoundUp( - MemoryChunkLayout::ObjectStartOffsetInDataPage() + reserve_area_size, - GetCommitPageSize()); - size_t commit_size = ::RoundUp( - MemoryChunkLayout::ObjectStartOffsetInDataPage() + commit_area_size, - GetCommitPageSize()); - base = - AllocateAlignedMemory(chunk_size, commit_size, MemoryChunk::kAlignment, - executable, address_hint, &reservation); - - if (base == kNullAddress) return nullptr; - - if (Heap::ShouldZapGarbage()) { - ZapBlock( - base, - MemoryChunkLayout::ObjectStartOffsetInDataPage() + commit_area_size, - kZapValue); - } - - area_start = base + MemoryChunkLayout::ObjectStartOffsetInDataPage(); - area_end = area_start + commit_area_size; - } - - // Use chunk_size for statistics because we assume that treat reserved but - // not-yet committed memory regions of chunks as allocated. - LOG(isolate_, - NewEvent("MemoryChunk", reinterpret_cast(base), chunk_size)); - - // We cannot use the last chunk in the address space because we would - // overflow when comparing top and limit if this chunk is used for a - // linear allocation area. - if ((base + chunk_size) == 0u) { - CHECK(!last_chunk_.IsReserved()); - last_chunk_ = std::move(reservation); - UncommitMemory(&last_chunk_); - size_ -= chunk_size; - if (executable == EXECUTABLE) { - size_executable_ -= chunk_size; - } - CHECK(last_chunk_.IsReserved()); - return AllocateChunk(reserve_area_size, commit_area_size, executable, - owner); - } - - MemoryChunk* chunk = - MemoryChunk::Initialize(heap, base, chunk_size, area_start, area_end, - executable, owner, std::move(reservation)); - - if (chunk->executable()) RegisterExecutableMemoryChunk(chunk); - return chunk; -} - void Page::ResetAllocationStatistics() { allocated_bytes_ = area_size(); wasted_memory_ = 0; @@ -366,241 +238,6 @@ void Page::DestroyBlackArea(Address start, Address end) { marking_state->IncrementLiveBytes(this, -static_cast(end - start)); } -void MemoryAllocator::PartialFreeMemory(MemoryChunk* chunk, Address start_free, - size_t bytes_to_free, - Address new_area_end) { - VirtualMemory* reservation = chunk->reserved_memory(); - DCHECK(reservation->IsReserved()); - chunk->set_size(chunk->size() - bytes_to_free); - chunk->set_area_end(new_area_end); - if (chunk->IsFlagSet(MemoryChunk::IS_EXECUTABLE)) { - // Add guard page at the end. - size_t page_size = GetCommitPageSize(); - DCHECK_EQ(0, chunk->area_end() % static_cast
(page_size)); - DCHECK_EQ(chunk->address() + chunk->size(), - chunk->area_end() + MemoryChunkLayout::CodePageGuardSize()); - reservation->SetPermissions(chunk->area_end(), page_size, - PageAllocator::kNoAccess); - } - // On e.g. Windows, a reservation may be larger than a page and releasing - // partially starting at |start_free| will also release the potentially - // unused part behind the current page. - const size_t released_bytes = reservation->Release(start_free); - DCHECK_GE(size_, released_bytes); - size_ -= released_bytes; -} - -void MemoryAllocator::UnregisterMemory(MemoryChunk* chunk) { - DCHECK(!chunk->IsFlagSet(MemoryChunk::UNREGISTERED)); - VirtualMemory* reservation = chunk->reserved_memory(); - const size_t size = - reservation->IsReserved() ? reservation->size() : chunk->size(); - DCHECK_GE(size_, static_cast(size)); - size_ -= size; - if (chunk->executable() == EXECUTABLE) { - DCHECK_GE(size_executable_, size); - size_executable_ -= size; - } - - if (chunk->executable()) UnregisterExecutableMemoryChunk(chunk); - chunk->SetFlag(MemoryChunk::UNREGISTERED); -} - -void MemoryAllocator::PreFreeMemory(MemoryChunk* chunk) { - DCHECK(!chunk->IsFlagSet(MemoryChunk::PRE_FREED)); - LOG(isolate_, DeleteEvent("MemoryChunk", chunk)); - UnregisterMemory(chunk); - isolate_->heap()->RememberUnmappedPage(reinterpret_cast
(chunk), - chunk->IsEvacuationCandidate()); - chunk->SetFlag(MemoryChunk::PRE_FREED); -} - -void MemoryAllocator::PerformFreeMemory(MemoryChunk* chunk) { - DCHECK(chunk->IsFlagSet(MemoryChunk::UNREGISTERED)); - DCHECK(chunk->IsFlagSet(MemoryChunk::PRE_FREED)); - chunk->ReleaseAllAllocatedMemory(); - - VirtualMemory* reservation = chunk->reserved_memory(); - if (chunk->IsFlagSet(MemoryChunk::POOLED)) { - UncommitMemory(reservation); - } else { - if (reservation->IsReserved()) { - reservation->Free(); - } else { - // Only read-only pages can have non-initialized reservation object. - DCHECK_EQ(RO_SPACE, chunk->owner_identity()); - FreeMemory(page_allocator(chunk->executable()), chunk->address(), - chunk->size()); - } - } -} - -template -void MemoryAllocator::Free(MemoryChunk* chunk) { - switch (mode) { - case kFull: - PreFreeMemory(chunk); - PerformFreeMemory(chunk); - break; - case kAlreadyPooled: - // Pooled pages cannot be touched anymore as their memory is uncommitted. - // Pooled pages are not-executable. - FreeMemory(data_page_allocator(), chunk->address(), - static_cast(MemoryChunk::kPageSize)); - break; - case kPooledAndQueue: - DCHECK_EQ(chunk->size(), static_cast(MemoryChunk::kPageSize)); - DCHECK_EQ(chunk->executable(), NOT_EXECUTABLE); - chunk->SetFlag(MemoryChunk::POOLED); - V8_FALLTHROUGH; - case kPreFreeAndQueue: - PreFreeMemory(chunk); - // The chunks added to this queue will be freed by a concurrent thread. - unmapper()->AddMemoryChunkSafe(chunk); - break; - } -} - -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< - MemoryAllocator::kFull>(MemoryChunk* chunk); - -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< - MemoryAllocator::kAlreadyPooled>(MemoryChunk* chunk); - -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< - MemoryAllocator::kPreFreeAndQueue>(MemoryChunk* chunk); - -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) void MemoryAllocator::Free< - MemoryAllocator::kPooledAndQueue>(MemoryChunk* chunk); - -template -Page* MemoryAllocator::AllocatePage(size_t size, SpaceType* owner, - Executability executable) { - MemoryChunk* chunk = nullptr; - if (alloc_mode == kPooled) { - DCHECK_EQ(size, static_cast( - MemoryChunkLayout::AllocatableMemoryInMemoryChunk( - owner->identity()))); - DCHECK_EQ(executable, NOT_EXECUTABLE); - chunk = AllocatePagePooled(owner); - } - if (chunk == nullptr) { - chunk = AllocateChunk(size, size, executable, owner); - } - if (chunk == nullptr) return nullptr; - return owner->InitializePage(chunk); -} - -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) - Page* MemoryAllocator::AllocatePage( - size_t size, PagedSpace* owner, Executability executable); -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) - Page* MemoryAllocator::AllocatePage( - size_t size, SemiSpace* owner, Executability executable); -template EXPORT_TEMPLATE_DEFINE(V8_EXPORT_PRIVATE) - Page* MemoryAllocator::AllocatePage( - size_t size, SemiSpace* owner, Executability executable); - -LargePage* MemoryAllocator::AllocateLargePage(size_t size, - LargeObjectSpace* owner, - Executability executable) { - MemoryChunk* chunk = AllocateChunk(size, size, executable, owner); - if (chunk == nullptr) return nullptr; - return LargePage::Initialize(isolate_->heap(), chunk, executable); -} - -template -MemoryChunk* MemoryAllocator::AllocatePagePooled(SpaceType* owner) { - MemoryChunk* chunk = unmapper()->TryGetPooledMemoryChunkSafe(); - if (chunk == nullptr) return nullptr; - const int size = MemoryChunk::kPageSize; - const Address start = reinterpret_cast
(chunk); - const Address area_start = - start + - MemoryChunkLayout::ObjectStartOffsetInMemoryChunk(owner->identity()); - const Address area_end = start + size; - // Pooled pages are always regular data pages. - DCHECK_NE(CODE_SPACE, owner->identity()); - VirtualMemory reservation(data_page_allocator(), start, size); - if (!CommitMemory(&reservation)) return nullptr; - if (Heap::ShouldZapGarbage()) { - ZapBlock(start, size, kZapValue); - } - MemoryChunk::Initialize(isolate_->heap(), start, size, area_start, area_end, - NOT_EXECUTABLE, owner, std::move(reservation)); - size_ += size; - return chunk; -} - -void MemoryAllocator::ZapBlock(Address start, size_t size, - uintptr_t zap_value) { - DCHECK(IsAligned(start, kTaggedSize)); - DCHECK(IsAligned(size, kTaggedSize)); - MemsetTagged(ObjectSlot(start), Object(static_cast
(zap_value)), - size >> kTaggedSizeLog2); -} - -intptr_t MemoryAllocator::GetCommitPageSize() { - if (FLAG_v8_os_page_size != 0) { - DCHECK(base::bits::IsPowerOfTwo(FLAG_v8_os_page_size)); - return FLAG_v8_os_page_size * KB; - } else { - return CommitPageSize(); - } -} - -base::AddressRegion MemoryAllocator::ComputeDiscardMemoryArea(Address addr, - size_t size) { - size_t page_size = MemoryAllocator::GetCommitPageSize(); - if (size < page_size + FreeSpace::kSize) { - return base::AddressRegion(0, 0); - } - Address discardable_start = RoundUp(addr + FreeSpace::kSize, page_size); - Address discardable_end = RoundDown(addr + size, page_size); - if (discardable_start >= discardable_end) return base::AddressRegion(0, 0); - return base::AddressRegion(discardable_start, - discardable_end - discardable_start); -} - -bool MemoryAllocator::CommitExecutableMemory(VirtualMemory* vm, Address start, - size_t commit_size, - size_t reserved_size) { - const size_t page_size = GetCommitPageSize(); - // All addresses and sizes must be aligned to the commit page size. - DCHECK(IsAligned(start, page_size)); - DCHECK_EQ(0, commit_size % page_size); - DCHECK_EQ(0, reserved_size % page_size); - const size_t guard_size = MemoryChunkLayout::CodePageGuardSize(); - const size_t pre_guard_offset = MemoryChunkLayout::CodePageGuardStartOffset(); - const size_t code_area_offset = - MemoryChunkLayout::ObjectStartOffsetInCodePage(); - // reserved_size includes two guard regions, commit_size does not. - DCHECK_LE(commit_size, reserved_size - 2 * guard_size); - const Address pre_guard_page = start + pre_guard_offset; - const Address code_area = start + code_area_offset; - const Address post_guard_page = start + reserved_size - guard_size; - // Commit the non-executable header, from start to pre-code guard page. - if (vm->SetPermissions(start, pre_guard_offset, PageAllocator::kReadWrite)) { - // Create the pre-code guard page, following the header. - if (vm->SetPermissions(pre_guard_page, page_size, - PageAllocator::kNoAccess)) { - // Commit the executable code body. - if (vm->SetPermissions(code_area, commit_size - pre_guard_offset, - PageAllocator::kReadWrite)) { - // Create the post-code guard page. - if (vm->SetPermissions(post_guard_page, page_size, - PageAllocator::kNoAccess)) { - UpdateAllocatedSpaceLimits(start, code_area + commit_size); - return true; - } - vm->SetPermissions(code_area, commit_size, PageAllocator::kNoAccess); - } - } - vm->SetPermissions(start, pre_guard_offset, PageAllocator::kNoAccess); - } - return false; -} - // ----------------------------------------------------------------------------- // PagedSpace implementation