This is a follow-up CL to https://crrev.com/c/3623542. When updating pointers during a full GC, a page might not be swept already. In such cases there might be invalid objects in free memory. Since these objects might be dead, their maps might have been reclaimed already as well. The previous CL cached the size of invalid objects in order to avoid accessing an invalid object's map. However, as soon as a slot is within an invalid object, we also need to check whether this slot is still a tagged pointer which would require map access. This CL checks marking bits on invalid objects to skip that check on such invalid objects. Bug: v8:12578, chromium:1316289 Change-Id: Ie1d736f897a2994dbed7bfb95ed37732cd3b0882 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3596123 Reviewed-by: Michael Lippautz <mlippautz@chromium.org> Commit-Queue: Dominik Inführ <dinfuehr@chromium.org> Cr-Commit-Position: refs/heads/main@{#80609}
446 lines
17 KiB
C++
446 lines
17 KiB
C++
// Copyright 2017 the V8 project authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include "src/heap/heap-inl.h"
|
|
#include "src/heap/heap.h"
|
|
#include "src/heap/invalidated-slots-inl.h"
|
|
#include "src/heap/invalidated-slots.h"
|
|
#include "src/heap/memory-chunk.h"
|
|
#include "src/init/v8.h"
|
|
#include "test/cctest/cctest.h"
|
|
#include "test/cctest/heap/heap-tester.h"
|
|
#include "test/cctest/heap/heap-utils.h"
|
|
|
|
namespace v8 {
|
|
namespace internal {
|
|
namespace heap {
|
|
|
|
Page* HeapTester::AllocateByteArraysOnPage(
|
|
Heap* heap, std::vector<ByteArray>* byte_arrays) {
|
|
PauseAllocationObserversScope pause_observers(heap);
|
|
const int kLength = 256 - ByteArray::kHeaderSize;
|
|
const int kSize = ByteArray::SizeFor(kLength);
|
|
CHECK_EQ(kSize, 256);
|
|
PagedSpace* old_space = heap->old_space();
|
|
Page* page;
|
|
// Fill a page with byte arrays.
|
|
{
|
|
AlwaysAllocateScopeForTesting always_allocate(heap);
|
|
heap::SimulateFullSpace(old_space);
|
|
ByteArray byte_array;
|
|
CHECK(AllocateByteArrayForTest(heap, kLength, AllocationType::kOld)
|
|
.To(&byte_array));
|
|
byte_arrays->push_back(byte_array);
|
|
page = Page::FromHeapObject(byte_array);
|
|
size_t n = page->area_size() / kSize;
|
|
for (size_t i = 1; i < n; i++) {
|
|
CHECK(AllocateByteArrayForTest(heap, kLength, AllocationType::kOld)
|
|
.To(&byte_array));
|
|
byte_arrays->push_back(byte_array);
|
|
CHECK_EQ(page, Page::FromHeapObject(byte_array));
|
|
}
|
|
}
|
|
CHECK_NULL(page->invalidated_slots<OLD_TO_OLD>());
|
|
return page;
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsNoInvalidatedRanges) {
|
|
FLAG_stress_concurrent_allocation = false; // For AllocateByteArraysOnPage.
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (ByteArray byte_array : byte_arrays) {
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
CHECK(filter.IsValid(addr));
|
|
}
|
|
}
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsSomeInvalidatedRanges) {
|
|
FLAG_stress_concurrent_allocation = false; // For AllocateByteArraysOnPage.
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Register every second byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i += 2) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_OLD>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
if (i % 2 == 0) {
|
|
CHECK(!filter.IsValid(addr));
|
|
} else {
|
|
CHECK(filter.IsValid(addr));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsAllInvalidatedRanges) {
|
|
FLAG_stress_concurrent_allocation = false; // For AllocateByteArraysOnPage.
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Register the all byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_OLD>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
CHECK(!filter.IsValid(addr));
|
|
}
|
|
}
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsAfterTrimming) {
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Register the all byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_OLD>(
|
|
byte_arrays[i], ByteArray::kHeaderSize);
|
|
}
|
|
// Trim byte arrays and check that the slots outside the byte arrays are
|
|
// considered invalid if the old space page was swept.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
heap->RightTrimFixedArray(byte_array, byte_array.length());
|
|
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
CHECK_EQ(filter.IsValid(addr), page->SweepingDone());
|
|
}
|
|
}
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsEvacuationCandidate) {
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
page->MarkEvacuationCandidate();
|
|
// Register the all byte arrays as invalidated.
|
|
// This should be no-op because the page is marked as evacuation
|
|
// candidate.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_OLD>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
// All slots must still be valid.
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
CHECK(filter.IsValid(addr));
|
|
}
|
|
}
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsResetObjectRegression) {
|
|
FLAG_stress_concurrent_allocation = false; // For AllocateByteArraysOnPage.
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Ensure that the first array has smaller size then the rest.
|
|
heap->RightTrimFixedArray(byte_arrays[0], byte_arrays[0].length() - 8);
|
|
// Register the all byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_OLD>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
// All slots must still be invalid.
|
|
InvalidatedSlotsFilter filter = InvalidatedSlotsFilter::OldToOld(
|
|
page, InvalidatedSlotsFilter::LivenessCheck::kNo);
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
Address start = byte_array.address() + ByteArray::kHeaderSize;
|
|
Address end = byte_array.address() + byte_array.Size();
|
|
for (Address addr = start; addr < end; addr += kTaggedSize) {
|
|
CHECK(!filter.IsValid(addr));
|
|
}
|
|
}
|
|
}
|
|
|
|
Handle<FixedArray> AllocateArrayOnFreshPage(Isolate* isolate,
|
|
PagedSpace* old_space, int length) {
|
|
AlwaysAllocateScopeForTesting always_allocate(isolate->heap());
|
|
heap::SimulateFullSpace(old_space);
|
|
return isolate->factory()->NewFixedArray(length, AllocationType::kOld);
|
|
}
|
|
|
|
Handle<FixedArray> AllocateArrayOnEvacuationCandidate(Isolate* isolate,
|
|
PagedSpace* old_space,
|
|
int length) {
|
|
Handle<FixedArray> object =
|
|
AllocateArrayOnFreshPage(isolate, old_space, length);
|
|
heap::ForceEvacuationCandidate(Page::FromHeapObject(*object));
|
|
return object;
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsRightTrimFixedArray) {
|
|
if (!FLAG_incremental_marking) return;
|
|
FLAG_manual_evacuation_candidates_selection = true;
|
|
FLAG_parallel_compaction = false;
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Isolate* isolate = CcTest::i_isolate();
|
|
Factory* factory = isolate->factory();
|
|
Heap* heap = CcTest::heap();
|
|
HandleScope scope(isolate);
|
|
PagedSpace* old_space = heap->old_space();
|
|
// Allocate a dummy page to be swept be the sweeper during evacuation.
|
|
AllocateArrayOnFreshPage(isolate, old_space, 1);
|
|
Handle<FixedArray> evacuated =
|
|
AllocateArrayOnEvacuationCandidate(isolate, old_space, 1);
|
|
Handle<FixedArray> trimmed = AllocateArrayOnFreshPage(isolate, old_space, 10);
|
|
heap::SimulateIncrementalMarking(heap);
|
|
for (int i = 1; i < trimmed->length(); i++) {
|
|
trimmed->set(i, *evacuated);
|
|
}
|
|
{
|
|
HandleScope new_scope(isolate);
|
|
Handle<HeapObject> dead = factory->NewFixedArray(1);
|
|
for (int i = 1; i < trimmed->length(); i++) {
|
|
trimmed->set(i, *dead);
|
|
}
|
|
heap->RightTrimFixedArray(*trimmed, trimmed->length() - 1);
|
|
}
|
|
CcTest::CollectGarbage(i::NEW_SPACE);
|
|
CcTest::CollectGarbage(i::OLD_SPACE);
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsRightTrimLargeFixedArray) {
|
|
if (!FLAG_incremental_marking) return;
|
|
FLAG_manual_evacuation_candidates_selection = true;
|
|
FLAG_parallel_compaction = false;
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Isolate* isolate = CcTest::i_isolate();
|
|
Factory* factory = isolate->factory();
|
|
Heap* heap = CcTest::heap();
|
|
HandleScope scope(isolate);
|
|
PagedSpace* old_space = heap->old_space();
|
|
// Allocate a dummy page to be swept be the sweeper during evacuation.
|
|
AllocateArrayOnFreshPage(isolate, old_space, 1);
|
|
Handle<FixedArray> evacuated =
|
|
AllocateArrayOnEvacuationCandidate(isolate, old_space, 1);
|
|
Handle<FixedArray> trimmed;
|
|
{
|
|
AlwaysAllocateScopeForTesting always_allocate(heap);
|
|
trimmed = factory->NewFixedArray(
|
|
kMaxRegularHeapObjectSize / kTaggedSize + 100, AllocationType::kOld);
|
|
DCHECK(MemoryChunk::FromHeapObject(*trimmed)->InLargeObjectSpace());
|
|
}
|
|
heap::SimulateIncrementalMarking(heap);
|
|
for (int i = 1; i < trimmed->length(); i++) {
|
|
trimmed->set(i, *evacuated);
|
|
}
|
|
{
|
|
HandleScope new_scope(isolate);
|
|
Handle<HeapObject> dead = factory->NewFixedArray(1);
|
|
for (int i = 1; i < trimmed->length(); i++) {
|
|
trimmed->set(i, *dead);
|
|
}
|
|
heap->RightTrimFixedArray(*trimmed, trimmed->length() - 1);
|
|
}
|
|
CcTest::CollectGarbage(i::NEW_SPACE);
|
|
CcTest::CollectGarbage(i::OLD_SPACE);
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsLeftTrimFixedArray) {
|
|
if (!FLAG_incremental_marking) return;
|
|
FLAG_manual_evacuation_candidates_selection = true;
|
|
FLAG_parallel_compaction = false;
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Isolate* isolate = CcTest::i_isolate();
|
|
Factory* factory = isolate->factory();
|
|
Heap* heap = CcTest::heap();
|
|
HandleScope scope(isolate);
|
|
PagedSpace* old_space = heap->old_space();
|
|
// Allocate a dummy page to be swept be the sweeper during evacuation.
|
|
AllocateArrayOnFreshPage(isolate, old_space, 1);
|
|
Handle<FixedArray> evacuated =
|
|
AllocateArrayOnEvacuationCandidate(isolate, old_space, 1);
|
|
Handle<FixedArray> trimmed = AllocateArrayOnFreshPage(isolate, old_space, 10);
|
|
heap::SimulateIncrementalMarking(heap);
|
|
for (int i = 0; i + 1 < trimmed->length(); i++) {
|
|
trimmed->set(i, *evacuated);
|
|
}
|
|
{
|
|
HandleScope new_scope(isolate);
|
|
Handle<HeapObject> dead = factory->NewFixedArray(1);
|
|
for (int i = 1; i < trimmed->length(); i++) {
|
|
trimmed->set(i, *dead);
|
|
}
|
|
heap->LeftTrimFixedArray(*trimmed, trimmed->length() - 1);
|
|
}
|
|
CcTest::CollectGarbage(i::NEW_SPACE);
|
|
CcTest::CollectGarbage(i::OLD_SPACE);
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsFastToSlow) {
|
|
if (!FLAG_incremental_marking) return;
|
|
FLAG_manual_evacuation_candidates_selection = true;
|
|
FLAG_parallel_compaction = false;
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Isolate* isolate = CcTest::i_isolate();
|
|
Factory* factory = isolate->factory();
|
|
Heap* heap = CcTest::heap();
|
|
PagedSpace* old_space = heap->old_space();
|
|
|
|
HandleScope scope(isolate);
|
|
|
|
Handle<String> name = factory->InternalizeUtf8String("TestObject");
|
|
Handle<String> prop_name1 = factory->InternalizeUtf8String("prop1");
|
|
Handle<String> prop_name2 = factory->InternalizeUtf8String("prop2");
|
|
Handle<String> prop_name3 = factory->InternalizeUtf8String("prop3");
|
|
// Allocate a dummy page to be swept be the sweeper during evacuation.
|
|
AllocateArrayOnFreshPage(isolate, old_space, 1);
|
|
Handle<FixedArray> evacuated =
|
|
AllocateArrayOnEvacuationCandidate(isolate, old_space, 1);
|
|
// Allocate a dummy page to ensure that the JSObject is allocated on
|
|
// a fresh page.
|
|
AllocateArrayOnFreshPage(isolate, old_space, 1);
|
|
Handle<JSObject> obj;
|
|
{
|
|
AlwaysAllocateScopeForTesting always_allocate(heap);
|
|
Handle<JSFunction> function = factory->NewFunctionForTesting(name);
|
|
function->shared().set_expected_nof_properties(3);
|
|
obj = factory->NewJSObject(function, AllocationType::kOld);
|
|
}
|
|
// Start incremental marking.
|
|
heap::SimulateIncrementalMarking(heap);
|
|
// Set properties to point to the evacuation candidate.
|
|
Object::SetProperty(isolate, obj, prop_name1, evacuated).Check();
|
|
Object::SetProperty(isolate, obj, prop_name2, evacuated).Check();
|
|
Object::SetProperty(isolate, obj, prop_name3, evacuated).Check();
|
|
|
|
{
|
|
HandleScope new_scope(isolate);
|
|
Handle<HeapObject> dead = factory->NewFixedArray(1);
|
|
Object::SetProperty(isolate, obj, prop_name1, dead).Check();
|
|
Object::SetProperty(isolate, obj, prop_name2, dead).Check();
|
|
Object::SetProperty(isolate, obj, prop_name3, dead).Check();
|
|
Handle<Map> map(obj->map(), isolate);
|
|
Handle<Map> normalized_map =
|
|
Map::Normalize(isolate, map, CLEAR_INOBJECT_PROPERTIES, "testing");
|
|
JSObject::MigrateToMap(isolate, obj, normalized_map);
|
|
}
|
|
CcTest::CollectGarbage(i::NEW_SPACE);
|
|
CcTest::CollectGarbage(i::OLD_SPACE);
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsCleanupFull) {
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Register all byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_NEW>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
|
|
// Mark full page as free
|
|
InvalidatedSlotsCleanup cleanup = InvalidatedSlotsCleanup::OldToNew(page);
|
|
cleanup.Free(page->area_start(), page->area_end());
|
|
|
|
// After cleanup there should be no invalidated objects on page left
|
|
CHECK(page->invalidated_slots<OLD_TO_NEW>()->empty());
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsCleanupEachObject) {
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
// Register all byte arrays as invalidated.
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
ByteArray byte_array = byte_arrays[i];
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_NEW>(byte_array,
|
|
byte_array.Size());
|
|
}
|
|
|
|
// Mark each object as free on page
|
|
InvalidatedSlotsCleanup cleanup = InvalidatedSlotsCleanup::OldToNew(page);
|
|
|
|
for (size_t i = 0; i < byte_arrays.size(); i++) {
|
|
Address free_start = byte_arrays[i].address();
|
|
Address free_end = free_start + byte_arrays[i].Size();
|
|
cleanup.Free(free_start, free_end);
|
|
}
|
|
|
|
// After cleanup there should be no invalidated objects on page left
|
|
CHECK(page->invalidated_slots<OLD_TO_NEW>()->empty());
|
|
}
|
|
|
|
HEAP_TEST(InvalidatedSlotsCleanupRightTrim) {
|
|
ManualGCScope manual_gc_scope;
|
|
CcTest::InitializeVM();
|
|
Heap* heap = CcTest::heap();
|
|
std::vector<ByteArray> byte_arrays;
|
|
Page* page = AllocateByteArraysOnPage(heap, &byte_arrays);
|
|
|
|
CHECK_GT(byte_arrays.size(), 1);
|
|
ByteArray& invalidated = byte_arrays[1];
|
|
|
|
heap->RightTrimFixedArray(invalidated, invalidated.length() - 8);
|
|
page->RegisterObjectWithInvalidatedSlots<OLD_TO_NEW>(invalidated,
|
|
invalidated.Size());
|
|
|
|
// Free memory at end of invalidated object
|
|
InvalidatedSlotsCleanup cleanup = InvalidatedSlotsCleanup::OldToNew(page);
|
|
Address free_start = invalidated.address() + invalidated.Size();
|
|
cleanup.Free(free_start, page->area_end());
|
|
|
|
// After cleanup the invalidated object should be smaller
|
|
InvalidatedSlots* invalidated_slots = page->invalidated_slots<OLD_TO_NEW>();
|
|
CHECK_EQ(invalidated_slots->size(), 1);
|
|
}
|
|
|
|
} // namespace heap
|
|
} // namespace internal
|
|
} // namespace v8
|