diff --git a/src/D3D12MemAlloc.cpp b/src/D3D12MemAlloc.cpp index 4839268..24e8135 100644 --- a/src/D3D12MemAlloc.cpp +++ b/src/D3D12MemAlloc.cpp @@ -5985,662 +5985,6 @@ bool BlockMetadata_TLSF::CheckBlock( return true; } -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata_TLSF implementation - -BlockMetadata_TLSF::BlockMetadata_TLSF(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) - : BlockMetadata(allocationCallbacks, isVirtual), - m_AllocCount(0), - m_BlocksFreeCount(0), - m_BlocksFreeSize(0), - m_IsFreeBitmap(0), - m_MemoryClasses(0), - m_ListsCount(0), - m_FreeList(NULL), - m_BlockAllocator(*allocationCallbacks, INITIAL_BLOCK_ALLOC_COUNT), - m_NullBlock(NULL) -{ - D3D12MA_ASSERT(allocationCallbacks); -} - -BlockMetadata_TLSF::~BlockMetadata_TLSF() -{ - if (m_FreeList) - D3D12MA_DELETE_ARRAY(*GetAllocs(), m_FreeList, m_ListsCount); -} - -void BlockMetadata_TLSF::Init(UINT64 size) -{ - BlockMetadata::Init(size); - - m_NullBlock = m_BlockAllocator.Alloc(); - m_NullBlock->size = size; - m_NullBlock->offset = 0; - m_NullBlock->prevPhysical = NULL; - m_NullBlock->nextPhysical = NULL; - m_NullBlock->MarkFree(); - m_NullBlock->NextFree() = NULL; - m_NullBlock->PrevFree() = NULL; - UINT8 memoryClass = SizeToMemoryClass(size); - UINT16 sli = SizeToSecondIndex(size, memoryClass); - m_ListsCount = (memoryClass == 0 ? 0 : (memoryClass - 1) * (1UL << SECOND_LEVEL_INDEX) + sli) + 1; - if (IsVirtual()) - m_ListsCount += 1UL << SECOND_LEVEL_INDEX; - else - m_ListsCount += 4; - - m_MemoryClasses = memoryClass + 2; - memset(m_InnerIsFreeBitmap, 0, MAX_MEMORY_CLASSES * sizeof(UINT32)); - - m_FreeList = D3D12MA_NEW_ARRAY(*GetAllocs(), Block*, m_ListsCount); - memset(m_FreeList, 0, m_ListsCount * sizeof(Block*)); -} - -bool BlockMetadata_TLSF::Validate() const -{ - D3D12MA_VALIDATE(GetSumFreeSize() <= GetSize()); - - UINT64 calculatedSize = m_NullBlock->size; - UINT64 calculatedFreeSize = m_NullBlock->size; - size_t allocCount = 0; - size_t freeCount = 0; - - // Check integrity of free lists - for (UINT32 list = 0; list < m_ListsCount; ++list) - { - Block* block = m_FreeList[list]; - if (block != NULL) - { - D3D12MA_VALIDATE(block->IsFree()); - D3D12MA_VALIDATE(block->PrevFree() == NULL); - while (block->NextFree()) - { - D3D12MA_VALIDATE(block->NextFree()->IsFree()); - D3D12MA_VALIDATE(block->NextFree()->PrevFree() == block); - block = block->NextFree(); - } - } - } - - D3D12MA_VALIDATE(m_NullBlock->nextPhysical == NULL); - if (m_NullBlock->prevPhysical) - { - D3D12MA_VALIDATE(m_NullBlock->prevPhysical->nextPhysical == m_NullBlock); - } - - // Check all blocks - UINT64 nextOffset = m_NullBlock->offset; - for (Block* prev = m_NullBlock->prevPhysical; prev != NULL; prev = prev->prevPhysical) - { - D3D12MA_VALIDATE(prev->offset + prev->size == nextOffset); - nextOffset = prev->offset; - calculatedSize += prev->size; - - UINT32 listIndex = GetListIndex(prev->size); - if (prev->IsFree()) - { - ++freeCount; - // Check if free block belongs to free list - Block* freeBlock = m_FreeList[listIndex]; - D3D12MA_VALIDATE(freeBlock != NULL); - - bool found = false; - do - { - if (freeBlock == prev) - found = true; - - freeBlock = freeBlock->NextFree(); - } while (!found && freeBlock != NULL); - - D3D12MA_VALIDATE(found); - calculatedFreeSize += prev->size; - } - else - { - ++allocCount; - // Check if taken block is not on a free list - Block* freeBlock = m_FreeList[listIndex]; - while (freeBlock) - { - D3D12MA_VALIDATE(freeBlock != prev); - freeBlock = freeBlock->NextFree(); - } - } - - if (prev->prevPhysical) - { - D3D12MA_VALIDATE(prev->prevPhysical->nextPhysical == prev); - } - } - - D3D12MA_VALIDATE(nextOffset == 0); - D3D12MA_VALIDATE(calculatedSize == GetSize()); - D3D12MA_VALIDATE(calculatedFreeSize == GetSumFreeSize()); - D3D12MA_VALIDATE(allocCount == m_AllocCount); - D3D12MA_VALIDATE(freeCount == m_BlocksFreeCount); - - return true; -} - -void BlockMetadata_TLSF::GetAllocationInfo(AllocHandle allocHandle, VIRTUAL_ALLOCATION_INFO& outInfo) const -{ - Block* block = (Block*)allocHandle; - D3D12MA_ASSERT(!block->IsFree() && "Cannot get allocation info for free block!"); - outInfo.offset = block->offset; - outInfo.size = block->size; - outInfo.pUserData = block->UserData(); -} - -bool BlockMetadata_TLSF::CreateAllocationRequest( - UINT64 allocSize, - UINT64 allocAlignment, - AllocationRequest* pAllocationRequest) -{ - D3D12MA_ASSERT(allocSize > 0 && "Cannot allocate empty block!"); - D3D12MA_ASSERT(pAllocationRequest != NULL); - D3D12MA_HEAVY_ASSERT(Validate()); - - allocSize += D3D12MA_DEBUG_MARGIN; - // Quick check for too small pool - if (allocSize > GetSumFreeSize()) - return false; - - // If no free blocks in pool then check only null block - if (m_BlocksFreeCount == 0) - return CheckBlock(*m_NullBlock, m_ListsCount, allocSize, allocAlignment, pAllocationRequest); - - // Round up to the next block - UINT64 sizeForNextList = allocSize; - UINT64 smallSizeStep = SMALL_BUFFER_SIZE / (IsVirtual() ? 1 << SECOND_LEVEL_INDEX : 4); - if (allocSize > SMALL_BUFFER_SIZE) - { - sizeForNextList += (1ULL << (BitScanMSB(allocSize) - SECOND_LEVEL_INDEX)); - } - else if (allocSize > SMALL_BUFFER_SIZE - smallSizeStep) - sizeForNextList = SMALL_BUFFER_SIZE + 1; - else - sizeForNextList += smallSizeStep; - - // Check larger bucket - UINT32 nextListIndex = 0; - Block* nextListBlock = FindFreeBlock(sizeForNextList, nextListIndex); - while (nextListBlock) - { - if (CheckBlock(*nextListBlock, nextListIndex, allocSize, allocAlignment, pAllocationRequest)) - return true; - nextListBlock = nextListBlock->NextFree(); - } - - // If failed check null block - if (CheckBlock(*m_NullBlock, m_ListsCount, allocSize, allocAlignment, pAllocationRequest)) - return true; - - // Check best fit bucket - UINT32 prevListIndex = 0; - Block* prevListBlock = FindFreeBlock(allocSize, prevListIndex); - while (prevListBlock) - { - if (CheckBlock(*prevListBlock, prevListIndex, allocSize, allocAlignment, pAllocationRequest)) - return true; - prevListBlock = prevListBlock->NextFree(); - } - - // Worst case, full search has to be done - while (++nextListIndex < m_ListsCount) - { - nextListBlock = m_FreeList[nextListIndex]; - while (nextListBlock) - { - if (CheckBlock(*nextListBlock, nextListIndex, allocSize, allocAlignment, pAllocationRequest)) - return true; - nextListBlock = nextListBlock->NextFree(); - } - } - - // No more memory sadly - return false; -} - -void BlockMetadata_TLSF::Alloc( - const AllocationRequest& request, - UINT64 allocSize, - void* userData) -{ - // Get block and pop it from the free list - Block* currentBlock = (Block*)request.allocHandle; - UINT64 offset = request.algorithmData; - D3D12MA_ASSERT(currentBlock != NULL); - D3D12MA_ASSERT(currentBlock->offset <= offset); - - if (currentBlock != m_NullBlock) - RemoveFreeBlock(currentBlock); - - // Append missing alignment to prev block or create new one - UINT64 misssingAlignment = offset - currentBlock->offset; - if (misssingAlignment) - { - Block* prevBlock = currentBlock->prevPhysical; - D3D12MA_ASSERT(prevBlock != NULL && "There should be no missing alignment at offset 0!"); - - if (prevBlock->IsFree() && prevBlock->size != D3D12MA_DEBUG_MARGIN) - { - UINT32 oldList = GetListIndex(prevBlock->size); - prevBlock->size += misssingAlignment; - // Check if new size crosses list bucket - if (oldList != GetListIndex(prevBlock->size)) - { - prevBlock->size -= misssingAlignment; - RemoveFreeBlock(prevBlock); - prevBlock->size += misssingAlignment; - InsertFreeBlock(prevBlock); - } - else - m_BlocksFreeSize += misssingAlignment; - } - else - { - Block* newBlock = m_BlockAllocator.Alloc(); - currentBlock->prevPhysical = newBlock; - prevBlock->nextPhysical = newBlock; - newBlock->prevPhysical = prevBlock; - newBlock->nextPhysical = currentBlock; - newBlock->size = misssingAlignment; - newBlock->offset = currentBlock->offset; - newBlock->MarkTaken(); - - InsertFreeBlock(newBlock); - } - - currentBlock->size -= misssingAlignment; - currentBlock->offset += misssingAlignment; - } - - UINT64 size = request.size + D3D12MA_DEBUG_MARGIN; - if (currentBlock->size == size) - { - if (currentBlock == m_NullBlock) - { - // Setup new null block - m_NullBlock = m_BlockAllocator.Alloc(); - m_NullBlock->size = 0; - m_NullBlock->offset = currentBlock->offset + size; - m_NullBlock->prevPhysical = currentBlock; - m_NullBlock->nextPhysical = NULL; - m_NullBlock->MarkFree(); - m_NullBlock->PrevFree() = NULL; - m_NullBlock->NextFree() = NULL; - currentBlock->nextPhysical = m_NullBlock; - currentBlock->MarkTaken(); - } - } - else - { - D3D12MA_ASSERT(currentBlock->size > size && "Proper block already found, shouldn't find smaller one!"); - - // Create new free block - Block* newBlock = m_BlockAllocator.Alloc(); - newBlock->size = currentBlock->size - size; - newBlock->offset = currentBlock->offset + size; - newBlock->prevPhysical = currentBlock; - newBlock->nextPhysical = currentBlock->nextPhysical; - currentBlock->nextPhysical = newBlock; - currentBlock->size = size; - - if (currentBlock == m_NullBlock) - { - m_NullBlock = newBlock; - m_NullBlock->MarkFree(); - m_NullBlock->NextFree() = NULL; - m_NullBlock->PrevFree() = NULL; - currentBlock->MarkTaken(); - } - else - { - newBlock->nextPhysical->prevPhysical = newBlock; - newBlock->MarkTaken(); - InsertFreeBlock(newBlock); - } - } - currentBlock->UserData() = userData; - - if (D3D12MA_DEBUG_MARGIN > 0) - { - currentBlock->size -= D3D12MA_DEBUG_MARGIN; - Block* newBlock = m_BlockAllocator.Alloc(); - newBlock->size = D3D12MA_DEBUG_MARGIN; - newBlock->offset = currentBlock->offset + currentBlock->size; - newBlock->prevPhysical = currentBlock; - newBlock->nextPhysical = currentBlock->nextPhysical; - newBlock->MarkTaken(); - currentBlock->nextPhysical->prevPhysical = newBlock; - currentBlock->nextPhysical = newBlock; - InsertFreeBlock(newBlock); - } - ++m_AllocCount; -} - -void BlockMetadata_TLSF::Free(AllocHandle allocHandle) -{ - Block* block = (Block*)allocHandle; - Block* next = block->nextPhysical; - D3D12MA_ASSERT(!block->IsFree() && "Block is already free!"); - - --m_AllocCount; - if (D3D12MA_DEBUG_MARGIN > 0) - { - RemoveFreeBlock(next); - MergeBlock(next, block); - block = next; - next = next->nextPhysical; - } - - // Try merging - Block* prev = block->prevPhysical; - if (prev != NULL && prev->IsFree() && prev->size != D3D12MA_DEBUG_MARGIN) - { - RemoveFreeBlock(prev); - MergeBlock(block, prev); - } - - if (!next->IsFree()) - InsertFreeBlock(block); - else if (next == m_NullBlock) - MergeBlock(m_NullBlock, block); - else - { - RemoveFreeBlock(next); - MergeBlock(next, block); - InsertFreeBlock(next); - } -} - -void BlockMetadata_TLSF::Clear() -{ - m_AllocCount = 0; - m_BlocksFreeCount = 0; - m_BlocksFreeSize = 0; - m_IsFreeBitmap = 0; - m_NullBlock->offset = 0; - m_NullBlock->size = GetSize(); - Block* block = m_NullBlock->prevPhysical; - m_NullBlock->prevPhysical = NULL; - while (block) - { - Block* prev = block->prevPhysical; - m_BlockAllocator.Free(block); - block = prev; - } - memset(m_FreeList, 0, m_ListsCount * sizeof(Block*)); - memset(m_InnerIsFreeBitmap, 0, m_MemoryClasses * sizeof(UINT32)); -} - -void BlockMetadata_TLSF::SetAllocationUserData(AllocHandle allocHandle, void* userData) -{ - Block* block = (Block*)allocHandle; - D3D12MA_ASSERT(!block->IsFree() && "Trying to set user data for not allocated block!"); - block->UserData() = userData; -} - -void BlockMetadata_TLSF::CalcAllocationStatInfo(StatInfo& outInfo) const -{ - outInfo.BlockCount = 1; - outInfo.AllocationCount = static_cast(m_AllocCount); - outInfo.UnusedRangeCount = static_cast(m_BlocksFreeCount); - - outInfo.UsedBytes = GetSize() - GetSumFreeSize(); - outInfo.UnusedBytes = GetSumFreeSize(); - - outInfo.AllocationSizeMin = UINT64_MAX; - outInfo.AllocationSizeMax = 0; - outInfo.UnusedRangeSizeMin = UINT64_MAX; - outInfo.UnusedRangeSizeMax = 0; - - for (Block* block = m_NullBlock->prevPhysical; block != NULL; block = block->prevPhysical) - { - if (block->IsFree()) - { - outInfo.UnusedRangeSizeMin = D3D12MA_MIN(block->size, outInfo.UnusedRangeSizeMin); - outInfo.UnusedRangeSizeMax = D3D12MA_MAX(block->size, outInfo.UnusedRangeSizeMax); - } - else - { - outInfo.AllocationSizeMin = D3D12MA_MIN(block->size, outInfo.AllocationSizeMin); - outInfo.AllocationSizeMax = D3D12MA_MAX(block->size, outInfo.AllocationSizeMax); - } - } - - if (m_NullBlock->size > 0) - { - ++outInfo.UnusedRangeCount; - outInfo.UnusedRangeSizeMin = D3D12MA_MIN(m_NullBlock->size, outInfo.UnusedRangeSizeMin); - outInfo.UnusedRangeSizeMax = D3D12MA_MAX(m_NullBlock->size, outInfo.UnusedRangeSizeMax); - } -} - -void BlockMetadata_TLSF::WriteAllocationInfoToJson(JsonWriter& json) const -{ - json.BeginObject(); - json.WriteString(L"TotalBytes"); - json.WriteNumber(GetSize()); - json.WriteString(L"UnusuedBytes"); - json.WriteNumber(GetSumFreeSize()); - json.WriteString(L"Allocations"); - json.WriteNumber(GetAllocationCount()); - json.WriteString(L"UnusedRanges"); - json.WriteNumber(m_BlocksFreeCount + static_cast(m_NullBlock->size)); - json.WriteString(L"Suballocations"); - - size_t blockCount = m_AllocCount + m_BlocksFreeCount; - Vector blockList(blockCount, *GetAllocs()); - - size_t i = blockCount; - if (m_NullBlock->size > 0) - { - ++blockCount; - blockList.push_back(m_NullBlock); - } - for (Block* block = m_NullBlock->prevPhysical; block != NULL; block = block->prevPhysical) - { - blockList[--i] = block; - } - D3D12MA_ASSERT(i == 0); - - - json.BeginArray(); - for (; i < blockCount; ++i) - { - Block* block = blockList[i]; - json.BeginObject(true); - json.WriteString(L"Offset"); - json.WriteNumber(block->offset); - - if (block->IsFree()) - { - json.WriteString(L"Type"); - json.WriteString(L"FREE"); - json.WriteString(L"Size"); - json.WriteNumber(block->size); - } - else if (IsVirtual()) - { - json.WriteString(L"Type"); - json.WriteString(L"ALLOCATION"); - json.WriteString(L"Size"); - json.WriteNumber(block->size); - if (block->UserData()) - { - json.WriteString(L"UserData"); - json.WriteNumber((uintptr_t)block->UserData()); - } - } - else - { - const Allocation* const alloc = (const Allocation*)block->UserData(); - D3D12MA_ASSERT(alloc); - json.AddAllocationToObject(*alloc); - } - json.EndObject(); - } - json.EndArray(); - json.EndObject(); -} - -UINT8 BlockMetadata_TLSF::SizeToMemoryClass(UINT64 size) const -{ - if (size > SMALL_BUFFER_SIZE) - return BitScanMSB(size) - MEMORY_CLASS_SHIFT; - return 0; -} - -UINT16 BlockMetadata_TLSF::SizeToSecondIndex(UINT64 size, UINT8 memoryClass) const -{ - if (memoryClass == 0) - { - if (IsVirtual()) - return static_cast((size - 1) / 8); - else - return static_cast((size - 1) / 64); - } - return static_cast((size >> (memoryClass + MEMORY_CLASS_SHIFT - SECOND_LEVEL_INDEX)) ^ (1U << SECOND_LEVEL_INDEX)); -} - -UINT32 BlockMetadata_TLSF::GetListIndex(UINT8 memoryClass, UINT16 secondIndex) const -{ - if (memoryClass == 0) - return secondIndex; - - const UINT32 index = static_cast(memoryClass - 1) * (1 << SECOND_LEVEL_INDEX) + secondIndex; - if (IsVirtual()) - return index + (1 << SECOND_LEVEL_INDEX); - else - return index + 4; -} - -UINT32 BlockMetadata_TLSF::GetListIndex(UINT64 size) const -{ - UINT8 memoryClass = SizeToMemoryClass(size); - return GetListIndex(memoryClass, SizeToSecondIndex(size, memoryClass)); -} - -void BlockMetadata_TLSF::RemoveFreeBlock(Block* block) -{ - D3D12MA_ASSERT(block != m_NullBlock); - D3D12MA_ASSERT(block->IsFree()); - - if (block->NextFree() != NULL) - block->NextFree()->PrevFree() = block->PrevFree(); - if (block->PrevFree() != NULL) - block->PrevFree()->NextFree() = block->NextFree(); - else - { - UINT8 memClass = SizeToMemoryClass(block->size); - UINT16 secondIndex = SizeToSecondIndex(block->size, memClass); - UINT32 index = GetListIndex(memClass, secondIndex); - m_FreeList[index] = block->NextFree(); - if (block->NextFree() == NULL) - { - m_InnerIsFreeBitmap[memClass] &= ~(1U << secondIndex); - if (m_InnerIsFreeBitmap[memClass] == 0) - m_IsFreeBitmap &= ~(1UL << memClass); - } - } - block->MarkTaken(); - block->UserData() = NULL; - --m_BlocksFreeCount; - m_BlocksFreeSize -= block->size; -} - -void BlockMetadata_TLSF::InsertFreeBlock(Block* block) -{ - D3D12MA_ASSERT(block != m_NullBlock); - D3D12MA_ASSERT(!block->IsFree() && "Cannot insert block twice!"); - - UINT8 memClass = SizeToMemoryClass(block->size); - UINT16 secondIndex = SizeToSecondIndex(block->size, memClass); - UINT32 index = GetListIndex(memClass, secondIndex); - block->PrevFree() = NULL; - block->NextFree() = m_FreeList[index]; - m_FreeList[index] = block; - if (block->NextFree() != NULL) - block->NextFree()->PrevFree() = block; - else - { - m_InnerIsFreeBitmap[memClass] |= 1U << secondIndex; - m_IsFreeBitmap |= 1UL << memClass; - } - ++m_BlocksFreeCount; - m_BlocksFreeSize += block->size; -} - -void BlockMetadata_TLSF::MergeBlock(Block* block, Block* prev) -{ - D3D12MA_ASSERT(block->prevPhysical == prev && "Cannot merge seperate physical regions!"); - D3D12MA_ASSERT(!prev->IsFree() && "Cannot merge block that belongs to free list!"); - - block->offset = prev->offset; - block->size += prev->size; - block->prevPhysical = prev->prevPhysical; - if (block->prevPhysical) - block->prevPhysical->nextPhysical = block; - m_BlockAllocator.Free(prev); -} - -BlockMetadata_TLSF::Block* BlockMetadata_TLSF::FindFreeBlock(UINT64 size, UINT32& listIndex) const -{ - UINT8 memoryClass = SizeToMemoryClass(size); - UINT32 innerFreeMap = m_InnerIsFreeBitmap[memoryClass] & (~0U << SizeToSecondIndex(size, memoryClass)); - if (!innerFreeMap) - { - // Check higher levels for avaiable blocks - UINT32 freeMap = m_IsFreeBitmap & (~0UL << (memoryClass + 1)); - if (!freeMap) - return NULL; // No more memory avaible - - // Find lowest free region - memoryClass = BitScanLSB(freeMap); - innerFreeMap = m_InnerIsFreeBitmap[memoryClass]; - D3D12MA_ASSERT(innerFreeMap != 0); - } - // Find lowest free subregion - listIndex = GetListIndex(memoryClass, BitScanLSB(innerFreeMap)); - return m_FreeList[listIndex]; -} - -bool BlockMetadata_TLSF::CheckBlock( - Block& block, - UINT32 listIndex, - UINT64 allocSize, - UINT64 allocAlignment, - AllocationRequest* pAllocationRequest) -{ - D3D12MA_ASSERT(block.IsFree() && "Block is already taken!"); - - UINT64 alignedOffset = AlignUp(block.offset, allocAlignment); - if (block.size < allocSize + alignedOffset - block.offset) - return false; - - // Alloc successful - pAllocationRequest->allocHandle = (AllocHandle)█ - pAllocationRequest->size = allocSize - D3D12MA_DEBUG_MARGIN; - pAllocationRequest->algorithmData = alignedOffset; - - // Place block at the start of list if it's normal block - if (listIndex != m_ListsCount && block.PrevFree()) - { - block.PrevFree()->NextFree() = block.NextFree(); - if (block.NextFree()) - block.NextFree()->PrevFree() = block.PrevFree(); - block.PrevFree() = NULL; - block.NextFree() = m_FreeList[listIndex]; - m_FreeList[listIndex] = █ - if (block.NextFree()) - block.NextFree()->PrevFree() = █ - } - - return true; -} - //////////////////////////////////////////////////////////////////////////////// // Private class NormalBlock implementation