diff --git a/include/vk_mem_alloc.h b/include/vk_mem_alloc.h index bdb4ff5..4f1c238 100644 --- a/include/vk_mem_alloc.h +++ b/include/vk_mem_alloc.h @@ -9988,8 +9988,11 @@ private: void MarkFree() { prevFree = VMA_NULL; } void MarkTaken() { prevFree = this; } - bool IsFree() const { return prevFree != this; } - void*& UserData() { VMA_HEAVY_ASSERT(!IsFree()); return userData; } + void MarkMargin() { prevFree = reinterpret_cast(UINTPTR_MAX); } + bool IsFree() const { return !IsTaken() && !IsMargin(); } + bool IsTaken() const { return prevFree == this; } + bool IsMargin() const { return prevFree == reinterpret_cast(UINTPTR_MAX); } + void*& UserData() { VMA_HEAVY_ASSERT(IsTaken()); return userData; } Block*& PrevFree() { return prevFree; } Block*& NextFree() { VMA_HEAVY_ASSERT(IsFree()); return nextFree; } @@ -10154,8 +10157,7 @@ bool VmaBlockMetadata_TLSF::Validate() const } else { - ++allocCount; - // Check if taken block is not on a free list + // Check if taken or margin block is not on a free list Block* freeBlock = m_FreeList[listIndex]; while (freeBlock) { @@ -10163,9 +10165,22 @@ bool VmaBlockMetadata_TLSF::Validate() const freeBlock = freeBlock->NextFree(); } - if (!IsVirtual()) + if (prev->IsMargin()) { - VMA_VALIDATE(m_GranularityHandler.Validate(validateCtx, prev->offset, prev->size)); + // Margin block belongs to free block category, + // but is not granted seat on the free list + ++freeCount; + calculatedFreeSize += prev->size; + VMA_VALIDATE(prev->prevPhysical != VMA_NULL); + VMA_VALIDATE(prev->prevPhysical->IsTaken()); + } + else + { + ++allocCount; + if (!IsVirtual()) + { + VMA_VALIDATE(m_GranularityHandler.Validate(validateCtx, prev->offset, prev->size)); + } } } @@ -10198,10 +10213,10 @@ void VmaBlockMetadata_TLSF::AddDetailedStatistics(VmaDetailedStatistics& inoutSt for (Block* block = m_NullBlock->prevPhysical; block != VMA_NULL; block = block->prevPhysical) { - if (block->IsFree()) - VmaAddDetailedStatisticsUnusedRange(inoutStats, block->size); - else + if (block->IsTaken()) VmaAddDetailedStatisticsAllocation(inoutStats, block->size); + else + VmaAddDetailedStatisticsUnusedRange(inoutStats, block->size); } } @@ -10239,10 +10254,10 @@ void VmaBlockMetadata_TLSF::PrintDetailedMap(class VmaJsonWriter& json) const for (; i < blockCount; ++i) { Block* block = blockList[i]; - if (block->IsFree()) - PrintDetailedMap_UnusedRange(json, block->offset, block->size); - else + if (block->IsTaken()) PrintDetailedMap_Allocation(json, block->offset, block->size, block->UserData()); + else + PrintDetailedMap_UnusedRange(json, block->offset, block->size); } if (m_NullBlock->size > 0) PrintDetailedMap_UnusedRange(json, m_NullBlock->offset, m_NullBlock->size); @@ -10417,7 +10432,7 @@ VkResult VmaBlockMetadata_TLSF::CheckCorruption(const void* pBlockData) { for (Block* block = m_NullBlock->prevPhysical; block != VMA_NULL; block = block->prevPhysical) { - if (!block->IsFree()) + if (block->IsTaken()) { if (!VmaValidateMagicValue(pBlockData, block->offset + block->size)) { @@ -10446,7 +10461,6 @@ void VmaBlockMetadata_TLSF::Alloc( if (currentBlock != m_NullBlock) RemoveFreeBlock(currentBlock); - VkDeviceSize debugMargin = GetDebugMargin(); VkDeviceSize misssingAlignment = offset - currentBlock->offset; // Append missing alignment to prev block or create new one @@ -10455,7 +10469,7 @@ void VmaBlockMetadata_TLSF::Alloc( Block* prevBlock = currentBlock->prevPhysical; VMA_ASSERT(prevBlock != VMA_NULL && "There should be no missing alignment at offset 0!"); - if (prevBlock->IsFree() && prevBlock->size != debugMargin) + if (prevBlock->IsFree()) { uint32_t oldList = GetListIndex(prevBlock->size); prevBlock->size += misssingAlignment; @@ -10488,6 +10502,7 @@ void VmaBlockMetadata_TLSF::Alloc( currentBlock->offset += misssingAlignment; } + VkDeviceSize debugMargin = GetDebugMargin(); VkDeviceSize size = request.size + debugMargin; if (currentBlock->size == size) { @@ -10544,10 +10559,13 @@ void VmaBlockMetadata_TLSF::Alloc( newBlock->offset = currentBlock->offset + currentBlock->size; newBlock->prevPhysical = currentBlock; newBlock->nextPhysical = currentBlock->nextPhysical; - newBlock->MarkTaken(); + newBlock->MarkMargin(); currentBlock->nextPhysical->prevPhysical = newBlock; currentBlock->nextPhysical = newBlock; - InsertFreeBlock(newBlock); + // Count margin block into stats, but don't place it on free list + // as it will never be selected for allocation + ++m_BlocksFreeCount; + m_BlocksFreeSize += debugMargin; } if (!IsVirtual()) @@ -10560,7 +10578,7 @@ void VmaBlockMetadata_TLSF::Free(VmaAllocHandle allocHandle) { Block* block = (Block*)allocHandle; Block* next = block->nextPhysical; - VMA_ASSERT(!block->IsFree() && "Block is already free!"); + VMA_ASSERT(block->IsTaken() && "Block is already free or margin!"); if (!IsVirtual()) m_GranularityHandler.FreePages(block->offset, block->size); @@ -10569,7 +10587,9 @@ void VmaBlockMetadata_TLSF::Free(VmaAllocHandle allocHandle) VkDeviceSize debugMargin = GetDebugMargin(); if (debugMargin > 0) { - RemoveFreeBlock(next); + // Adjust stats for one less block + --m_BlocksFreeCount; + m_BlocksFreeSize -= block->size; MergeBlock(next, block); block = next; next = next->nextPhysical; @@ -10598,7 +10618,7 @@ void VmaBlockMetadata_TLSF::Free(VmaAllocHandle allocHandle) void VmaBlockMetadata_TLSF::GetAllocationInfo(VmaAllocHandle allocHandle, VmaVirtualAllocationInfo& outInfo) { Block* block = (Block*)allocHandle; - VMA_ASSERT(!block->IsFree() && "Cannot get allocation info for free block!"); + VMA_ASSERT(block->IsTaken() && "Cannot get allocation info for not taken block!"); outInfo.offset = block->offset; outInfo.size = block->size; outInfo.pUserData = block->UserData(); @@ -10607,7 +10627,7 @@ void VmaBlockMetadata_TLSF::GetAllocationInfo(VmaAllocHandle allocHandle, VmaVir void* VmaBlockMetadata_TLSF::GetAllocationUserData(VmaAllocHandle allocHandle) const { Block* block = (Block*)allocHandle; - VMA_ASSERT(!block->IsFree() && "Cannot get user data for free block!"); + VMA_ASSERT(block->IsTaken() && "Cannot get user data for not taken block!"); return block->UserData(); } @@ -10618,7 +10638,7 @@ VmaAllocHandle VmaBlockMetadata_TLSF::GetAllocationListBegin() const for (Block* block = m_NullBlock->prevPhysical; block; block = block->prevPhysical) { - if (!block->IsFree()) + if (block->IsTaken()) return (VmaAllocHandle)block; } VMA_ASSERT(false && "If m_AllocCount > 0 then should find any allocation!"); @@ -10628,11 +10648,11 @@ VmaAllocHandle VmaBlockMetadata_TLSF::GetAllocationListBegin() const VmaAllocHandle VmaBlockMetadata_TLSF::GetNextAllocation(VmaAllocHandle prevAlloc) const { Block* startBlock = (Block*)prevAlloc; - VMA_ASSERT(!startBlock->IsFree() && "Incorrect block!"); + VMA_ASSERT(startBlock->IsTaken() && "Incorrect block!"); for (Block* block = startBlock->prevPhysical; block; block = block->prevPhysical) { - if (!block->IsFree()) + if (block->IsTaken()) return (VmaAllocHandle)block; } return VK_NULL_HANDLE; @@ -10641,7 +10661,7 @@ VmaAllocHandle VmaBlockMetadata_TLSF::GetNextAllocation(VmaAllocHandle prevAlloc VkDeviceSize VmaBlockMetadata_TLSF::GetNextFreeRegionSize(VmaAllocHandle alloc) const { Block* block = (Block*)alloc; - VMA_ASSERT(!block->IsFree() && "Incorrect block!"); + VMA_ASSERT(block->IsTaken() && "Incorrect block!"); if (block->prevPhysical) return block->prevPhysical->IsFree() ? block->prevPhysical->size : 0; @@ -10672,14 +10692,14 @@ void VmaBlockMetadata_TLSF::Clear() void VmaBlockMetadata_TLSF::SetAllocationUserData(VmaAllocHandle allocHandle, void* userData) { Block* block = (Block*)allocHandle; - VMA_ASSERT(!block->IsFree() && "Trying to set user data for not allocated block!"); + VMA_ASSERT(block->IsTaken() && "Trying to set user data for not allocated block!"); block->UserData() = userData; } void VmaBlockMetadata_TLSF::DebugLogAllAllocations() const { for (Block* block = m_NullBlock->prevPhysical; block != VMA_NULL; block = block->prevPhysical) - if (!block->IsFree()) + if (block->IsTaken()) DebugLogAllocation(block->offset, block->size, block->UserData()); } diff --git a/src/Tests.cpp b/src/Tests.cpp index c25b23c..eb586d9 100644 --- a/src/Tests.cpp +++ b/src/Tests.cpp @@ -3943,10 +3943,6 @@ void TestHeapSizeLimit() vmaDestroyAllocator(hAllocator); } -#ifndef VMA_DEBUG_MARGIN - #define VMA_DEBUG_MARGIN (0) -#endif - static void TestDebugMargin() { if(VMA_DEBUG_MARGIN == 0) @@ -3990,7 +3986,7 @@ static void TestDebugMargin() const bool isLast = allocIndex == BUF_COUNT - 1; bufInfo.size = (VkDeviceSize)(allocIndex + 1) * 256; // Last one will be mapped. - allocCreateInfo.flags = isLast ? VMA_ALLOCATION_CREATE_MAPPED_BIT : 0; + allocCreateInfo.flags = isLast ? VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT : 0; VkResult res = vmaCreateBuffer(g_hAllocator, &bufInfo, &allocCreateInfo, &buffers[allocIndex].Buffer, &buffers[allocIndex].Allocation, &allocInfo[allocIndex]); TEST(res == VK_SUCCESS);