From 2a318103dac4b56bfdeae17cd23641f93a760bfc Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Thu, 3 Mar 2022 15:22:36 +0100 Subject: [PATCH] Refactored internal code structure similary to VMA. Code by @medranSolus --- src/D3D12MemAlloc.cpp | 8293 +++++++++++++++++++++-------------------- 1 file changed, 4173 insertions(+), 4120 deletions(-) diff --git a/src/D3D12MemAlloc.cpp b/src/D3D12MemAlloc.cpp index 92b9008..46c44b2 100644 --- a/src/D3D12MemAlloc.cpp +++ b/src/D3D12MemAlloc.cpp @@ -22,20 +22,12 @@ #include "D3D12MemAlloc.h" -#ifndef D3D12MA_D3D12_HEADERS_ALREADY_INCLUDED - #include - #if D3D12MA_DXGI_1_4 - #include - #endif -#endif - #include #include #include #include #include #include // for _aligned_malloc, _aligned_free - #ifndef _WIN32 #include #endif @@ -47,6 +39,20 @@ // //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// +#ifndef _D3D12MA_CONFIGURATION + +#ifdef _WIN32 + #if !defined(WINVER) || WINVER < 0x0600 + #error Required at least WinAPI version supporting: client = Windows Vista, server = Windows Server 2008. + #endif +#endif + +#ifndef D3D12MA_D3D12_HEADERS_ALREADY_INCLUDED + #include + #if D3D12MA_DXGI_1_4 + #include + #endif +#endif #ifndef D3D12MA_ASSERT #include @@ -96,6 +102,7 @@ especially to test compatibility with D3D12_RESOURCE_HEAP_TIER_1 on modern GPUs. #define D3D12MA_DEFAULT_BLOCK_SIZE (64ull * 1024 * 1024) #endif +#endif // _D3D12MA_CONFIGURATION //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// // @@ -108,9 +115,49 @@ especially to test compatibility with D3D12_RESOURCE_HEAP_TIER_1 on modern GPUs. namespace D3D12MA { +static constexpr UINT HEAP_TYPE_COUNT = 4; +static constexpr UINT STANDARD_HEAP_TYPE_COUNT = 3; // Only DEFAULT, UPLOAD, READBACK. +static constexpr UINT DEFAULT_POOL_MAX_COUNT = 9; +static const UINT NEW_BLOCK_SIZE_SHIFT_MAX = 3; +// Minimum size of a free suballocation to register it in the free suballocation collection. +static const UINT64 MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16; -//////////////////////////////////////////////////////////////////////////////// -// Private globals - CPU memory allocation +static const WCHAR* const HeapTypeNames[] = +{ + L"DEFAULT", + L"UPLOAD", + L"READBACK", + L"CUSTOM", +}; + +static const D3D12_HEAP_FLAGS RESOURCE_CLASS_HEAP_FLAGS = + D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; + +#ifndef _D3D12MA_ENUM_DECLARATIONS + +// Local copy of this enum, as it is provided only by , so it may not be available. +enum DXGI_MEMORY_SEGMENT_GROUP_COPY +{ + DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY = 0, + DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY = 1, + DXGI_MEMORY_SEGMENT_GROUP_COUNT +}; + +enum class ResourceClass +{ + Unknown, Buffer, Non_RT_DS_Texture, RT_DS_Texture +}; + +enum SuballocationType +{ + SUBALLOCATION_TYPE_FREE = 0, + SUBALLOCATION_TYPE_ALLOCATION = 1, +}; + +#endif // _D3D12MA_ENUM_DECLARATIONS + + +#ifndef _D3D12MA_FUNCTIONS static void* DefaultAllocate(size_t Size, size_t Alignment, void* /*pUserData*/) { @@ -157,7 +204,7 @@ static T* AllocateArray(const ALLOCATION_CALLBACKS& allocs, size_t count) template void D3D12MA_DELETE(const ALLOCATION_CALLBACKS& allocs, T* memory) { - if(memory) + if (memory) { memory->~T(); Free(allocs, memory); @@ -166,9 +213,9 @@ void D3D12MA_DELETE(const ALLOCATION_CALLBACKS& allocs, T* memory) template void D3D12MA_DELETE_ARRAY(const ALLOCATION_CALLBACKS& allocs, T* memory, size_t count) { - if(memory) + if (memory) { - for(size_t i = count; i--; ) + for (size_t i = count; i--; ) { memory[i].~T(); } @@ -178,7 +225,7 @@ void D3D12MA_DELETE_ARRAY(const ALLOCATION_CALLBACKS& allocs, T* memory, size_t static void SetupAllocationCallbacks(ALLOCATION_CALLBACKS& outAllocs, const ALLOCATION_CALLBACKS* allocationCallbacks) { - if(allocationCallbacks) + if (allocationCallbacks) { outAllocs = *allocationCallbacks; D3D12MA_ASSERT(outAllocs.pAllocate != NULL && outAllocs.pFree != NULL); @@ -191,84 +238,23 @@ static void SetupAllocationCallbacks(ALLOCATION_CALLBACKS& outAllocs, const ALLO } } -//////////////////////////////////////////////////////////////////////////////// -// Private globals - basic facilities - #define SAFE_RELEASE(ptr) do { if(ptr) { (ptr)->Release(); (ptr) = NULL; } } while(false) #define D3D12MA_VALIDATE(cond) do { if(!(cond)) { \ - D3D12MA_ASSERT(0 && "Validation failed: " #cond); \ - return false; \ - } } while(false) - -const UINT NEW_BLOCK_SIZE_SHIFT_MAX = 3; + D3D12MA_ASSERT(0 && "Validation failed: " #cond); \ + return false; \ +} } while(false) template -static inline T D3D12MA_MIN(const T& a, const T& b) -{ - return a <= b ? a : b; -} +static T D3D12MA_MIN(const T& a, const T& b) { return a <= b ? a : b; } template -static inline T D3D12MA_MAX(const T& a, const T& b) -{ - return a <= b ? b : a; -} +static T D3D12MA_MAX(const T& a, const T& b) { return a <= b ? b : a; } template -static inline void D3D12MA_SWAP(T& a, T& b) -{ - T tmp = a; a = b; b = tmp; -} - -#ifndef D3D12MA_MUTEX - class Mutex - { - public: - void Lock() { m_Mutex.lock(); } - void Unlock() { m_Mutex.unlock(); } - private: - std::mutex m_Mutex; - }; - #define D3D12MA_MUTEX Mutex -#endif - -#ifdef _WIN32 - #if !defined(WINVER) || WINVER < 0x0600 - #error Required at least WinAPI version supporting: client = Windows Vista, server = Windows Server 2008. - #endif -#endif // #ifdef _WIN32 - -#ifndef D3D12MA_RW_MUTEX -#ifdef _WIN32 - class RWMutex - { - public: - RWMutex() { InitializeSRWLock(&m_Lock); } - void LockRead() { AcquireSRWLockShared(&m_Lock); } - void UnlockRead() { ReleaseSRWLockShared(&m_Lock); } - void LockWrite() { AcquireSRWLockExclusive(&m_Lock); } - void UnlockWrite() { ReleaseSRWLockExclusive(&m_Lock); } - private: - SRWLOCK m_Lock; - }; -#else // #ifdef _WIN32 - class RWMutex - { - public: - RWMutex() {} - void LockRead() { m_Mutex.lock_shared(); } - void UnlockRead() { m_Mutex.unlock_shared(); } - void LockWrite() { m_Mutex.lock(); } - void UnlockWrite() { m_Mutex.unlock(); } - private: - std::shared_timed_mutex m_Mutex; - }; -#endif // #ifdef _WIN32 - #define D3D12MA_RW_MUTEX RWMutex -#endif // #ifndef D3D12MA_RW_MUTEX +static void D3D12MA_SWAP(T& a, T& b) { T tmp = a; a = b; b = tmp; } // Scans integer for index of first nonzero bit from the Least Significant Bit (LSB). If mask is 0 then returns UINT8_MAX -static inline UINT8 BitScanLSB(UINT64 mask) +static UINT8 BitScanLSB(UINT64 mask) { #if defined(_MSC_VER) && defined(_WIN64) unsigned long pos; @@ -289,9 +275,8 @@ static inline UINT8 BitScanLSB(UINT64 mask) return UINT8_MAX; #endif } - // Scans integer for index of first nonzero bit from the Least Significant Bit (LSB). If mask is 0 then returns UINT8_MAX -static inline UINT8 BitScanLSB(UINT32 mask) +static UINT8 BitScanLSB(UINT32 mask) { #ifdef _MSC_VER unsigned long pos; @@ -314,7 +299,7 @@ static inline UINT8 BitScanLSB(UINT32 mask) } // Scans integer for index of first nonzero bit from the Most Significant Bit (MSB). If mask is 0 then returns UINT8_MAX -static inline UINT8 BitScanMSB(UINT64 mask) +static UINT8 BitScanMSB(UINT64 mask) { #if defined(_MSC_VER) && defined(_WIN64) unsigned long pos; @@ -335,9 +320,8 @@ static inline UINT8 BitScanMSB(UINT64 mask) #endif return UINT8_MAX; } - // Scans integer for index of first nonzero bit from the Most Significant Bit (MSB). If mask is 0 then returns UINT8_MAX -static inline UINT8 BitScanMSB(UINT32 mask) +static UINT8 BitScanMSB(UINT32 mask) { #ifdef _MSC_VER unsigned long pos; @@ -365,23 +349,20 @@ T must be unsigned integer number or signed integer but always nonnegative. For 0 returns true. */ template -inline bool IsPow2(T x) -{ - return (x & (x-1)) == 0; -} +static bool IsPow2(T x) { return (x & (x - 1)) == 0; } // Aligns given value up to nearest multiply of align value. For example: AlignUp(11, 8) = 16. // Use types like UINT, uint64_t as T. template -static inline T AlignUp(T val, T alignment) +static T AlignUp(T val, T alignment) { D3D12MA_HEAVY_ASSERT(IsPow2(alignment)); - return (val + alignment - 1) & ~(alignment - 1); + return (val + alignment - 1) & ~(alignment - 1); } // Aligns given value down to nearest multiply of align value. For example: AlignUp(11, 8) = 8. // Use types like UINT, uint64_t as T. template -static inline T AlignDown(T val, T alignment) +static T AlignDown(T val, T alignment) { D3D12MA_HEAVY_ASSERT(IsPow2(alignment)); return val & ~(alignment - 1); @@ -389,109 +370,10 @@ static inline T AlignDown(T val, T alignment) // Division with mathematical rounding to nearest number. template -static inline T RoundDiv(T x, T y) -{ - return (x + (y / (T)2)) / y; -} +static T RoundDiv(T x, T y) { return (x + (y / (T)2)) / y; } template -static inline T DivideRoundingUp(T x, T y) -{ - return (x + y - 1) / y; -} - -// Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope). -struct MutexLock -{ -public: - MutexLock(D3D12MA_MUTEX& mutex, bool useMutex = true) : - m_pMutex(useMutex ? &mutex : NULL) - { - if(m_pMutex) - { - m_pMutex->Lock(); - } - } - ~MutexLock() - { - if(m_pMutex) - { - m_pMutex->Unlock(); - } - } -private: - D3D12MA_MUTEX* m_pMutex; - - D3D12MA_CLASS_NO_COPY(MutexLock) -}; - -// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for reading. -struct MutexLockRead -{ -public: - MutexLockRead(D3D12MA_RW_MUTEX& mutex, bool useMutex) : - m_pMutex(useMutex ? &mutex : NULL) - { - if(m_pMutex) - { - m_pMutex->LockRead(); - } - } - ~MutexLockRead() - { - if(m_pMutex) - { - m_pMutex->UnlockRead(); - } - } -private: - D3D12MA_RW_MUTEX* m_pMutex; - - D3D12MA_CLASS_NO_COPY(MutexLockRead) -}; - -// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for writing. -struct MutexLockWrite -{ -public: - MutexLockWrite(D3D12MA_RW_MUTEX& mutex, bool useMutex) : - m_pMutex(useMutex ? &mutex : NULL) - { - if(m_pMutex) - { - m_pMutex->LockWrite(); - } - } - ~MutexLockWrite() - { - if(m_pMutex) - { - m_pMutex->UnlockWrite(); - } - } -private: - D3D12MA_RW_MUTEX* m_pMutex; - - D3D12MA_CLASS_NO_COPY(MutexLockWrite) -}; - -#if D3D12MA_DEBUG_GLOBAL_MUTEX - static D3D12MA_MUTEX g_DebugGlobalMutex; - #define D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK MutexLock debugGlobalMutexLock(g_DebugGlobalMutex, true); -#else - #define D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK -#endif - -// Minimum size of a free suballocation to register it in the free suballocation collection. -static const UINT64 MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16; - -// Local copy of this enum, as it is provided only by , so it may not be available. -enum DXGI_MEMORY_SEGMENT_GROUP_COPY -{ - DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY = 0, - DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY = 1, - DXGI_MEMORY_SEGMENT_GROUP_COUNT -}; - +static T DivideRoundingUp(T x, T y) { return (x + y - 1) / y; } + /* Performs binary search and returns iterator to first element that is greater or equal to `key`, according to comparison `cmp`. @@ -502,13 +384,13 @@ Returned value is the found element, if present in the collection or place where new element with value (key) should be inserted. */ template -static IterT BinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, const CmpLess& cmp) +static IterT BinaryFindFirstNotLess(IterT beg, IterT end, const KeyT& key, const CmpLess& cmp) { size_t down = 0, up = (end - beg); - while(down < up) + while (down < up) { const size_t mid = (down + up) / 2; - if(cmp(*(beg+mid), key)) + if (cmp(*(beg + mid), key)) { down = mid + 1; } @@ -530,10 +412,10 @@ Returned value is the found element, if present in the collection or end if not found. */ template -IterT BinaryFindSorted(const IterT& beg, const IterT& end, const KeyT& value, const CmpLess& cmp) +static IterT BinaryFindSorted(const IterT& beg, const IterT& end, const KeyT& value, const CmpLess& cmp) { IterT it = BinaryFindFirstNotLess(beg, end, value, cmp); - if(it == end || + if (it == end || (!cmp(*it, value) && !cmp(value, *it))) { return it; @@ -543,7 +425,7 @@ IterT BinaryFindSorted(const IterT& beg, const IterT& end, const KeyT& value, co static UINT HeapTypeToIndex(D3D12_HEAP_TYPE type) { - switch(type) + switch (type) { case D3D12_HEAP_TYPE_DEFAULT: return 0; case D3D12_HEAP_TYPE_UPLOAD: return 1; @@ -560,66 +442,6 @@ static D3D12_HEAP_TYPE IndexToHeapType(UINT heapTypeIndex) return (D3D12_HEAP_TYPE)(heapTypeIndex + 1); } -static const WCHAR* const HeapTypeNames[] = { - L"DEFAULT", - L"UPLOAD", - L"READBACK", - L"CUSTOM", -}; - -// Stat helper functions - -static void ClearStatistics(Statistics& outStats) -{ - outStats.BlockCount = 0; - outStats.AllocationCount = 0; - outStats.BlockBytes = 0; - outStats.AllocationBytes = 0; -} - -static void ClearDetailedStatistics(DetailedStatistics& outStats) -{ - ClearStatistics(outStats.Stats); - outStats.UnusedRangeCount = 0; - outStats.AllocationSizeMin = UINT64_MAX; - outStats.AllocationSizeMax = 0; - outStats.UnusedRangeSizeMin = UINT64_MAX; - outStats.UnusedRangeSizeMax = 0; -} - -static void AddStatistics(Statistics& inoutStats, const Statistics& src) -{ - inoutStats.BlockCount += src.BlockCount; - inoutStats.AllocationCount += src.AllocationCount; - inoutStats.BlockBytes += src.BlockBytes; - inoutStats.AllocationBytes += src.AllocationBytes; -} - -static void AddDetailedStatistics(DetailedStatistics& inoutStats, const DetailedStatistics& src) -{ - AddStatistics(inoutStats.Stats, src.Stats); - inoutStats.UnusedRangeCount += src.UnusedRangeCount; - inoutStats.AllocationSizeMin = D3D12MA_MIN(inoutStats.AllocationSizeMin, src.AllocationSizeMin); - inoutStats.AllocationSizeMax = D3D12MA_MAX(inoutStats.AllocationSizeMax, src.AllocationSizeMax); - inoutStats.UnusedRangeSizeMin = D3D12MA_MIN(inoutStats.UnusedRangeSizeMin, src.UnusedRangeSizeMin); - inoutStats.UnusedRangeSizeMax = D3D12MA_MAX(inoutStats.UnusedRangeSizeMax, src.UnusedRangeSizeMax); -} - -static void AddDetailedStatisticsAllocation(DetailedStatistics& inoutStats, UINT64 size) -{ - inoutStats.Stats.AllocationCount++; - inoutStats.Stats.AllocationBytes += size; - inoutStats.AllocationSizeMin = D3D12MA_MIN(inoutStats.AllocationSizeMin, size); - inoutStats.AllocationSizeMax = D3D12MA_MAX(inoutStats.AllocationSizeMax, size); -} - -static void AddDetailedStatisticsUnusedRange(DetailedStatistics& inoutStats, UINT64 size) -{ - inoutStats.UnusedRangeCount++; - inoutStats.UnusedRangeSizeMin = D3D12MA_MIN(inoutStats.UnusedRangeSizeMin, size); - inoutStats.UnusedRangeSizeMax = D3D12MA_MAX(inoutStats.UnusedRangeSizeMax, size); -} - static UINT64 HeapFlagsToAlignment(D3D12_HEAP_FLAGS flags) { /* @@ -641,9 +463,41 @@ static UINT64 HeapFlagsToAlignment(D3D12_HEAP_FLAGS flags) D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT : D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; } +static ResourceClass HeapFlagsToResourceClass(D3D12_HEAP_FLAGS heapFlags) +{ + const bool allowBuffers = (heapFlags & D3D12_HEAP_FLAG_DENY_BUFFERS) == 0; + const bool allowRtDsTextures = (heapFlags & D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES) == 0; + const bool allowNonRtDsTextures = (heapFlags & D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES) == 0; + + const uint8_t allowedGroupCount = (allowBuffers ? 1 : 0) + (allowRtDsTextures ? 1 : 0) + (allowNonRtDsTextures ? 1 : 0); + if (allowedGroupCount != 1) + return ResourceClass::Unknown; + + if (allowRtDsTextures) + return ResourceClass::RT_DS_Texture; + if (allowNonRtDsTextures) + return ResourceClass::Non_RT_DS_Texture; + return ResourceClass::Buffer; +} + +static bool IsHeapTypeStandard(D3D12_HEAP_TYPE type) +{ + return type == D3D12_HEAP_TYPE_DEFAULT || + type == D3D12_HEAP_TYPE_UPLOAD || + type == D3D12_HEAP_TYPE_READBACK; +} + +static D3D12_HEAP_PROPERTIES StandardHeapTypeToHeapProperties(D3D12_HEAP_TYPE type) +{ + D3D12MA_ASSERT(IsHeapTypeStandard(type)); + D3D12_HEAP_PROPERTIES result = {}; + result.Type = type; + return result; +} + static bool IsFormatCompressed(DXGI_FORMAT format) { - switch(format) + switch (format) { case DXGI_FORMAT_BC1_TYPELESS: case DXGI_FORMAT_BC1_UNORM: @@ -675,7 +529,7 @@ static bool IsFormatCompressed(DXGI_FORMAT format) // Only some formats are supported. For others it returns 0. static UINT GetBitsPerPixel(DXGI_FORMAT format) { - switch(format) + switch (format) { case DXGI_FORMAT_R32G32B32A32_TYPELESS: case DXGI_FORMAT_R32G32B32A32_FLOAT: @@ -787,63 +641,38 @@ static UINT GetBitsPerPixel(DXGI_FORMAT format) return 0; } } - -static const D3D12_HEAP_FLAGS RESOURCE_CLASS_HEAP_FLAGS = - D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; - -enum class ResourceClass -{ - Unknown, Buffer, Non_RT_DS_Texture, RT_DS_Texture -}; - + template -static inline ResourceClass ResourceDescToResourceClass(const D3D12_RESOURCE_DESC_T& resDesc) +static ResourceClass ResourceDescToResourceClass(const D3D12_RESOURCE_DESC_T& resDesc) { - if(resDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) + if (resDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) return ResourceClass::Buffer; // Else: it's surely a texture. const bool isRenderTargetOrDepthStencil = (resDesc.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) != 0; return isRenderTargetOrDepthStencil ? ResourceClass::RT_DS_Texture : ResourceClass::Non_RT_DS_Texture; } - -static inline ResourceClass HeapFlagsToResourceClass(D3D12_HEAP_FLAGS heapFlags) -{ - const bool allowBuffers = (heapFlags & D3D12_HEAP_FLAG_DENY_BUFFERS ) == 0; - const bool allowRtDsTextures = (heapFlags & D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES ) == 0; - const bool allowNonRtDsTextures = (heapFlags & D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES) == 0; - - const uint8_t allowedGroupCount = (allowBuffers ? 1 : 0) + (allowRtDsTextures ? 1 : 0) + (allowNonRtDsTextures ? 1 : 0); - if(allowedGroupCount != 1) - return ResourceClass::Unknown; - if(allowRtDsTextures) - return ResourceClass::RT_DS_Texture; - if(allowNonRtDsTextures) - return ResourceClass::Non_RT_DS_Texture; - return ResourceClass::Buffer; -} - // This algorithm is overly conservative. template static bool CanUseSmallAlignment(const D3D12_RESOURCE_DESC_T& resourceDesc) { - if(resourceDesc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D) + if (resourceDesc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D) return false; - if((resourceDesc.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) != 0) + if ((resourceDesc.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) != 0) return false; - if(resourceDesc.SampleDesc.Count > 1) + if (resourceDesc.SampleDesc.Count > 1) return false; - if(resourceDesc.DepthOrArraySize != 1) + if (resourceDesc.DepthOrArraySize != 1) return false; UINT sizeX = (UINT)resourceDesc.Width; UINT sizeY = resourceDesc.Height; UINT bitsPerPixel = GetBitsPerPixel(resourceDesc.Format); - if(bitsPerPixel == 0) + if (bitsPerPixel == 0) return false; - if(IsFormatCompressed(resourceDesc.Format)) + if (IsFormatCompressed(resourceDesc.Format)) { sizeX = DivideRoundingUp(sizeX, 4u); sizeY = DivideRoundingUp(sizeY, 4u); @@ -851,7 +680,7 @@ static bool CanUseSmallAlignment(const D3D12_RESOURCE_DESC_T& resourceDesc) } UINT tileSizeX = 0, tileSizeY = 0; - switch(bitsPerPixel) + switch (bitsPerPixel) { case 8: tileSizeX = 64; tileSizeY = 64; break; case 16: tileSizeX = 64; tileSizeY = 32; break; @@ -864,25 +693,186 @@ static bool CanUseSmallAlignment(const D3D12_RESOURCE_DESC_T& resourceDesc) const UINT tileCount = DivideRoundingUp(sizeX, tileSizeX) * DivideRoundingUp(sizeY, tileSizeY); return tileCount <= 16; } - -static inline bool IsHeapTypeStandard(D3D12_HEAP_TYPE type) + +static bool ValidateAllocateMemoryParameters( + const ALLOCATION_DESC* pAllocDesc, + const D3D12_RESOURCE_ALLOCATION_INFO* pAllocInfo, + Allocation** ppAllocation) { - return type == D3D12_HEAP_TYPE_DEFAULT || - type == D3D12_HEAP_TYPE_UPLOAD || - type == D3D12_HEAP_TYPE_READBACK; + return pAllocDesc && + pAllocInfo && + ppAllocation && + (pAllocInfo->Alignment == 0 || + pAllocInfo->Alignment == D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT || + pAllocInfo->Alignment == D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT) && + pAllocInfo->SizeInBytes != 0 && + pAllocInfo->SizeInBytes % (64ull * 1024) == 0; } -static inline D3D12_HEAP_PROPERTIES StandardHeapTypeToHeapProperties(D3D12_HEAP_TYPE type) +#endif // _D3D12MA_FUNCTIONS + +#ifndef _D3D12MA_STATISTICS_FUNCTIONS + +static void ClearStatistics(Statistics& outStats) { - D3D12MA_ASSERT(IsHeapTypeStandard(type)); - D3D12_HEAP_PROPERTIES result = {}; - result.Type = type; - return result; + outStats.BlockCount = 0; + outStats.AllocationCount = 0; + outStats.BlockBytes = 0; + outStats.AllocationBytes = 0; } -//////////////////////////////////////////////////////////////////////////////// -// Private class Vector +static void ClearDetailedStatistics(DetailedStatistics& outStats) +{ + ClearStatistics(outStats.Stats); + outStats.UnusedRangeCount = 0; + outStats.AllocationSizeMin = UINT64_MAX; + outStats.AllocationSizeMax = 0; + outStats.UnusedRangeSizeMin = UINT64_MAX; + outStats.UnusedRangeSizeMax = 0; +} +static void AddStatistics(Statistics& inoutStats, const Statistics& src) +{ + inoutStats.BlockCount += src.BlockCount; + inoutStats.AllocationCount += src.AllocationCount; + inoutStats.BlockBytes += src.BlockBytes; + inoutStats.AllocationBytes += src.AllocationBytes; +} + +static void AddDetailedStatistics(DetailedStatistics& inoutStats, const DetailedStatistics& src) +{ + AddStatistics(inoutStats.Stats, src.Stats); + inoutStats.UnusedRangeCount += src.UnusedRangeCount; + inoutStats.AllocationSizeMin = D3D12MA_MIN(inoutStats.AllocationSizeMin, src.AllocationSizeMin); + inoutStats.AllocationSizeMax = D3D12MA_MAX(inoutStats.AllocationSizeMax, src.AllocationSizeMax); + inoutStats.UnusedRangeSizeMin = D3D12MA_MIN(inoutStats.UnusedRangeSizeMin, src.UnusedRangeSizeMin); + inoutStats.UnusedRangeSizeMax = D3D12MA_MAX(inoutStats.UnusedRangeSizeMax, src.UnusedRangeSizeMax); +} + +static void AddDetailedStatisticsAllocation(DetailedStatistics& inoutStats, UINT64 size) +{ + inoutStats.Stats.AllocationCount++; + inoutStats.Stats.AllocationBytes += size; + inoutStats.AllocationSizeMin = D3D12MA_MIN(inoutStats.AllocationSizeMin, size); + inoutStats.AllocationSizeMax = D3D12MA_MAX(inoutStats.AllocationSizeMax, size); +} + +static void AddDetailedStatisticsUnusedRange(DetailedStatistics& inoutStats, UINT64 size) +{ + inoutStats.UnusedRangeCount++; + inoutStats.UnusedRangeSizeMin = D3D12MA_MIN(inoutStats.UnusedRangeSizeMin, size); + inoutStats.UnusedRangeSizeMax = D3D12MA_MAX(inoutStats.UnusedRangeSizeMax, size); +} + +#endif // _D3D12MA_STATISTICS_FUNCTIONS + + +#ifndef _D3D12MA_MUTEX + +#ifndef D3D12MA_MUTEX + class Mutex + { + public: + void Lock() { m_Mutex.lock(); } + void Unlock() { m_Mutex.unlock(); } + + private: + std::mutex m_Mutex; + }; + #define D3D12MA_MUTEX Mutex +#endif + +#ifndef D3D12MA_RW_MUTEX +#ifdef _WIN32 + class RWMutex + { + public: + RWMutex() { InitializeSRWLock(&m_Lock); } + void LockRead() { AcquireSRWLockShared(&m_Lock); } + void UnlockRead() { ReleaseSRWLockShared(&m_Lock); } + void LockWrite() { AcquireSRWLockExclusive(&m_Lock); } + void UnlockWrite() { ReleaseSRWLockExclusive(&m_Lock); } + + private: + SRWLOCK m_Lock; + }; +#else // #ifdef _WIN32 + class RWMutex + { + public: + RWMutex() {} + void LockRead() { m_Mutex.lock_shared(); } + void UnlockRead() { m_Mutex.unlock_shared(); } + void LockWrite() { m_Mutex.lock(); } + void UnlockWrite() { m_Mutex.unlock(); } + + private: + std::shared_timed_mutex m_Mutex; + }; +#endif // #ifdef _WIN32 + #define D3D12MA_RW_MUTEX RWMutex +#endif // #ifndef D3D12MA_RW_MUTEX + +// Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope). +struct MutexLock +{ + D3D12MA_CLASS_NO_COPY(MutexLock); +public: + MutexLock(D3D12MA_MUTEX& mutex, bool useMutex = true) : + m_pMutex(useMutex ? &mutex : NULL) + { + if (m_pMutex) m_pMutex->Lock(); + } + ~MutexLock() { if (m_pMutex) m_pMutex->Unlock(); } + +private: + D3D12MA_MUTEX* m_pMutex; +}; + +// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for reading. +struct MutexLockRead +{ + D3D12MA_CLASS_NO_COPY(MutexLockRead); +public: + MutexLockRead(D3D12MA_RW_MUTEX& mutex, bool useMutex) + : m_pMutex(useMutex ? &mutex : NULL) + { + if(m_pMutex) + { + m_pMutex->LockRead(); + } + } + ~MutexLockRead() { if (m_pMutex) m_pMutex->UnlockRead(); } + +private: + D3D12MA_RW_MUTEX* m_pMutex; +}; + +// Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for writing. +struct MutexLockWrite +{ + D3D12MA_CLASS_NO_COPY(MutexLockWrite); +public: + MutexLockWrite(D3D12MA_RW_MUTEX& mutex, bool useMutex) + : m_pMutex(useMutex ? &mutex : NULL) + { + if (m_pMutex) m_pMutex->LockWrite(); + } + ~MutexLockWrite() { if (m_pMutex) m_pMutex->UnlockWrite(); } + +private: + D3D12MA_RW_MUTEX* m_pMutex; +}; + +#if D3D12MA_DEBUG_GLOBAL_MUTEX + static D3D12MA_MUTEX g_DebugGlobalMutex; + #define D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK MutexLock debugGlobalMutexLock(g_DebugGlobalMutex, true); +#else + #define D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK +#endif +#endif // _D3D12MA_MUTEX + +#ifndef _D3D12MA_VECTOR /* Dynamically resizing continuous array. Class with interface similar to std::vector. T must be POD because constructors and destructors are not called and memcpy is @@ -893,235 +883,53 @@ class Vector { public: using value_type = T; + using iterator = T*; // allocationCallbacks externally owned, must outlive this object. - Vector(const ALLOCATION_CALLBACKS& allocationCallbacks) : - m_AllocationCallbacks(allocationCallbacks), - m_pArray(NULL), - m_Count(0), - m_Capacity(0) - { - } - - Vector(size_t count, const ALLOCATION_CALLBACKS& allocationCallbacks) : - m_AllocationCallbacks(allocationCallbacks), - m_pArray(count ? AllocateArray(allocationCallbacks, count) : NULL), - m_Count(count), - m_Capacity(count) - { - } - - Vector(const Vector& src) : - m_AllocationCallbacks(src.m_AllocationCallbacks), - m_pArray(src.m_Count ? AllocateArray(src.m_AllocationCallbacks, src.m_Count) : NULL), - m_Count(src.m_Count), - m_Capacity(src.m_Count) - { - if(m_Count > 0) - { - memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T)); - } - } - - ~Vector() - { - Free(m_AllocationCallbacks, m_pArray); - } - - Vector& operator=(const Vector& rhs) - { - if(&rhs != this) - { - resize(rhs.m_Count); - if(m_Count != 0) - { - memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T)); - } - } - return *this; - } + Vector(const ALLOCATION_CALLBACKS& allocationCallbacks); + Vector(size_t count, const ALLOCATION_CALLBACKS& allocationCallbacks); + Vector(const Vector& src); + ~Vector(); bool empty() const { return m_Count == 0; } size_t size() const { return m_Count; } T* data() { return m_pArray; } const T* data() const { return m_pArray; } - - T& operator[](size_t index) - { - D3D12MA_HEAVY_ASSERT(index < m_Count); - return m_pArray[index]; - } - const T& operator[](size_t index) const - { - D3D12MA_HEAVY_ASSERT(index < m_Count); - return m_pArray[index]; - } - - T& front() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - return m_pArray[0]; - } - const T& front() const - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - return m_pArray[0]; - } - T& back() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - return m_pArray[m_Count - 1]; - } - const T& back() const - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - return m_pArray[m_Count - 1]; - } - - void reserve(size_t newCapacity, bool freeMemory = false) - { - newCapacity = D3D12MA_MAX(newCapacity, m_Count); - - if((newCapacity < m_Capacity) && !freeMemory) - { - newCapacity = m_Capacity; - } - - if(newCapacity != m_Capacity) - { - T* const newArray = newCapacity ? AllocateArray(m_AllocationCallbacks, newCapacity) : NULL; - if(m_Count != 0) - { - memcpy(newArray, m_pArray, m_Count * sizeof(T)); - } - Free(m_AllocationCallbacks, m_pArray); - m_Capacity = newCapacity; - m_pArray = newArray; - } - } - - void resize(size_t newCount, bool freeMemory = false) - { - size_t newCapacity = m_Capacity; - if(newCount > m_Capacity) - { - newCapacity = D3D12MA_MAX(newCount, D3D12MA_MAX(m_Capacity * 3 / 2, (size_t)8)); - } - else if(freeMemory) - { - newCapacity = newCount; - } - - if(newCapacity != m_Capacity) - { - T* const newArray = newCapacity ? AllocateArray(m_AllocationCallbacks, newCapacity) : NULL; - const size_t elementsToCopy = D3D12MA_MIN(m_Count, newCount); - if(elementsToCopy != 0) - { - memcpy(newArray, m_pArray, elementsToCopy * sizeof(T)); - } - Free(m_AllocationCallbacks, m_pArray); - m_Capacity = newCapacity; - m_pArray = newArray; - } - - m_Count = newCount; - } - - void clear(bool freeMemory = false) - { - resize(0, freeMemory); - } - - void insert(size_t index, const T& src) - { - D3D12MA_HEAVY_ASSERT(index <= m_Count); - const size_t oldCount = size(); - resize(oldCount + 1); - if(index < oldCount) - { - memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T)); - } - m_pArray[index] = src; - } - - void remove(size_t index) - { - D3D12MA_HEAVY_ASSERT(index < m_Count); - const size_t oldCount = size(); - if(index < oldCount - 1) - { - memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T)); - } - resize(oldCount - 1); - } - - void push_back(const T& src) - { - const size_t newIndex = size(); - resize(newIndex + 1); - m_pArray[newIndex] = src; - } - - void pop_back() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - resize(size() - 1); - } - - void push_front(const T& src) - { - insert(0, src); - } - - void pop_front() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - remove(0); - } - - using iterator = T*; + void clear(bool freeMemory = false) { resize(0, freeMemory); } iterator begin() { return m_pArray; } - const iterator cbegin() const { return m_pArray; } - - iterator rbegin() { return end() - 1; } - const iterator crbegin() const { return cend() - 1; } - iterator end() { return m_pArray + m_Count; } - const iterator cend() const { return m_pArray + m_Count; } - iterator rend() { return begin() - 1; } + iterator rbegin() { return end() - 1; } + + const iterator cbegin() const { return m_pArray; } + const iterator cend() const { return m_pArray + m_Count; } + const iterator crbegin() const { return cend() - 1; } const iterator crend() const { return cbegin() - 1; } - template - size_t InsertSorted(const T& value, const CmpLess& cmp) - { - const size_t indexToInsert = BinaryFindFirstNotLess( - m_pArray, - m_pArray + m_Count, - value, - cmp) - m_pArray; - insert(indexToInsert, value); - return indexToInsert; - } + void push_front(const T& src) { insert(0, src); } + void push_back(const T& src); + void pop_front(); + void pop_back(); + + T& front(); + T& back(); + const T& front() const; + const T& back() const; + + void reserve(size_t newCapacity, bool freeMemory = false); + void resize(size_t newCount, bool freeMemory = false); + void insert(size_t index, const T& src); + void remove(size_t index); template - bool RemoveSorted(const T& value, const CmpLess& cmp) - { - const iterator it = BinaryFindFirstNotLess( - m_pArray, - m_pArray + m_Count, - value, - cmp); - if((it != end()) && !cmp(*it, value) && !cmp(value, *it)) - { - size_t indexToRemove = it - begin(); - remove(indexToRemove); - return true; - } - return false; - } + size_t InsertSorted(const T& value, const CmpLess& cmp); + template + bool RemoveSorted(const T& value, const CmpLess& cmp); + + Vector& operator=(const Vector& rhs); + T& operator[](size_t index); + const T& operator[](size_t index) const; private: const ALLOCATION_CALLBACKS& m_AllocationCallbacks; @@ -1130,13 +938,231 @@ private: size_t m_Capacity; }; -//////////////////////////////////////////////////////////////////////////////// -// Private class StringBuilder +#ifndef _D3D12MA_VECTOR_FUNCTIONS +template +Vector::Vector(const ALLOCATION_CALLBACKS& allocationCallbacks) + : m_AllocationCallbacks(allocationCallbacks), + m_pArray(NULL), + m_Count(0), + m_Capacity(0) {} +template +Vector::Vector(size_t count, const ALLOCATION_CALLBACKS& allocationCallbacks) + : m_AllocationCallbacks(allocationCallbacks), + m_pArray(count ? AllocateArray(allocationCallbacks, count) : NULL), + m_Count(count), + m_Capacity(count) {} + +template +Vector::Vector(const Vector& src) + : m_AllocationCallbacks(src.m_AllocationCallbacks), + m_pArray(src.m_Count ? AllocateArray(src.m_AllocationCallbacks, src.m_Count) : NULL), + m_Count(src.m_Count), + m_Capacity(src.m_Count) +{ + if (m_Count > 0) + { + memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T)); + } +} + +template +Vector::~Vector() +{ + Free(m_AllocationCallbacks, m_pArray); +} + +template +void Vector::push_back(const T& src) +{ + const size_t newIndex = size(); + resize(newIndex + 1); + m_pArray[newIndex] = src; +} + +template +void Vector::pop_front() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + remove(0); +} + +template +void Vector::pop_back() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + resize(size() - 1); +} + +template +T& Vector::front() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + return m_pArray[0]; +} + +template +T& Vector::back() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + return m_pArray[m_Count - 1]; +} + +template +const T& Vector::front() const +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + return m_pArray[0]; +} + +template +const T& Vector::back() const +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + return m_pArray[m_Count - 1]; +} + +template +void Vector::reserve(size_t newCapacity, bool freeMemory) +{ + newCapacity = D3D12MA_MAX(newCapacity, m_Count); + + if ((newCapacity < m_Capacity) && !freeMemory) + { + newCapacity = m_Capacity; + } + + if (newCapacity != m_Capacity) + { + T* const newArray = newCapacity ? AllocateArray(m_AllocationCallbacks, newCapacity) : NULL; + if (m_Count != 0) + { + memcpy(newArray, m_pArray, m_Count * sizeof(T)); + } + Free(m_AllocationCallbacks, m_pArray); + m_Capacity = newCapacity; + m_pArray = newArray; + } +} + +template +void Vector::resize(size_t newCount, bool freeMemory) +{ + size_t newCapacity = m_Capacity; + if (newCount > m_Capacity) + { + newCapacity = D3D12MA_MAX(newCount, D3D12MA_MAX(m_Capacity * 3 / 2, (size_t)8)); + } + else if (freeMemory) + { + newCapacity = newCount; + } + + if (newCapacity != m_Capacity) + { + T* const newArray = newCapacity ? AllocateArray(m_AllocationCallbacks, newCapacity) : NULL; + const size_t elementsToCopy = D3D12MA_MIN(m_Count, newCount); + if (elementsToCopy != 0) + { + memcpy(newArray, m_pArray, elementsToCopy * sizeof(T)); + } + Free(m_AllocationCallbacks, m_pArray); + m_Capacity = newCapacity; + m_pArray = newArray; + } + + m_Count = newCount; +} + +template +void Vector::insert(size_t index, const T& src) +{ + D3D12MA_HEAVY_ASSERT(index <= m_Count); + const size_t oldCount = size(); + resize(oldCount + 1); + if (index < oldCount) + { + memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T)); + } + m_pArray[index] = src; +} + +template +void Vector::remove(size_t index) +{ + D3D12MA_HEAVY_ASSERT(index < m_Count); + const size_t oldCount = size(); + if (index < oldCount - 1) + { + memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T)); + } + resize(oldCount - 1); +} + +template template +size_t Vector::InsertSorted(const T& value, const CmpLess& cmp) +{ + const size_t indexToInsert = BinaryFindFirstNotLess( + m_pArray, + m_pArray + m_Count, + value, + cmp) - m_pArray; + insert(indexToInsert, value); + return indexToInsert; +} + +template template +bool Vector::RemoveSorted(const T& value, const CmpLess& cmp) +{ + const iterator it = BinaryFindFirstNotLess( + m_pArray, + m_pArray + m_Count, + value, + cmp); + if ((it != end()) && !cmp(*it, value) && !cmp(value, *it)) + { + size_t indexToRemove = it - begin(); + remove(indexToRemove); + return true; + } + return false; +} + +template +Vector& Vector::operator=(const Vector& rhs) +{ + if (&rhs != this) + { + resize(rhs.m_Count); + if (m_Count != 0) + { + memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T)); + } + } + return *this; +} + +template +T& Vector::operator[](size_t index) +{ + D3D12MA_HEAVY_ASSERT(index < m_Count); + return m_pArray[index]; +} + +template +const T& Vector::operator[](size_t index) const +{ + D3D12MA_HEAVY_ASSERT(index < m_Count); + return m_pArray[index]; +} +#endif // _D3D12MA_VECTOR_FUNCTIONS +#endif // _D3D12MA_VECTOR + +#ifndef _D3D12MA_STRING_BUILDER class StringBuilder { public: - StringBuilder(const ALLOCATION_CALLBACKS& allocationCallbacks) : m_Data(allocationCallbacks) { } + StringBuilder(const ALLOCATION_CALLBACKS& allocationCallbacks) : m_Data(allocationCallbacks) {} size_t GetLength() const { return m_Data.size(); } LPCWSTR GetData() const { return m_Data.data(); } @@ -1151,6 +1177,7 @@ private: Vector m_Data; }; +#ifndef _D3D12MA_STRING_BUILDER_FUNCTIONS void StringBuilder::Add(LPCWSTR str) { const size_t len = wcslen(str); @@ -1189,10 +1216,10 @@ void StringBuilder::AddNumber(UINT64 num) while (num); Add(p); } +#endif // _D3D12MA_STRING_BUILDER_FUNCTIONS +#endif // _D3D12MA_STRING_BUILDER -//////////////////////////////////////////////////////////////////////////////// -// Private class JsonWriter - +#ifndef _D3D12MA_JSON_WRITER /* Allows to conveniently build a correct JSON document to be written to the StringBuilder passed to the constructor. @@ -1238,8 +1265,6 @@ public: // Ends writing a string value by writing '"'. void EndString(LPCWSTR pStr = NULL); - void AddAllocationToObject(const Allocation& alloc); - // Writes a number value. void WriteNumber(UINT num); void WriteNumber(UINT64 num); @@ -1248,6 +1273,9 @@ public: // Writes a null value. void WriteNull(); + void AddAllocationToObject(const Allocation& alloc); + void AddDetailedStatisticsInfoObject(const DetailedStatistics& stats); + private: static const WCHAR* const INDENT; @@ -1271,14 +1299,13 @@ private: void WriteIndent(bool oneLess = false); }; +#ifndef _D3D12MA_JSON_WRITER_FUNCTIONS const WCHAR* const JsonWriter::INDENT = L" "; -JsonWriter::JsonWriter(const ALLOCATION_CALLBACKS& allocationCallbacks, StringBuilder& stringBuilder) : - m_SB(stringBuilder), +JsonWriter::JsonWriter(const ALLOCATION_CALLBACKS& allocationCallbacks, StringBuilder& stringBuilder) + : m_SB(stringBuilder), m_Stack(allocationCallbacks), - m_InsideString(false) -{ -} + m_InsideString(false) {} JsonWriter::~JsonWriter() { @@ -1458,6 +1485,85 @@ void JsonWriter::WriteNull() m_SB.Add(L"null"); } +void JsonWriter::AddAllocationToObject(const Allocation& alloc) +{ + WriteString(L"Type"); + switch (alloc.m_PackedData.GetResourceDimension()) { + case D3D12_RESOURCE_DIMENSION_UNKNOWN: + WriteString(L"UNKNOWN"); + break; + case D3D12_RESOURCE_DIMENSION_BUFFER: + WriteString(L"BUFFER"); + break; + case D3D12_RESOURCE_DIMENSION_TEXTURE1D: + WriteString(L"TEXTURE1D"); + break; + case D3D12_RESOURCE_DIMENSION_TEXTURE2D: + WriteString(L"TEXTURE2D"); + break; + case D3D12_RESOURCE_DIMENSION_TEXTURE3D: + WriteString(L"TEXTURE3D"); + break; + default: D3D12MA_ASSERT(0); break; + } + WriteString(L"Size"); + WriteNumber(alloc.GetSize()); + LPCWSTR name = alloc.GetName(); + if (name != NULL) + { + WriteString(L"Name"); + WriteString(name); + } + if (alloc.m_PackedData.GetResourceFlags()) + { + WriteString(L"Flags"); + WriteNumber((UINT)alloc.m_PackedData.GetResourceFlags()); + } + if (alloc.m_PackedData.GetTextureLayout()) + { + WriteString(L"Layout"); + WriteNumber((UINT)alloc.m_PackedData.GetTextureLayout()); + } + if (alloc.m_CreationFrameIndex) + { + WriteString(L"CreationFrameIndex"); + WriteNumber(alloc.m_CreationFrameIndex); + } +} + +void JsonWriter::AddDetailedStatisticsInfoObject(const DetailedStatistics& stats) +{ + BeginObject(); + WriteString(L"BlockCount"); + WriteNumber(stats.Stats.BlockCount); + WriteString(L"AllocationCount"); + WriteNumber(stats.Stats.AllocationCount); + WriteString(L"UnusedRangeCount"); + WriteNumber(stats.UnusedRangeCount); + WriteString(L"BlockBytes"); + WriteNumber(stats.Stats.BlockBytes); + WriteString(L"AllocationBytes"); + WriteNumber(stats.Stats.AllocationBytes); + + WriteString(L"AllocationSize"); + BeginObject(true); + WriteString(L"Min"); + WriteNumber(stats.AllocationSizeMin); + WriteString(L"Max"); + WriteNumber(stats.AllocationSizeMax); + EndObject(); + + WriteString(L"UnusedRangeSize"); + BeginObject(true); + WriteString(L"Min"); + WriteNumber(stats.UnusedRangeSizeMin); + WriteString(L"Max"); + WriteNumber(stats.UnusedRangeSizeMax); + EndObject(); + + EndObject(); +} + void JsonWriter::BeginValue(bool isString) { if (!m_Stack.empty()) @@ -1502,56 +1608,10 @@ void JsonWriter::WriteIndent(bool oneLess) } } } +#endif // _D3D12MA_JSON_WRITER_FUNCTIONS +#endif // _D3D12MA_JSON_WRITER -void JsonWriter::AddAllocationToObject(const Allocation& alloc) -{ - WriteString(L"Type"); - switch (alloc.m_PackedData.GetResourceDimension()) { - case D3D12_RESOURCE_DIMENSION_UNKNOWN: - WriteString(L"UNKNOWN"); - break; - case D3D12_RESOURCE_DIMENSION_BUFFER: - WriteString(L"BUFFER"); - break; - case D3D12_RESOURCE_DIMENSION_TEXTURE1D: - WriteString(L"TEXTURE1D"); - break; - case D3D12_RESOURCE_DIMENSION_TEXTURE2D: - WriteString(L"TEXTURE2D"); - break; - case D3D12_RESOURCE_DIMENSION_TEXTURE3D: - WriteString(L"TEXTURE3D"); - break; - default: D3D12MA_ASSERT(0); break; - } - WriteString(L"Size"); - WriteNumber(alloc.GetSize()); - LPCWSTR name = alloc.GetName(); - if(name != NULL) - { - WriteString(L"Name"); - WriteString(name); - } - if(alloc.m_PackedData.GetResourceFlags()) - { - WriteString(L"Flags"); - WriteNumber((UINT)alloc.m_PackedData.GetResourceFlags()); - } - if(alloc.m_PackedData.GetTextureLayout()) - { - WriteString(L"Layout"); - WriteNumber((UINT)alloc.m_PackedData.GetTextureLayout()); - } - if(alloc.m_CreationFrameIndex) - { - WriteString(L"CreationFrameIndex"); - WriteNumber(alloc.m_CreationFrameIndex); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Private class PoolAllocator - +#ifndef _D3D12MA_POOL_ALLOCATOR /* Allocator for objects of type T using a list of arrays (pools) to speed up allocation. Number of elements that can be allocated is not bounded because @@ -1567,8 +1627,10 @@ public: // allocationCallbacks externally owned, must outlive this object. PoolAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks, UINT firstBlockCapacity); ~PoolAllocator() { Clear(); } + void Clear(); - template T* Alloc(Types... args); + template + T* Alloc(Types... args); void Free(T* ptr); private: @@ -1592,9 +1654,10 @@ private: ItemBlock& CreateNewBlock(); }; +#ifndef _D3D12MA_POOL_ALLOCATOR_FUNCTIONS template -PoolAllocator::PoolAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks, UINT firstBlockCapacity) : - m_AllocationCallbacks(allocationCallbacks), +PoolAllocator::PoolAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks, UINT firstBlockCapacity) + : m_AllocationCallbacks(allocationCallbacks), m_FirstBlockCapacity(firstBlockCapacity), m_ItemBlocks(allocationCallbacks) { @@ -1611,8 +1674,8 @@ void PoolAllocator::Clear() m_ItemBlocks.clear(true); } -template -template T* PoolAllocator::Alloc(Types... args) +template template +T* PoolAllocator::Alloc(Types... args) { for(size_t i = m_ItemBlocks.size(); i--; ) { @@ -1682,10 +1745,10 @@ typename PoolAllocator::ItemBlock& PoolAllocator::CreateNewBlock() newBlock.pItems[newBlockCapacity - 1].NextFreeIndex = UINT32_MAX; return m_ItemBlocks.back(); } +#endif // _D3D12MA_POOL_ALLOCATOR_FUNCTIONS +#endif // _D3D12MA_POOL_ALLOCATOR -//////////////////////////////////////////////////////////////////////////////// -// Private class List - +#ifndef _D3D12MA_LIST /* Doubly linked list, with elements allocated out of PoolAllocator. Has custom interface, as well as STL-style interface, including iterator and @@ -1703,15 +1766,137 @@ public: T Value; }; + class reverse_iterator; + class const_reverse_iterator; + class iterator + { + friend class List; + friend class const_iterator; + + public: + iterator() = default; + iterator(const reverse_iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + + T& operator*() const; + T* operator->() const; + + iterator& operator++(); + iterator& operator--(); + iterator operator++(int); + iterator operator--(int); + + bool operator==(const iterator& rhs) const; + bool operator!=(const iterator& rhs) const; + + private: + List* m_pList = NULL; + Item* m_pItem = NULL; + + iterator(List* pList, Item* pItem) : m_pList(pList), m_pItem(pItem) {} + }; + + class reverse_iterator + { + friend class List; + friend class const_reverse_iterator; + + public: + reverse_iterator() = default; + reverse_iterator(const iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + + T& operator*() const; + T* operator->() const; + + reverse_iterator& operator++(); + reverse_iterator& operator--(); + reverse_iterator operator++(int); + reverse_iterator operator--(int); + + bool operator==(const reverse_iterator& rhs) const; + bool operator!=(const reverse_iterator& rhs) const; + + private: + List* m_pList = NULL; + Item* m_pItem = NULL; + + reverse_iterator(List* pList, Item* pItem) + : m_pList(pList), m_pItem(pItem) {} + }; + + class const_iterator + { + friend class List; + + public: + const_iterator() = default; + const_iterator(const iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + const_iterator(const reverse_iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + const_iterator(const const_reverse_iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + + iterator dropConst() const; + const T& operator*() const; + const T* operator->() const; + + const_iterator& operator++(); + const_iterator& operator--(); + const_iterator operator++(int); + const_iterator operator--(int); + + bool operator==(const const_iterator& rhs) const; + bool operator!=(const const_iterator& rhs) const; + + private: + const List* m_pList = NULL; + const Item* m_pItem = NULL; + + const_iterator(const List* pList, const Item* pItem) + : m_pList(pList), m_pItem(pItem) {} + }; + + class const_reverse_iterator + { + friend class List; + + public: + const_reverse_iterator() = default; + const_reverse_iterator(const iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + const_reverse_iterator(const reverse_iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + const_reverse_iterator(const const_iterator& src) + : m_pList(src.m_pList), m_pItem(src.m_pItem) {} + + reverse_iterator dropConst() const; + const T& operator*() const; + const T* operator->() const; + + const_reverse_iterator& operator++(); + const_reverse_iterator& operator--(); + const_reverse_iterator operator++(int); + const_reverse_iterator operator--(int); + + bool operator==(const const_reverse_iterator& rhs) const; + bool operator!=(const const_reverse_iterator& rhs) const; + + private: + const List* m_pList = NULL; + const Item* m_pItem = NULL; + + const_reverse_iterator(const List* pList, const Item* pItem) + : m_pList(pList), m_pItem(pItem) {} + }; + // allocationCallbacks externally owned, must outlive this object. List(const ALLOCATION_CALLBACKS& allocationCallbacks); - // Intentionally not calling Clear, because that would be unnecessary // computations to return all items to m_ItemAllocator as free. - // ~List() {} + ~List() = default; - void Clear(); - size_t GetCount() const { return m_Count; } bool IsEmpty() const { return m_Count == 0; } @@ -1720,6 +1905,28 @@ public: Item* Back() { return m_pBack; } const Item* Back() const { return m_pBack; } + bool empty() const { return IsEmpty(); } + size_t size() const { return GetCount(); } + void push_back(const T& value) { PushBack(value); } + iterator insert(iterator it, const T& value) { return iterator(this, InsertBefore(it.m_pItem, value)); } + void clear() { Clear(); } + void erase(iterator it) { Remove(it.m_pItem); } + + iterator begin() { return iterator(this, Front()); } + iterator end() { return iterator(this, NULL); } + reverse_iterator rbegin() { return reverse_iterator(this, Back()); } + reverse_iterator rend() { return reverse_iterator(this, NULL); } + + const_iterator cbegin() const { return const_iterator(this, Front()); } + const_iterator cend() const { return const_iterator(this, NULL); } + const_iterator begin() const { return cbegin(); } + const_iterator end() const { return cend(); } + + const_reverse_iterator crbegin() const { return const_reverse_iterator(this, Back()); } + const_reverse_iterator crend() const { return const_reverse_iterator(this, NULL); } + const_reverse_iterator rbegin() const { return crbegin(); } + const_reverse_iterator rend() const { return crend(); } + Item* PushBack(); Item* PushFront(); Item* PushBack(const T& value); @@ -1731,408 +1938,12 @@ public: Item* InsertBefore(Item* pItem); // Item can be null - it means PushFront. Item* InsertAfter(Item* pItem); - Item* InsertBefore(Item* pItem, const T& value); Item* InsertAfter(Item* pItem, const T& value); + void Clear(); void Remove(Item* pItem); - class reverse_iterator; - class const_reverse_iterator; - class iterator - { - public: - iterator() : - m_pList(NULL), - m_pItem(NULL) - { - } - - iterator(const reverse_iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - T& operator*() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return m_pItem->Value; - } - T* operator->() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return &m_pItem->Value; - } - - iterator& operator++() - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - m_pItem = m_pItem->pNext; - return *this; - } - iterator& operator--() - { - if(m_pItem != NULL) - { - m_pItem = m_pItem->pPrev; - } - else - { - D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); - m_pItem = m_pList->Back(); - } - return *this; - } - - iterator operator++(int) - { - iterator result = *this; - ++*this; - return result; - } - iterator operator--(int) - { - iterator result = *this; - --*this; - return result; - } - - bool operator==(const iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem == rhs.m_pItem; - } - bool operator!=(const iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem != rhs.m_pItem; - } - - private: - List* m_pList; - Item* m_pItem; - - iterator(List* pList, Item* pItem) : - m_pList(pList), - m_pItem(pItem) - { - } - - friend class List; - friend class const_iterator; - }; - - class reverse_iterator - { - public: - reverse_iterator() : - m_pList(NULL), - m_pItem(NULL) - { - } - - reverse_iterator(const iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - T& operator*() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return m_pItem->Value; - } - T* operator->() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return &m_pItem->Value; - } - - reverse_iterator& operator++() - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - m_pItem = m_pItem->pPrev; - return *this; - } - reverse_iterator& operator--() - { - if(m_pItem != NULL) - { - m_pItem = m_pItem->pNext; - } - else - { - D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); - m_pItem = m_pList->Front(); - } - return *this; - } - - reverse_iterator operator++(int) - { - reverse_iterator result = *this; - ++*this; - return result; - } - reverse_iterator operator--(int) - { - reverse_iterator result = *this; - --*this; - return result; - } - - bool operator==(const reverse_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem == rhs.m_pItem; - } - bool operator!=(const reverse_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem != rhs.m_pItem; - } - - private: - List* m_pList; - Item* m_pItem; - - reverse_iterator(List* pList, Item* pItem) : - m_pList(pList), - m_pItem(pItem) - { - } - - friend class List; - friend class const_reverse_iterator; - }; - - class const_iterator - { - public: - const_iterator() : - m_pList(NULL), - m_pItem(NULL) - { - } - - const_iterator(const iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - const_iterator(const reverse_iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - const_iterator(const const_reverse_iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - iterator dropConst() const - { - return iterator(const_cast*>(m_pList), const_cast(m_pItem)); - } - - const T& operator*() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return m_pItem->Value; - } - const T* operator->() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return &m_pItem->Value; - } - - const_iterator& operator++() - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - m_pItem = m_pItem->pNext; - return *this; - } - const_iterator& operator--() - { - if(m_pItem != NULL) - { - m_pItem = m_pItem->pPrev; - } - else - { - D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); - m_pItem = m_pList->Back(); - } - return *this; - } - - const_iterator operator++(int) - { - const_iterator result = *this; - ++*this; - return result; - } - const_iterator operator--(int) - { - const_iterator result = *this; - --*this; - return result; - } - - bool operator==(const const_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem == rhs.m_pItem; - } - bool operator!=(const const_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem != rhs.m_pItem; - } - - private: - const_iterator(const List* pList, const Item* pItem) : - m_pList(pList), - m_pItem(pItem) - { - } - - const List* m_pList; - const Item* m_pItem; - - friend class List; - }; - - class const_reverse_iterator - { - public: - const_reverse_iterator() : - m_pList(NULL), - m_pItem(NULL) - { - } - - const_reverse_iterator(const iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - const_reverse_iterator(const reverse_iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - const_reverse_iterator(const const_iterator& src) : - m_pList(src.m_pList), - m_pItem(src.m_pItem) - { - } - - reverse_iterator dropConst() const - { - return reverse_iterator(const_cast*>(m_pList), const_cast(m_pItem)); - } - - const T& operator*() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return m_pItem->Value; - } - const T* operator->() const - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - return &m_pItem->Value; - } - - const_reverse_iterator& operator++() - { - D3D12MA_HEAVY_ASSERT(m_pItem != NULL); - m_pItem = m_pItem->pPrev; - return *this; - } - const_reverse_iterator& operator--() - { - if(m_pItem != NULL) - { - m_pItem = m_pItem->pNext; - } - else - { - D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); - m_pItem = m_pList->Front(); - } - return *this; - } - - const_reverse_iterator operator++(int) - { - const_reverse_iterator result = *this; - ++*this; - return result; - } - const_reverse_iterator operator--(int) - { - const_reverse_iterator result = *this; - --*this; - return result; - } - - bool operator==(const const_reverse_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem == rhs.m_pItem; - } - bool operator!=(const const_reverse_iterator& rhs) const - { - D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); - return m_pItem != rhs.m_pItem; - } - - private: - const_reverse_iterator(const List* pList, const Item* pItem) : - m_pList(pList), - m_pItem(pItem) - { - } - - const List* m_pList; - const Item* m_pItem; - - friend class List; - }; - - bool empty() const { return IsEmpty(); } - size_t size() const { return GetCount(); } - - iterator begin() { return iterator(this, Front()); } - iterator end() { return iterator(this, NULL); } - - const_iterator cbegin() const { return const_iterator(this, Front()); } - const_iterator cend() const { return const_iterator(this, NULL); } - - const_iterator begin() const { return cbegin(); } - const_iterator end() const { return cend(); } - - reverse_iterator rbegin() { return reverse_iterator(this, Back()); } - reverse_iterator rend() { return reverse_iterator(this, NULL); } - - const_reverse_iterator crbegin() const { return const_reverse_iterator(this, Back()); } - const_reverse_iterator crend() const { return const_reverse_iterator(this, NULL); } - - const_reverse_iterator rbegin() const { return crbegin(); } - const_reverse_iterator rend() const { return crend(); } - - void clear() { Clear(); } - void push_back(const T& value) { PushBack(value); } - void erase(iterator it) { Remove(it.m_pItem); } - iterator insert(iterator it, const T& value) { return iterator(this, InsertBefore(it.m_pItem, value)); } - private: const ALLOCATION_CALLBACKS& m_AllocationCallbacks; PoolAllocator m_ItemAllocator; @@ -2141,15 +1952,302 @@ private: size_t m_Count; }; +#ifndef _D3D12MA_LIST_ITERATOR_FUNCTIONS template -List::List(const ALLOCATION_CALLBACKS& allocationCallbacks) : - m_AllocationCallbacks(allocationCallbacks), +T& List::iterator::operator*() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return m_pItem->Value; +} + +template +T* List::iterator::operator->() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return &m_pItem->Value; +} + +template +typename List::iterator& List::iterator::operator++() +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + m_pItem = m_pItem->pNext; + return *this; +} + +template +typename List::iterator& List::iterator::operator--() +{ + if (m_pItem != NULL) + { + m_pItem = m_pItem->pPrev; + } + else + { + D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); + m_pItem = m_pList->Back(); + } + return *this; +} + +template +typename List::iterator List::iterator::operator++(int) +{ + iterator result = *this; + ++* this; + return result; +} + +template +typename List::iterator List::iterator::operator--(int) +{ + iterator result = *this; + --* this; + return result; +} + +template +bool List::iterator::operator==(const iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem == rhs.m_pItem; +} + +template +bool List::iterator::operator!=(const iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem != rhs.m_pItem; +} +#endif // _D3D12MA_LIST_ITERATOR_FUNCTIONS + +#ifndef _D3D12MA_LIST_REVERSE_ITERATOR_FUNCTIONS +template +T& List::reverse_iterator::operator*() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return m_pItem->Value; +} + +template +T* List::reverse_iterator::operator->() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return &m_pItem->Value; +} + +template +typename List::reverse_iterator& List::reverse_iterator::operator++() +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + m_pItem = m_pItem->pPrev; + return *this; +} + +template +typename List::reverse_iterator& List::reverse_iterator::operator--() +{ + if (m_pItem != NULL) + { + m_pItem = m_pItem->pNext; + } + else + { + D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); + m_pItem = m_pList->Front(); + } + return *this; +} + +template +typename List::reverse_iterator List::reverse_iterator::operator++(int) +{ + reverse_iterator result = *this; + ++* this; + return result; +} + +template +typename List::reverse_iterator List::reverse_iterator::operator--(int) +{ + reverse_iterator result = *this; + --* this; + return result; +} + +template +bool List::reverse_iterator::operator==(const reverse_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem == rhs.m_pItem; +} + +template +bool List::reverse_iterator::operator!=(const reverse_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem != rhs.m_pItem; +} +#endif // _D3D12MA_LIST_REVERSE_ITERATOR_FUNCTIONS + +#ifndef _D3D12MA_LIST_CONST_ITERATOR_FUNCTIONS +template +typename List::iterator List::const_iterator::dropConst() const +{ + return iterator(const_cast*>(m_pList), const_cast(m_pItem)); +} + +template +const T& List::const_iterator::operator*() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return m_pItem->Value; +} + +template +const T* List::const_iterator::operator->() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return &m_pItem->Value; +} + +template +typename List::const_iterator& List::const_iterator::operator++() +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + m_pItem = m_pItem->pNext; + return *this; +} + +template +typename List::const_iterator& List::const_iterator::operator--() +{ + if (m_pItem != NULL) + { + m_pItem = m_pItem->pPrev; + } + else + { + D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); + m_pItem = m_pList->Back(); + } + return *this; +} + +template +typename List::const_iterator List::const_iterator::operator++(int) +{ + const_iterator result = *this; + ++* this; + return result; +} + +template +typename List::const_iterator List::const_iterator::operator--(int) +{ + const_iterator result = *this; + --* this; + return result; +} + +template +bool List::const_iterator::operator==(const const_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem == rhs.m_pItem; +} + +template +bool List::const_iterator::operator!=(const const_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem != rhs.m_pItem; +} +#endif // _D3D12MA_LIST_CONST_ITERATOR_FUNCTIONS + +#ifndef _D3D12MA_LIST_CONST_REVERSE_ITERATOR_FUNCTIONS +template +typename List::reverse_iterator List::const_reverse_iterator::dropConst() const +{ + return reverse_iterator(const_cast*>(m_pList), const_cast(m_pItem)); +} + +template +const T& List::const_reverse_iterator::operator*() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return m_pItem->Value; +} + +template +const T* List::const_reverse_iterator::operator->() const +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + return &m_pItem->Value; +} + +template +typename List::const_reverse_iterator& List::const_reverse_iterator::operator++() +{ + D3D12MA_HEAVY_ASSERT(m_pItem != NULL); + m_pItem = m_pItem->pPrev; + return *this; +} + +template +typename List::const_reverse_iterator& List::const_reverse_iterator::operator--() +{ + if (m_pItem != NULL) + { + m_pItem = m_pItem->pNext; + } + else + { + D3D12MA_HEAVY_ASSERT(!m_pList->IsEmpty()); + m_pItem = m_pList->Front(); + } + return *this; +} + +template +typename List::const_reverse_iterator List::const_reverse_iterator::operator++(int) +{ + const_reverse_iterator result = *this; + ++* this; + return result; +} + +template +typename List::const_reverse_iterator List::const_reverse_iterator::operator--(int) +{ + const_reverse_iterator result = *this; + --* this; + return result; +} + +template +bool List::const_reverse_iterator::operator==(const const_reverse_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem == rhs.m_pItem; +} + +template +bool List::const_reverse_iterator::operator!=(const const_reverse_iterator& rhs) const +{ + D3D12MA_HEAVY_ASSERT(m_pList == rhs.m_pList); + return m_pItem != rhs.m_pItem; +} +#endif // _D3D12MA_LIST_CONST_REVERSE_ITERATOR_FUNCTIONS + +#ifndef _D3D12MA_LIST_FUNCTIONS +template +List::List(const ALLOCATION_CALLBACKS& allocationCallbacks) + : m_AllocationCallbacks(allocationCallbacks), m_ItemAllocator(allocationCallbacks, 128), m_pFront(NULL), m_pBack(NULL), - m_Count(0) -{ -} + m_Count(0) {} template void List::Clear() @@ -2358,10 +2456,10 @@ typename List::Item* List::InsertAfter(Item* pItem, const T& value) newItem->Value = value; return newItem; } +#endif // _D3D12MA_LIST_FUNCTIONS +#endif // _D3D12MA_LIST -//////////////////////////////////////////////////////////////////////////////// -// Private class IntrusiveLinkedList - +#ifndef _D3D12MA_INTRUSIVE_LINKED_LIST /* Expected interface of ItemTypeTraits: struct MyItemTypeTraits @@ -2380,204 +2478,238 @@ public: using ItemType = typename ItemTypeTraits::ItemType; static ItemType* GetPrev(const ItemType* item) { return ItemTypeTraits::GetPrev(item); } static ItemType* GetNext(const ItemType* item) { return ItemTypeTraits::GetNext(item); } + // Movable, not copyable. - IntrusiveLinkedList() { } - IntrusiveLinkedList(const IntrusiveLinkedList& src) = delete; - IntrusiveLinkedList(IntrusiveLinkedList&& src) : - m_Front(src.m_Front), m_Back(src.m_Back), m_Count(src.m_Count) - { - src.m_Front = src.m_Back = NULL; - src.m_Count = 0; - } - ~IntrusiveLinkedList() - { - D3D12MA_HEAVY_ASSERT(IsEmpty()); - } - IntrusiveLinkedList& operator=(const IntrusiveLinkedList& src) = delete; - IntrusiveLinkedList& operator=(IntrusiveLinkedList&& src) - { - if(&src != this) - { - D3D12MA_HEAVY_ASSERT(IsEmpty()); - m_Front = src.m_Front; - m_Back = src.m_Back; - m_Count = src.m_Count; - src.m_Front = src.m_Back = NULL; - src.m_Count = 0; - } - return *this; - } - void RemoveAll() - { - if(!IsEmpty()) - { - ItemType* item = m_Back; - while(item != NULL) - { - ItemType* const prevItem = ItemTypeTraits::AccessPrev(item); - ItemTypeTraits::AccessPrev(item) = NULL; - ItemTypeTraits::AccessNext(item) = NULL; - item = prevItem; - } - m_Front = NULL; - m_Back = NULL; - m_Count = 0; - } - } + IntrusiveLinkedList() = default; + IntrusiveLinkedList(const IntrusiveLinkedList&) = delete; + IntrusiveLinkedList(IntrusiveLinkedList&& src); + IntrusiveLinkedList& operator=(const IntrusiveLinkedList&) = delete; + IntrusiveLinkedList& operator=(IntrusiveLinkedList&& src); + ~IntrusiveLinkedList() { D3D12MA_HEAVY_ASSERT(IsEmpty()); } + size_t GetCount() const { return m_Count; } bool IsEmpty() const { return m_Count == 0; } + ItemType* Front() { return m_Front; } - const ItemType* Front() const { return m_Front; } ItemType* Back() { return m_Back; } + const ItemType* Front() const { return m_Front; } const ItemType* Back() const { return m_Back; } - void PushBack(ItemType* item) - { - D3D12MA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == NULL && ItemTypeTraits::GetNext(item) == NULL); - if(IsEmpty()) - { - m_Front = item; - m_Back = item; - m_Count = 1; - } - else - { - ItemTypeTraits::AccessPrev(item) = m_Back; - ItemTypeTraits::AccessNext(m_Back) = item; - m_Back = item; - ++m_Count; - } - } - void PushFront(ItemType* item) - { - D3D12MA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == NULL && ItemTypeTraits::GetNext(item) == NULL); - if(IsEmpty()) - { - m_Front = item; - m_Back = item; - m_Count = 1; - } - else - { - ItemTypeTraits::AccessNext(item) = m_Front; - ItemTypeTraits::AccessPrev(m_Front) = item; - m_Front = item; - ++m_Count; - } - } - ItemType* PopBack() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - ItemType* const backItem = m_Back; - ItemType* const prevItem = ItemTypeTraits::GetPrev(backItem); - if(prevItem != NULL) - { - ItemTypeTraits::AccessNext(prevItem) = NULL; - } - m_Back = prevItem; - --m_Count; - ItemTypeTraits::AccessPrev(backItem) = NULL; - ItemTypeTraits::AccessNext(backItem) = NULL; - return backItem; - } - ItemType* PopFront() - { - D3D12MA_HEAVY_ASSERT(m_Count > 0); - ItemType* const frontItem = m_Front; - ItemType* const nextItem = ItemTypeTraits::GetNext(frontItem); - if(nextItem != NULL) - { - ItemTypeTraits::AccessPrev(nextItem) = NULL; - } - m_Front = nextItem; - --m_Count; - ItemTypeTraits::AccessPrev(frontItem) = NULL; - ItemTypeTraits::AccessNext(frontItem) = NULL; - return frontItem; - } + + void PushBack(ItemType* item); + void PushFront(ItemType* item); + ItemType* PopBack(); + ItemType* PopFront(); // MyItem can be null - it means PushBack. - void InsertBefore(ItemType* existingItem, ItemType* newItem) - { - D3D12MA_HEAVY_ASSERT(newItem != NULL && ItemTypeTraits::GetPrev(newItem) == NULL && ItemTypeTraits::GetNext(newItem) == NULL); - if(existingItem != NULL) - { - ItemType* const prevItem = ItemTypeTraits::GetPrev(existingItem); - ItemTypeTraits::AccessPrev(newItem) = prevItem; - ItemTypeTraits::AccessNext(newItem) = existingItem; - ItemTypeTraits::AccessPrev(existingItem) = newItem; - if(prevItem != NULL) - { - ItemTypeTraits::AccessNext(prevItem) = newItem; - } - else - { - D3D12MA_HEAVY_ASSERT(m_Front == existingItem); - m_Front = newItem; - } - ++m_Count; - } - else - PushBack(newItem); - } + void InsertBefore(ItemType* existingItem, ItemType* newItem); // MyItem can be null - it means PushFront. - void InsertAfter(ItemType* existingItem, ItemType* newItem) - { - D3D12MA_HEAVY_ASSERT(newItem != NULL && ItemTypeTraits::GetPrev(newItem) == NULL && ItemTypeTraits::GetNext(newItem) == NULL); - if(existingItem != NULL) - { - ItemType* const nextItem = ItemTypeTraits::GetNext(existingItem); - ItemTypeTraits::AccessNext(newItem) = nextItem; - ItemTypeTraits::AccessPrev(newItem) = existingItem; - ItemTypeTraits::AccessNext(existingItem) = newItem; - if(nextItem != NULL) - { - ItemTypeTraits::AccessPrev(nextItem) = newItem; - } - else - { - D3D12MA_HEAVY_ASSERT(m_Back == existingItem); - m_Back = newItem; - } - ++m_Count; - } - else - return PushFront(newItem); - } - void Remove(ItemType* item) - { - D3D12MA_HEAVY_ASSERT(item != NULL && m_Count > 0); - if(ItemTypeTraits::GetPrev(item) != NULL) - { - ItemTypeTraits::AccessNext(ItemTypeTraits::AccessPrev(item)) = ItemTypeTraits::GetNext(item); - } - else - { - D3D12MA_HEAVY_ASSERT(m_Front == item); - m_Front = ItemTypeTraits::GetNext(item); - } + void InsertAfter(ItemType* existingItem, ItemType* newItem); + + void Remove(ItemType* item); + void RemoveAll(); - if(ItemTypeTraits::GetNext(item) != NULL) - { - ItemTypeTraits::AccessPrev(ItemTypeTraits::AccessNext(item)) = ItemTypeTraits::GetPrev(item); - } - else - { - D3D12MA_HEAVY_ASSERT(m_Back == item); - m_Back = ItemTypeTraits::GetPrev(item); - } - ItemTypeTraits::AccessPrev(item) = NULL; - ItemTypeTraits::AccessNext(item) = NULL; - --m_Count; - } private: ItemType* m_Front = NULL; ItemType* m_Back = NULL; size_t m_Count = 0; }; -//////////////////////////////////////////////////////////////////////////////// -// Private class AllocationObjectAllocator definition +#ifndef _D3D12MA_INTRUSIVE_LINKED_LIST_FUNCTIONS +template +IntrusiveLinkedList::IntrusiveLinkedList(IntrusiveLinkedList&& src) + : m_Front(src.m_Front), m_Back(src.m_Back), m_Count(src.m_Count) +{ + src.m_Front = src.m_Back = NULL; + src.m_Count = 0; +} +template +IntrusiveLinkedList& IntrusiveLinkedList::operator=(IntrusiveLinkedList&& src) +{ + if (&src != this) + { + D3D12MA_HEAVY_ASSERT(IsEmpty()); + m_Front = src.m_Front; + m_Back = src.m_Back; + m_Count = src.m_Count; + src.m_Front = src.m_Back = NULL; + src.m_Count = 0; + } + return *this; +} + +template +void IntrusiveLinkedList::PushBack(ItemType* item) +{ + D3D12MA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == NULL && ItemTypeTraits::GetNext(item) == NULL); + if (IsEmpty()) + { + m_Front = item; + m_Back = item; + m_Count = 1; + } + else + { + ItemTypeTraits::AccessPrev(item) = m_Back; + ItemTypeTraits::AccessNext(m_Back) = item; + m_Back = item; + ++m_Count; + } +} + +template +void IntrusiveLinkedList::PushFront(ItemType* item) +{ + D3D12MA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == NULL && ItemTypeTraits::GetNext(item) == NULL); + if (IsEmpty()) + { + m_Front = item; + m_Back = item; + m_Count = 1; + } + else + { + ItemTypeTraits::AccessNext(item) = m_Front; + ItemTypeTraits::AccessPrev(m_Front) = item; + m_Front = item; + ++m_Count; + } +} + +template +typename IntrusiveLinkedList::ItemType* IntrusiveLinkedList::PopBack() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + ItemType* const backItem = m_Back; + ItemType* const prevItem = ItemTypeTraits::GetPrev(backItem); + if (prevItem != NULL) + { + ItemTypeTraits::AccessNext(prevItem) = NULL; + } + m_Back = prevItem; + --m_Count; + ItemTypeTraits::AccessPrev(backItem) = NULL; + ItemTypeTraits::AccessNext(backItem) = NULL; + return backItem; +} + +template +typename IntrusiveLinkedList::ItemType* IntrusiveLinkedList::PopFront() +{ + D3D12MA_HEAVY_ASSERT(m_Count > 0); + ItemType* const frontItem = m_Front; + ItemType* const nextItem = ItemTypeTraits::GetNext(frontItem); + if (nextItem != NULL) + { + ItemTypeTraits::AccessPrev(nextItem) = NULL; + } + m_Front = nextItem; + --m_Count; + ItemTypeTraits::AccessPrev(frontItem) = NULL; + ItemTypeTraits::AccessNext(frontItem) = NULL; + return frontItem; +} + +template +void IntrusiveLinkedList::InsertBefore(ItemType* existingItem, ItemType* newItem) +{ + D3D12MA_HEAVY_ASSERT(newItem != NULL && ItemTypeTraits::GetPrev(newItem) == NULL && ItemTypeTraits::GetNext(newItem) == NULL); + if (existingItem != NULL) + { + ItemType* const prevItem = ItemTypeTraits::GetPrev(existingItem); + ItemTypeTraits::AccessPrev(newItem) = prevItem; + ItemTypeTraits::AccessNext(newItem) = existingItem; + ItemTypeTraits::AccessPrev(existingItem) = newItem; + if (prevItem != NULL) + { + ItemTypeTraits::AccessNext(prevItem) = newItem; + } + else + { + D3D12MA_HEAVY_ASSERT(m_Front == existingItem); + m_Front = newItem; + } + ++m_Count; + } + else + PushBack(newItem); +} + +template +void IntrusiveLinkedList::InsertAfter(ItemType* existingItem, ItemType* newItem) +{ + D3D12MA_HEAVY_ASSERT(newItem != NULL && ItemTypeTraits::GetPrev(newItem) == NULL && ItemTypeTraits::GetNext(newItem) == NULL); + if (existingItem != NULL) + { + ItemType* const nextItem = ItemTypeTraits::GetNext(existingItem); + ItemTypeTraits::AccessNext(newItem) = nextItem; + ItemTypeTraits::AccessPrev(newItem) = existingItem; + ItemTypeTraits::AccessNext(existingItem) = newItem; + if (nextItem != NULL) + { + ItemTypeTraits::AccessPrev(nextItem) = newItem; + } + else + { + D3D12MA_HEAVY_ASSERT(m_Back == existingItem); + m_Back = newItem; + } + ++m_Count; + } + else + return PushFront(newItem); +} + +template +void IntrusiveLinkedList::Remove(ItemType* item) +{ + D3D12MA_HEAVY_ASSERT(item != NULL && m_Count > 0); + if (ItemTypeTraits::GetPrev(item) != NULL) + { + ItemTypeTraits::AccessNext(ItemTypeTraits::AccessPrev(item)) = ItemTypeTraits::GetNext(item); + } + else + { + D3D12MA_HEAVY_ASSERT(m_Front == item); + m_Front = ItemTypeTraits::GetNext(item); + } + + if (ItemTypeTraits::GetNext(item) != NULL) + { + ItemTypeTraits::AccessPrev(ItemTypeTraits::AccessNext(item)) = ItemTypeTraits::GetPrev(item); + } + else + { + D3D12MA_HEAVY_ASSERT(m_Back == item); + m_Back = ItemTypeTraits::GetPrev(item); + } + ItemTypeTraits::AccessPrev(item) = NULL; + ItemTypeTraits::AccessNext(item) = NULL; + --m_Count; +} + +template +void IntrusiveLinkedList::RemoveAll() +{ + if (!IsEmpty()) + { + ItemType* item = m_Back; + while (item != NULL) + { + ItemType* const prevItem = ItemTypeTraits::AccessPrev(item); + ItemTypeTraits::AccessPrev(item) = NULL; + ItemTypeTraits::AccessNext(item) = NULL; + item = prevItem; + } + m_Front = NULL; + m_Back = NULL; + m_Count = 0; + } +} +#endif // _D3D12MA_INTRUSIVE_LINKED_LIST_FUNCTIONS +#endif // _D3D12MA_INTRUSIVE_LINKED_LIST + +#ifndef _D3D12MA_ALLOCATION_OBJECT_ALLOCATOR /* Thread-safe wrapper over PoolAllocator free list, for allocation of Allocation objects. */ @@ -2585,9 +2717,11 @@ class AllocationObjectAllocator { D3D12MA_CLASS_NO_COPY(AllocationObjectAllocator); public: - AllocationObjectAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks); + AllocationObjectAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks) + : m_Allocator(allocationCallbacks, 1024) {} - template Allocation* Allocate(Types... args); + template + Allocation* Allocate(Types... args); void Free(Allocation* alloc); private: @@ -2595,15 +2729,23 @@ private: PoolAllocator m_Allocator; }; -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata and derived classes - declarations - -enum SuballocationType +#ifndef _D3D12MA_ALLOCATION_OBJECT_ALLOCATOR_FUNCTIONS +template +Allocation* AllocationObjectAllocator::Allocate(Types... args) { - SUBALLOCATION_TYPE_FREE = 0, - SUBALLOCATION_TYPE_ALLOCATION = 1, -}; + MutexLock mutexLock(m_Mutex); + return m_Allocator.Alloc(std::forward(args)...); +} +void AllocationObjectAllocator::Free(Allocation* alloc) +{ + MutexLock mutexLock(m_Mutex); + m_Allocator.Free(alloc); +} +#endif // _D3D12MA_ALLOCATION_OBJECT_ALLOCATOR_FUNCTIONS +#endif // _D3D12MA_ALLOCATION_OBJECT_ALLOCATOR + +#ifndef _D3D12MA_SUBALLOCATION /* Represents a region of NormalBlock that is either assigned and returned as allocated memory block or free. @@ -2615,6 +2757,7 @@ struct Suballocation void* userData; SuballocationType type; }; +using SuballocationList = List; // Comparator for offsets. struct SuballocationOffsetLess @@ -2624,6 +2767,7 @@ struct SuballocationOffsetLess return lhs.offset < rhs.offset; } }; + struct SuballocationOffsetGreater { bool operator()(const Suballocation& lhs, const Suballocation& rhs) const @@ -2632,8 +2776,6 @@ struct SuballocationOffsetGreater } }; -using SuballocationList = List; - struct SuballocationItemSizeLess { bool operator()(const SuballocationList::iterator lhs, const SuballocationList::iterator rhs) const @@ -2645,7 +2787,9 @@ struct SuballocationItemSizeLess return lhs->size < rhsSize; } }; +#endif // _D3D12MA_SUBALLOCATION +#ifndef _D3D12MA_ALLOCATION_REQUEST /* Parameters of planned allocation inside a NormalBlock. */ @@ -2659,7 +2803,9 @@ struct AllocationRequest SuballocationList::iterator item; BOOL zeroInitialized; }; +#endif // _D3D12MA_ALLOCATION_REQUEST +#ifndef _D3D12MA_ZERO_INITIALIZED_RANGE /* Keeps track of the range of bytes that are surely initialized with zeros. Everything outside of it is considered uninitialized memory that may contain @@ -2670,53 +2816,61 @@ The range is left-inclusive. class ZeroInitializedRange { public: - void Reset(UINT64 size) - { - D3D12MA_ASSERT(size > 0); - m_ZeroBeg = 0; - m_ZeroEnd = size; - } - - BOOL IsRangeZeroInitialized(UINT64 beg, UINT64 end) const - { - D3D12MA_ASSERT(beg < end); - return m_ZeroBeg <= beg && end <= m_ZeroEnd; - } - - void MarkRangeAsUsed(UINT64 usedBeg, UINT64 usedEnd) - { - D3D12MA_ASSERT(usedBeg < usedEnd); - // No new bytes marked. - if(usedEnd <= m_ZeroBeg || m_ZeroEnd <= usedBeg) - { - return; - } - // All bytes marked. - if(usedBeg <= m_ZeroBeg && m_ZeroEnd <= usedEnd) - { - m_ZeroBeg = m_ZeroEnd = 0; - } - // Some bytes marked. - else - { - const UINT64 remainingZeroBefore = usedBeg > m_ZeroBeg ? usedBeg - m_ZeroBeg : 0; - const UINT64 remainingZeroAfter = usedEnd < m_ZeroEnd ? m_ZeroEnd - usedEnd : 0; - D3D12MA_ASSERT(remainingZeroBefore > 0 || remainingZeroAfter > 0); - if(remainingZeroBefore > remainingZeroAfter) - { - m_ZeroEnd = usedBeg; - } - else - { - m_ZeroBeg = usedEnd; - } - } - } + void Reset(UINT64 size); + BOOL IsRangeZeroInitialized(UINT64 beg, UINT64 end) const; + void MarkRangeAsUsed(UINT64 usedBeg, UINT64 usedEnd); private: UINT64 m_ZeroBeg = 0, m_ZeroEnd = 0; }; +#ifndef _D3D12MA_ZERO_INITIALIZED_RANGE_FUNCTIONS +void ZeroInitializedRange::Reset(UINT64 size) +{ + D3D12MA_ASSERT(size > 0); + m_ZeroBeg = 0; + m_ZeroEnd = size; +} + +BOOL ZeroInitializedRange::IsRangeZeroInitialized(UINT64 beg, UINT64 end) const +{ + D3D12MA_ASSERT(beg < end); + return m_ZeroBeg <= beg && end <= m_ZeroEnd; +} + +void ZeroInitializedRange::MarkRangeAsUsed(UINT64 usedBeg, UINT64 usedEnd) +{ + D3D12MA_ASSERT(usedBeg < usedEnd); + // No new bytes marked. + if (usedEnd <= m_ZeroBeg || m_ZeroEnd <= usedBeg) + { + return; + } + // All bytes marked. + if (usedBeg <= m_ZeroBeg && m_ZeroEnd <= usedEnd) + { + m_ZeroBeg = m_ZeroEnd = 0; + } + // Some bytes marked. + else + { + const UINT64 remainingZeroBefore = usedBeg > m_ZeroBeg ? usedBeg - m_ZeroBeg : 0; + const UINT64 remainingZeroAfter = usedEnd < m_ZeroEnd ? m_ZeroEnd - usedEnd : 0; + D3D12MA_ASSERT(remainingZeroBefore > 0 || remainingZeroAfter > 0); + if (remainingZeroBefore > remainingZeroAfter) + { + m_ZeroEnd = usedBeg; + } + else + { + m_ZeroBeg = usedEnd; + } + } +} +#endif // _D3D12MA_ZERO_INITIALIZED_RANGE_FUNCTIONS +#endif // _D3D12MA_ZERO_INITIALIZED_RANGE + +#ifndef _D3D12MA_BLOCK_METADATA /* Data structure used for bookkeeping of allocations and unused ranges of memory in a single ID3D12Heap memory block. @@ -2726,8 +2880,8 @@ class BlockMetadata public: BlockMetadata(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual); virtual ~BlockMetadata() = default; - virtual void Init(UINT64 size) { m_Size = size; } + virtual void Init(UINT64 size) { m_Size = size; } // Validates all data structures inside this object. If not valid, returns false. virtual bool Validate() const = 0; UINT64 GetSize() const { return m_Size; } @@ -2788,12 +2942,97 @@ private: D3D12MA_CLASS_NO_COPY(BlockMetadata); }; +#ifndef _D3D12MA_BLOCK_METADATA_FUNCTIONS +BlockMetadata::BlockMetadata(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) + : m_Size(0), + m_IsVirtual(isVirtual), + m_pAllocationCallbacks(allocationCallbacks) +{ + D3D12MA_ASSERT(allocationCallbacks); +} + +void BlockMetadata::PrintDetailedMap_Begin(JsonWriter& json, + UINT64 unusedBytes, size_t allocationCount, size_t unusedRangeCount) const +{ + json.BeginObject(); + + json.WriteString(L"TotalBytes"); + json.WriteNumber(GetSize()); + + json.WriteString(L"UnusedBytes"); + json.WriteNumber(unusedBytes); + + json.WriteString(L"Allocations"); + json.WriteNumber(allocationCount); + + json.WriteString(L"UnusedRanges"); + json.WriteNumber(unusedRangeCount); + + json.WriteString(L"Suballocations"); + json.BeginArray(); +} + +void BlockMetadata::PrintDetailedMap_Allocation(JsonWriter& json, + UINT64 offset, UINT64 size, void* userData) const +{ + json.BeginObject(true); + + json.WriteString(L"Offset"); + json.WriteNumber(offset); + + if (IsVirtual()) + { + json.WriteString(L"Type"); + json.WriteString(L"ALLOCATION"); + json.WriteString(L"Size"); + json.WriteNumber(size); + if (userData) + { + json.WriteString(L"UserData"); + json.WriteNumber((uintptr_t)userData); + } + } + else + { + const Allocation* const alloc = (const Allocation*)userData; + D3D12MA_ASSERT(alloc); + json.AddAllocationToObject(*alloc); + } + json.EndObject(); +} + +void BlockMetadata::PrintDetailedMap_UnusedRange(JsonWriter& json, + UINT64 offset, UINT64 size) const +{ + json.BeginObject(true); + + json.WriteString(L"Offset"); + json.WriteNumber(offset); + + json.WriteString(L"Type"); + json.WriteString(L"FREE"); + + json.WriteString(L"Size"); + json.WriteNumber(size); + + json.EndObject(); +} + +void BlockMetadata::PrintDetailedMap_End(JsonWriter& json) const +{ + json.EndArray(); + json.EndObject(); +} +#endif // _D3D12MA_BLOCK_METADATA_FUNCTIONS +#endif // _D3D12MA_BLOCK_METADATA + #if 0 +#ifndef _D3D12MA_BLOCK_METADATA_GENERIC class BlockMetadata_Generic : public BlockMetadata { public: BlockMetadata_Generic(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual); - virtual ~BlockMetadata_Generic(); + virtual ~BlockMetadata_Generic() = default; size_t GetAllocationCount() const override { return m_Suballocations.size() - m_FreeCount; } UINT64 GetSumFreeSize() const override { return m_SumFreeSize; } @@ -2861,8 +3100,537 @@ private: D3D12MA_CLASS_NO_COPY(BlockMetadata_Generic) }; + +#ifndef _D3D12MA_BLOCK_METADATA_GENERIC_FUNCTIONS +BlockMetadata_Generic::BlockMetadata_Generic(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) + : BlockMetadata(allocationCallbacks, isVirtual), + m_FreeCount(0), + m_SumFreeSize(0), + m_Suballocations(*allocationCallbacks), + m_FreeSuballocationsBySize(*allocationCallbacks) +{ + D3D12MA_ASSERT(allocationCallbacks); +} + +void BlockMetadata_Generic::Init(UINT64 size) +{ + BlockMetadata::Init(size); + m_ZeroInitializedRange.Reset(size); + + m_FreeCount = 1; + m_SumFreeSize = size; + + Suballocation suballoc = {}; + suballoc.offset = 0; + suballoc.size = size; + suballoc.type = SUBALLOCATION_TYPE_FREE; + suballoc.userData = NULL; + + D3D12MA_ASSERT(size > MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); + m_Suballocations.push_back(suballoc); + SuballocationList::iterator suballocItem = m_Suballocations.end(); + --suballocItem; + m_FreeSuballocationsBySize.push_back(suballocItem); +} + +bool BlockMetadata_Generic::Validate() const +{ + D3D12MA_VALIDATE(!m_Suballocations.empty()); + + // Expected offset of new suballocation as calculated from previous ones. + UINT64 calculatedOffset = 0; + // Expected number of free suballocations as calculated from traversing their list. + UINT calculatedFreeCount = 0; + // Expected sum size of free suballocations as calculated from traversing their list. + UINT64 calculatedSumFreeSize = 0; + // Expected number of free suballocations that should be registered in + // m_FreeSuballocationsBySize calculated from traversing their list. + size_t freeSuballocationsToRegister = 0; + // True if previous visited suballocation was free. + bool prevFree = false; + + for (const auto& subAlloc : m_Suballocations) + { + // Actual offset of this suballocation doesn't match expected one. + D3D12MA_VALIDATE(subAlloc.offset == calculatedOffset); + + const bool currFree = (subAlloc.type == SUBALLOCATION_TYPE_FREE); + // Two adjacent free suballocations are invalid. They should be merged. + D3D12MA_VALIDATE(!prevFree || !currFree); + + const Allocation* const alloc = (Allocation*)subAlloc.userData; + if (!IsVirtual()) + { + D3D12MA_VALIDATE(currFree == (alloc == NULL)); + } + + if (currFree) + { + calculatedSumFreeSize += subAlloc.size; + ++calculatedFreeCount; + if (subAlloc.size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + ++freeSuballocationsToRegister; + } + + // Margin required between allocations - every free space must be at least that large. + D3D12MA_VALIDATE(subAlloc.size >= GetDebugMargin()); + } + else + { + if (!IsVirtual()) + { + D3D12MA_VALIDATE(alloc->GetOffset() == subAlloc.offset); + D3D12MA_VALIDATE(alloc->GetSize() == subAlloc.size); + } + + // Margin required between allocations - previous allocation must be free. + D3D12MA_VALIDATE(GetDebugMargin() == 0 || prevFree); + } + + calculatedOffset += subAlloc.size; + prevFree = currFree; + } + + // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't + // match expected one. + D3D12MA_VALIDATE(m_FreeSuballocationsBySize.size() == freeSuballocationsToRegister); + + UINT64 lastSize = 0; + for (size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i) + { + SuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i]; + + // Only free suballocations can be registered in m_FreeSuballocationsBySize. + D3D12MA_VALIDATE(suballocItem->type == SUBALLOCATION_TYPE_FREE); + // They must be sorted by size ascending. + D3D12MA_VALIDATE(suballocItem->size >= lastSize); + + lastSize = suballocItem->size; + } + + // Check if totals match calculacted values. + D3D12MA_VALIDATE(ValidateFreeSuballocationList()); + D3D12MA_VALIDATE(calculatedOffset == GetSize()); + D3D12MA_VALIDATE(calculatedSumFreeSize == m_SumFreeSize); + D3D12MA_VALIDATE(calculatedFreeCount == m_FreeCount); + + return true; +} + +bool BlockMetadata_Generic::IsEmpty() const +{ + return (m_Suballocations.size() == 1) && (m_FreeCount == 1); +} + +void BlockMetadata_Generic::GetAllocationInfo(AllocHandle allocHandle, VIRTUAL_ALLOCATION_INFO& outInfo) const +{ + Suballocation& suballoc = *FindAtOffset((UINT64)allocHandle - 1).dropConst(); + outInfo.Offset = suballoc.offset; + outInfo.Size = suballoc.size; + outInfo.pUserData = suballoc.userData; +} + +bool BlockMetadata_Generic::CreateAllocationRequest( + UINT64 allocSize, + UINT64 allocAlignment, + bool upperAddress, + AllocationRequest* pAllocationRequest) +{ + D3D12MA_ASSERT(allocSize > 0); + D3D12MA_ASSERT(!upperAddress && "ALLOCATION_FLAG_UPPER_ADDRESS can be used only with linear algorithm."); + D3D12MA_ASSERT(pAllocationRequest != NULL); + D3D12MA_HEAVY_ASSERT(Validate()); + + // There is not enough total free space in this block to fullfill the request: Early return. + if (m_SumFreeSize < allocSize + GetDebugMargin()) + { + return false; + } + + // New algorithm, efficiently searching freeSuballocationsBySize. + const size_t freeSuballocCount = m_FreeSuballocationsBySize.size(); + if (freeSuballocCount > 0) + { + // Find first free suballocation with size not less than allocSize + GetDebugMargin(). + SuballocationList::iterator* const it = BinaryFindFirstNotLess( + m_FreeSuballocationsBySize.data(), + m_FreeSuballocationsBySize.data() + freeSuballocCount, + allocSize + GetDebugMargin(), + SuballocationItemSizeLess()); + size_t index = it - m_FreeSuballocationsBySize.data(); + for (; index < freeSuballocCount; ++index) + { + if (CheckAllocation( + allocSize, + allocAlignment, + m_FreeSuballocationsBySize[index], + &pAllocationRequest->allocHandle, + &pAllocationRequest->sumFreeSize, + &pAllocationRequest->sumItemSize, + &pAllocationRequest->zeroInitialized)) + { + pAllocationRequest->item = m_FreeSuballocationsBySize[index]; + return true; + } + } + } + + return false; +} + +void BlockMetadata_Generic::Alloc( + const AllocationRequest& request, + UINT64 allocSize, + void* userData) +{ + D3D12MA_ASSERT(request.item != m_Suballocations.end()); + Suballocation& suballoc = *request.item; + // Given suballocation is a free block. + D3D12MA_ASSERT(suballoc.type == SUBALLOCATION_TYPE_FREE); + // Given offset is inside this suballocation. + UINT64 offset = (UINT64)request.allocHandle - 1; + D3D12MA_ASSERT(offset >= suballoc.offset); + const UINT64 paddingBegin = offset - suballoc.offset; + D3D12MA_ASSERT(suballoc.size >= paddingBegin + allocSize); + const UINT64 paddingEnd = suballoc.size - paddingBegin - allocSize; + + // Unregister this free suballocation from m_FreeSuballocationsBySize and update + // it to become used. + UnregisterFreeSuballocation(request.item); + + suballoc.offset = offset; + suballoc.size = allocSize; + suballoc.type = SUBALLOCATION_TYPE_ALLOCATION; + suballoc.userData = userData; + + // If there are any free bytes remaining at the end, insert new free suballocation after current one. + if (paddingEnd) + { + Suballocation paddingSuballoc = {}; + paddingSuballoc.offset = offset + allocSize; + paddingSuballoc.size = paddingEnd; + paddingSuballoc.type = SUBALLOCATION_TYPE_FREE; + SuballocationList::iterator next = request.item; + ++next; + const SuballocationList::iterator paddingEndItem = + m_Suballocations.insert(next, paddingSuballoc); + RegisterFreeSuballocation(paddingEndItem); + } + + // If there are any free bytes remaining at the beginning, insert new free suballocation before current one. + if (paddingBegin) + { + Suballocation paddingSuballoc = {}; + paddingSuballoc.offset = offset - paddingBegin; + paddingSuballoc.size = paddingBegin; + paddingSuballoc.type = SUBALLOCATION_TYPE_FREE; + const SuballocationList::iterator paddingBeginItem = + m_Suballocations.insert(request.item, paddingSuballoc); + RegisterFreeSuballocation(paddingBeginItem); + } + + // Update totals. + m_FreeCount = m_FreeCount - 1; + if (paddingBegin > 0) + { + ++m_FreeCount; + } + if (paddingEnd > 0) + { + ++m_FreeCount; + } + m_SumFreeSize -= allocSize; + + m_ZeroInitializedRange.MarkRangeAsUsed(offset, offset + allocSize); +} + +void BlockMetadata_Generic::Free(AllocHandle allocHandle) +{ + FreeSuballocation(FindAtOffset((UINT64)allocHandle - 1).dropConst()); +} + +void BlockMetadata_Generic::Clear() +{ + m_FreeCount = 1; + m_SumFreeSize = GetSize(); + + m_Suballocations.clear(); + Suballocation suballoc = {}; + suballoc.offset = 0; + suballoc.size = GetSize(); + suballoc.type = SUBALLOCATION_TYPE_FREE; + m_Suballocations.push_back(suballoc); + + m_FreeSuballocationsBySize.clear(); + m_FreeSuballocationsBySize.push_back(m_Suballocations.begin()); +} + +SuballocationList::const_iterator BlockMetadata_Generic::FindAtOffset(UINT64 offset) const +{ + const UINT64 last = m_Suballocations.crbegin()->offset; + if (last == offset) + return m_Suballocations.crbegin(); + const UINT64 first = m_Suballocations.cbegin()->offset; + if (first == offset) + return m_Suballocations.cbegin(); + + const size_t suballocCount = m_Suballocations.size(); + const UINT64 step = (last - first + m_Suballocations.cbegin()->size) / suballocCount; + auto findSuballocation = [&](auto begin, auto end) -> SuballocationList::const_iterator + { + for (auto suballocItem = begin; + suballocItem != end; + ++suballocItem) + { + const Suballocation& suballoc = *suballocItem; + if (suballoc.offset == offset) + return suballocItem; + } + D3D12MA_ASSERT(false && "Not found!"); + return m_Suballocations.end(); + }; + // If requested offset is closer to the end of range, search from the end + if ((offset - first) > suballocCount * step / 2) + { + return findSuballocation(m_Suballocations.crbegin(), m_Suballocations.crend()); + } + return findSuballocation(m_Suballocations.cbegin(), m_Suballocations.cend()); +} + +bool BlockMetadata_Generic::ValidateFreeSuballocationList() const +{ + UINT64 lastSize = 0; + for (size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i) + { + const SuballocationList::iterator it = m_FreeSuballocationsBySize[i]; + + D3D12MA_VALIDATE(it->type == SUBALLOCATION_TYPE_FREE); + D3D12MA_VALIDATE(it->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); + D3D12MA_VALIDATE(it->size >= lastSize); + lastSize = it->size; + } + return true; +} + +bool BlockMetadata_Generic::CheckAllocation( + UINT64 allocSize, + UINT64 allocAlignment, + SuballocationList::const_iterator suballocItem, + AllocHandle* pAllocHandle, + UINT64* pSumFreeSize, + UINT64* pSumItemSize, + BOOL* pZeroInitialized) const +{ + D3D12MA_ASSERT(allocSize > 0); + D3D12MA_ASSERT(suballocItem != m_Suballocations.cend()); + D3D12MA_ASSERT(pAllocHandle != NULL && pZeroInitialized != NULL); + + *pSumFreeSize = 0; + *pSumItemSize = 0; + *pZeroInitialized = FALSE; + + const Suballocation& suballoc = *suballocItem; + D3D12MA_ASSERT(suballoc.type == SUBALLOCATION_TYPE_FREE); + + *pSumFreeSize = suballoc.size; + + // Size of this suballocation is too small for this request: Early return. + if (suballoc.size < allocSize) + { + return false; + } + + // Start from offset equal to beginning of this suballocation and debug margin of previous allocation if present. + UINT64 offset = suballoc.offset + (suballocItem == m_Suballocations.cbegin() ? 0 : GetDebugMargin()); + + // Apply alignment. + offset = AlignUp(offset, allocAlignment); + + // Calculate padding at the beginning based on current offset. + const UINT64 paddingBegin = offset - suballoc.offset; + + // Fail if requested size plus margin after is bigger than size of this suballocation. + if (paddingBegin + allocSize + GetDebugMargin() > suballoc.size) + { + return false; + } + + // All tests passed: Success. Offset is already filled. + *pZeroInitialized = m_ZeroInitializedRange.IsRangeZeroInitialized(offset, offset + allocSize); + *pAllocHandle = (AllocHandle)(offset + 1); + return true; +} + +void BlockMetadata_Generic::MergeFreeWithNext(SuballocationList::iterator item) +{ + D3D12MA_ASSERT(item != m_Suballocations.end()); + D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); + + SuballocationList::iterator nextItem = item; + ++nextItem; + D3D12MA_ASSERT(nextItem != m_Suballocations.end()); + D3D12MA_ASSERT(nextItem->type == SUBALLOCATION_TYPE_FREE); + + item->size += nextItem->size; + --m_FreeCount; + m_Suballocations.erase(nextItem); +} + +SuballocationList::iterator BlockMetadata_Generic::FreeSuballocation(SuballocationList::iterator suballocItem) +{ + // Change this suballocation to be marked as free. + Suballocation& suballoc = *suballocItem; + suballoc.type = SUBALLOCATION_TYPE_FREE; + suballoc.userData = NULL; + + // Update totals. + ++m_FreeCount; + m_SumFreeSize += suballoc.size; + + // Merge with previous and/or next suballocation if it's also free. + bool mergeWithNext = false; + bool mergeWithPrev = false; + + SuballocationList::iterator nextItem = suballocItem; + ++nextItem; + if ((nextItem != m_Suballocations.end()) && (nextItem->type == SUBALLOCATION_TYPE_FREE)) + { + mergeWithNext = true; + } + + SuballocationList::iterator prevItem = suballocItem; + if (suballocItem != m_Suballocations.begin()) + { + --prevItem; + if (prevItem->type == SUBALLOCATION_TYPE_FREE) + { + mergeWithPrev = true; + } + } + + if (mergeWithNext) + { + UnregisterFreeSuballocation(nextItem); + MergeFreeWithNext(suballocItem); + } + + if (mergeWithPrev) + { + UnregisterFreeSuballocation(prevItem); + MergeFreeWithNext(prevItem); + RegisterFreeSuballocation(prevItem); + return prevItem; + } + else + { + RegisterFreeSuballocation(suballocItem); + return suballocItem; + } +} + +void BlockMetadata_Generic::RegisterFreeSuballocation(SuballocationList::iterator item) +{ + D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); + D3D12MA_ASSERT(item->size > 0); + + // You may want to enable this validation at the beginning or at the end of + // this function, depending on what do you want to check. + D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); + + if (item->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + if (m_FreeSuballocationsBySize.empty()) + { + m_FreeSuballocationsBySize.push_back(item); + } + else + { + m_FreeSuballocationsBySize.InsertSorted(item, SuballocationItemSizeLess()); + } + } + + //D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); +} + +void BlockMetadata_Generic::UnregisterFreeSuballocation(SuballocationList::iterator item) +{ + D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); + D3D12MA_ASSERT(item->size > 0); + + // You may want to enable this validation at the beginning or at the end of + // this function, depending on what do you want to check. + D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); + + if (item->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + SuballocationList::iterator* const it = BinaryFindFirstNotLess( + m_FreeSuballocationsBySize.data(), + m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(), + item, + SuballocationItemSizeLess()); + for (size_t index = it - m_FreeSuballocationsBySize.data(); + index < m_FreeSuballocationsBySize.size(); + ++index) + { + if (m_FreeSuballocationsBySize[index] == item) + { + m_FreeSuballocationsBySize.remove(index); + return; + } + D3D12MA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found."); + } + D3D12MA_ASSERT(0 && "Not found."); + } + + //D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); +} + +void BlockMetadata_Generic::SetAllocationUserData(AllocHandle allocHandle, void* userData) +{ + Suballocation& suballoc = *FindAtOffset((UINT64)allocHandle - 1).dropConst(); + suballoc.userData = userData; +} + +void BlockMetadata_Generic::AddStatistics(Statistics& inoutStats) const +{ + inoutStats.BlockCount++; + inoutStats.AllocationCount += (UINT)m_Suballocations.size() - m_FreeCount; + inoutStats.BlockBytes += GetSize(); + inoutStats.AllocationBytes += GetSize() - m_SumFreeSize; +} + +void BlockMetadata_Generic::AddDetailedStatistics(DetailedStatistics& inoutStats) const +{ + inoutStats.Stats.BlockCount++; + inoutStats.Stats.BlockBytes += GetSize(); + + for (const auto& suballoc : m_Suballocations) + { + if (suballoc.type == SUBALLOCATION_TYPE_FREE) + AddDetailedStatisticsUnusedRange(inoutStats, suballoc.size); + else + AddDetailedStatisticsAllocation(inoutStats, suballoc.size); + } +} + +void BlockMetadata_Generic::WriteAllocationInfoToJson(JsonWriter& json) const +{ + PrintDetailedMap_Begin(json, GetSumFreeSize(), GetAllocationCount(), m_FreeCount); + for (const auto& suballoc : m_Suballocations) + { + if (suballoc.type == SUBALLOCATION_TYPE_FREE) + PrintDetailedMap_UnusedRange(json, suballoc.offset, suballoc.size); + else + PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.size, suballoc.userData); + } + PrintDetailedMap_End(json); +} +#endif // _D3D12MA_BLOCK_METADATA_GENERIC_FUNCTIONS +#endif // _D3D12MA_BLOCK_METADATA_GENERIC #endif // #if 0 +#ifndef _D3D12MA_BLOCK_METADATA_LINEAR class BlockMetadata_Linear : public BlockMetadata { public: @@ -2963,1316 +3731,7 @@ private: D3D12MA_CLASS_NO_COPY(BlockMetadata_Linear) }; -class BlockMetadata_TLSF : public BlockMetadata -{ -public: - BlockMetadata_TLSF(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual); - virtual ~BlockMetadata_TLSF(); - - size_t GetAllocationCount() const override { return m_AllocCount; } - UINT64 GetSumFreeSize() const override { return m_BlocksFreeSize + m_NullBlock->size; } - bool IsEmpty() const override { return m_NullBlock->offset == 0; } - UINT64 GetAllocationOffset(AllocHandle allocHandle) const override { return ((Block*)allocHandle)->offset; }; - - void Init(UINT64 size) override; - bool Validate() const override; - void GetAllocationInfo(AllocHandle allocHandle, VIRTUAL_ALLOCATION_INFO& outInfo) const override; - - bool CreateAllocationRequest( - UINT64 allocSize, - UINT64 allocAlignment, - bool upperAddress, - AllocationRequest* pAllocationRequest) override; - - void Alloc( - const AllocationRequest& request, - UINT64 allocSize, - void* userData) override; - - void Free(AllocHandle allocHandle) override; - void Clear() override; - - void SetAllocationUserData(AllocHandle allocHandle, void* userData) override; - - void AddStatistics(Statistics& inoutStats) const override; - void AddDetailedStatistics(DetailedStatistics& inoutStats) const override; - void WriteAllocationInfoToJson(JsonWriter& json) const override; - -private: - // According to original paper it should be preferable 4 or 5: - // M. Masmano, I. Ripoll, A. Crespo, and J. Real "TLSF: a New Dynamic Memory Allocator for Real-Time Systems" - // http://www.gii.upv.es/tlsf/files/ecrts04_tlsf.pdf - static const UINT8 SECOND_LEVEL_INDEX = 5; - static const UINT16 SMALL_BUFFER_SIZE = 256; - static const UINT INITIAL_BLOCK_ALLOC_COUNT = 16; - static const UINT8 MEMORY_CLASS_SHIFT = 7; - static const UINT8 MAX_MEMORY_CLASSES = 65 - MEMORY_CLASS_SHIFT; - - class Block - { - public: - UINT64 offset; - UINT64 size; - Block* prevPhysical; - Block* nextPhysical; - - void MarkFree() { prevFree = NULL; } - void MarkTaken() { prevFree = this; } - bool IsFree() const { return prevFree != this; } - void*& UserData() { D3D12MA_HEAVY_ASSERT(!IsFree()); return userData; } - Block*& PrevFree() { return prevFree; } - Block*& NextFree() { D3D12MA_HEAVY_ASSERT(IsFree()); return nextFree; } - - private: - Block* prevFree; // Address of the same block here indicates that block is taken - union - { - Block* nextFree; - void* userData; - }; - }; - - size_t m_AllocCount; - // Total number of free blocks besides null block - size_t m_BlocksFreeCount; - // Total size of free blocks excluding null block - UINT64 m_BlocksFreeSize; - UINT32 m_IsFreeBitmap; - UINT8 m_MemoryClasses; - UINT32 m_InnerIsFreeBitmap[MAX_MEMORY_CLASSES]; - UINT32 m_ListsCount; - /* - * 0: 0-3 lists for small buffers - * 1+: 0-(2^SLI-1) lists for normal buffers - */ - Block** m_FreeList; - PoolAllocator m_BlockAllocator; - Block* m_NullBlock; - - UINT8 SizeToMemoryClass(UINT64 size) const; - UINT16 SizeToSecondIndex(UINT64 size, UINT8 memoryClass) const; - UINT32 GetListIndex(UINT8 memoryClass, UINT16 secondIndex) const; - UINT32 GetListIndex(UINT64 size) const; - - void RemoveFreeBlock(Block* block); - void InsertFreeBlock(Block* block); - void MergeBlock(Block* block, Block* prev); - - Block* FindFreeBlock(UINT64 size, UINT32& listIndex) const; - bool CheckBlock( - Block& block, - UINT32 listIndex, - UINT64 allocSize, - UINT64 allocAlignment, - AllocationRequest* pAllocationRequest); - - D3D12MA_CLASS_NO_COPY(BlockMetadata_TLSF) -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class MemoryBlock definition - -/* -Represents a single block of device memory (heap). -Base class for inheritance. -Thread-safety: This class must be externally synchronized. -*/ -class MemoryBlock -{ -public: - MemoryBlock( - AllocatorPimpl* allocator, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 size, - UINT id); - virtual ~MemoryBlock(); - // Creates the ID3D12Heap. - - const D3D12_HEAP_PROPERTIES& GetHeapProperties() const { return m_HeapProps; } - D3D12_HEAP_FLAGS GetHeapFlags() const { return m_HeapFlags; } - UINT64 GetSize() const { return m_Size; } - UINT GetId() const { return m_Id; } - ID3D12Heap* GetHeap() const { return m_Heap; } - -protected: - AllocatorPimpl* const m_Allocator; - const D3D12_HEAP_PROPERTIES m_HeapProps; - const D3D12_HEAP_FLAGS m_HeapFlags; - const UINT64 m_Size; - const UINT m_Id; - - HRESULT Init(ID3D12ProtectedResourceSession* pProtectedSession); - -private: - ID3D12Heap* m_Heap = NULL; - - D3D12MA_CLASS_NO_COPY(MemoryBlock) -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class NormalBlock definition - -/* -Represents a single block of device memory (heap) with all the data about its -regions (aka suballocations, Allocation), assigned and free. -Thread-safety: This class must be externally synchronized. -*/ -class NormalBlock : public MemoryBlock -{ -public: - BlockMetadata* m_pMetadata; - - NormalBlock( - AllocatorPimpl* allocator, - BlockVector* blockVector, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 size, - UINT id); - virtual ~NormalBlock(); - - BlockVector* GetBlockVector() const { return m_BlockVector; } - - // 'algorithm' should be one of the *_ALGORITHM_* flags in enums POOL_FLAGS or VIRTUAL_BLOCK_FLAGS - HRESULT Init(UINT32 algorithm, ID3D12ProtectedResourceSession* pProtectedSession); - - // Validates all data structures inside this object. If not valid, returns false. - bool Validate() const; - -private: - BlockVector* m_BlockVector; - - D3D12MA_CLASS_NO_COPY(NormalBlock) -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class CommittedAllocationList definition - -struct CommittedAllocationListItemTraits -{ - using ItemType = Allocation; - static ItemType* GetPrev(const ItemType* item) - { - D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); - return item->m_Committed.prev; - } - static ItemType* GetNext(const ItemType* item) - { - D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); - return item->m_Committed.next; - } - static ItemType*& AccessPrev(ItemType* item) - { - D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); - return item->m_Committed.prev; - } - static ItemType*& AccessNext(ItemType* item) - { - D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); - return item->m_Committed.next; - } -}; - -/* -Stores linked list of Allocation objects that are of TYPE_COMMITTED or TYPE_HEAP. -Thread-safe, synchronized internally. -*/ -class CommittedAllocationList -{ -public: - CommittedAllocationList(); - void Init(bool useMutex, D3D12_HEAP_TYPE heapType, PoolPimpl* pool); - ~CommittedAllocationList(); - - D3D12_HEAP_TYPE GetHeapType() const { return m_HeapType; } - PoolPimpl* GetPool() const { return m_Pool; } - UINT GetMemorySegmentGroup(AllocatorPimpl* allocator) const; - - void AddStatistics(Statistics& inoutStats); - void AddDetailedStatistics(DetailedStatistics& inoutStats); - // Writes JSON array with the list of allocations. - void BuildStatsString(JsonWriter& json); - - void Register(Allocation* alloc); - void Unregister(Allocation* alloc); - -private: - bool m_UseMutex = true; - D3D12_HEAP_TYPE m_HeapType = D3D12_HEAP_TYPE_CUSTOM; - PoolPimpl* m_Pool = NULL; - - D3D12MA_RW_MUTEX m_Mutex; - using CommittedAllocationLinkedList = IntrusiveLinkedList; - CommittedAllocationLinkedList m_AllocationList; -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockVector definition - -/* -Sequence of NormalBlock. Represents memory blocks allocated for a specific -heap type and possibly resource type (if only Tier 1 is supported). - -Synchronized internally with a mutex. -*/ -class BlockVector -{ - D3D12MA_CLASS_NO_COPY(BlockVector) -public: - BlockVector( - AllocatorPimpl* hAllocator, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 preferredBlockSize, - size_t minBlockCount, - size_t maxBlockCount, - bool explicitBlockSize, - UINT64 minAllocationAlignment, - UINT32 algorithm, - ID3D12ProtectedResourceSession* pProtectedSession); - ~BlockVector(); - - HRESULT CreateMinBlocks(); - - const D3D12_HEAP_PROPERTIES& GetHeapProperties() const { return m_HeapProps; } - UINT64 GetPreferredBlockSize() const { return m_PreferredBlockSize; } - - bool IsEmpty(); - - HRESULT Allocate( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - size_t allocationCount, - Allocation** pAllocations); - - void Free( - Allocation* hAllocation); - - HRESULT CreateResource( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - const D3D12_RESOURCE_DESC& resourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource); - -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - HRESULT CreateResource2( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - const D3D12_RESOURCE_DESC1& resourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource); -#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ - - void AddStatistics(Statistics& inoutStats); - void AddDetailedStatistics(DetailedStatistics& inoutStats); - - void WriteBlockInfoToJson(JsonWriter& json); - -private: - AllocatorPimpl* const m_hAllocator; - const D3D12_HEAP_PROPERTIES m_HeapProps; - const D3D12_HEAP_FLAGS m_HeapFlags; - const UINT64 m_PreferredBlockSize; - const size_t m_MinBlockCount; - const size_t m_MaxBlockCount; - const bool m_ExplicitBlockSize; - const UINT64 m_MinAllocationAlignment; - const UINT32 m_Algorithm; - ID3D12ProtectedResourceSession* const m_ProtectedSession; - /* There can be at most one allocation that is completely empty - a - hysteresis to avoid pessimistic case of alternating creation and destruction - of a VkDeviceMemory. */ - bool m_HasEmptyBlock; - D3D12MA_RW_MUTEX m_Mutex; - // Incrementally sorted by sumFreeSize, ascending. - Vector m_Blocks; - UINT m_NextBlockId; - - UINT64 CalcSumBlockSize() const; - UINT64 CalcMaxBlockSize() const; - - // Finds and removes given block from vector. - void Remove(NormalBlock* pBlock); - - // Performs single step in sorting m_Blocks. They may not be fully sorted - // after this call. - void IncrementallySortBlocks(); - - HRESULT AllocatePage( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - Allocation** pAllocation); - - HRESULT AllocateFromBlock( - NormalBlock* pBlock, - UINT64 size, - UINT64 alignment, - ALLOCATION_FLAGS allocFlags, - Allocation** pAllocation); - - HRESULT CreateBlock( - UINT64 blockSize, - size_t* pNewBlockIndex); -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class AllocatorPimpl definition - -static constexpr UINT HEAP_TYPE_COUNT = 4; -static constexpr UINT STANDARD_HEAP_TYPE_COUNT = 3; // Only DEFAULT, UPLOAD, READBACK. -static constexpr UINT DEFAULT_POOL_MAX_COUNT = 9; - -class CurrentBudgetData -{ -public: - void GetStatistics(Statistics& outStats, UINT group) const - { - outStats.BlockCount = m_BlockCount[group]; - outStats.AllocationCount = m_AllocationCount[group]; - outStats.BlockBytes = m_BlockBytes[group]; - outStats.AllocationBytes = m_AllocationBytes[group]; - } - void GetBudget(bool useMutex, - UINT64* outLocalUsage, UINT64* outLocalBudget, - UINT64* outNonLocalUsage, UINT64* outNonLocalBudget); - - void AddAllocation(UINT group, UINT64 allocationBytes) - { - ++m_AllocationCount[group]; - m_AllocationBytes[group] += allocationBytes; - ++m_OperationsSinceBudgetFetch; - } - void RemoveAllocation(UINT group, UINT64 allocationBytes) - { - D3D12MA_ASSERT(m_AllocationBytes[group] >= allocationBytes); - D3D12MA_ASSERT(m_AllocationCount[group] > 0); - m_AllocationBytes[group] -= allocationBytes; - --m_AllocationCount[group]; - ++m_OperationsSinceBudgetFetch; - } - void AddBlock(UINT group, UINT64 blockBytes) - { - ++m_BlockCount[group]; - m_BlockBytes[group] += blockBytes; - ++m_OperationsSinceBudgetFetch; - } - void RemoveBlock(UINT group, UINT64 blockBytes) - { - D3D12MA_ASSERT(m_BlockBytes[group] >= blockBytes); - D3D12MA_ASSERT(m_BlockCount[group] > 0); - m_BlockBytes[group] -= blockBytes; - --m_BlockCount[group]; - ++m_OperationsSinceBudgetFetch; - } - bool ShouldUpdateBudget() const - { - return m_OperationsSinceBudgetFetch >= 30; - } -#if D3D12MA_DXGI_1_4 - HRESULT UpdateBudget(IDXGIAdapter3* adapter3, bool useMutex); -#endif - -private: - D3D12MA_ATOMIC_UINT32 m_BlockCount[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - D3D12MA_ATOMIC_UINT32 m_AllocationCount[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - D3D12MA_ATOMIC_UINT64 m_BlockBytes[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - D3D12MA_ATOMIC_UINT64 m_AllocationBytes[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - - D3D12MA_ATOMIC_UINT32 m_OperationsSinceBudgetFetch = 0; - D3D12MA_RW_MUTEX m_BudgetMutex; - UINT64 m_D3D12Usage[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - UINT64 m_D3D12Budget[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; - UINT64 m_BlockBytesAtD3D12Fetch[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; -}; - -class PoolPimpl -{ -public: - PoolPimpl(AllocatorPimpl* allocator, const POOL_DESC& desc); - HRESULT Init(); - ~PoolPimpl(); - - AllocatorPimpl* GetAllocator() const { return m_Allocator; } - const POOL_DESC& GetDesc() const { return m_Desc; } - bool SupportsCommittedAllocations() const { return m_Desc.BlockSize == 0; } - - BlockVector* GetBlockVector() { return m_BlockVector; } - CommittedAllocationList* GetCommittedAllocationList() { return SupportsCommittedAllocations() ? &m_CommittedAllocations : NULL; } - - void GetStatistics(Statistics& outStats); - void CalculateStatistics(DetailedStatistics& outStats); - void AddDetailedStatistics(DetailedStatistics& inoutStats); - - void SetName(LPCWSTR Name); - LPCWSTR GetName() const { return m_Name; } - -private: - friend class Allocator; - friend struct PoolListItemTraits; - - AllocatorPimpl* m_Allocator; // Externally owned object. - POOL_DESC m_Desc; - BlockVector* m_BlockVector; // Owned object. - CommittedAllocationList m_CommittedAllocations; - wchar_t* m_Name; - PoolPimpl* m_PrevPool = NULL; - PoolPimpl* m_NextPool = NULL; - - void FreeName(); -}; - -struct PoolListItemTraits -{ - using ItemType = PoolPimpl; - static ItemType* GetPrev(const ItemType* item) { return item->m_PrevPool; } - static ItemType* GetNext(const ItemType* item) { return item->m_NextPool; } - static ItemType*& AccessPrev(ItemType* item) { return item->m_PrevPool; } - static ItemType*& AccessNext(ItemType* item) { return item->m_NextPool; } -}; - -struct CommittedAllocationParameters -{ - CommittedAllocationList* m_List = NULL; - D3D12_HEAP_PROPERTIES m_HeapProperties = {}; - D3D12_HEAP_FLAGS m_HeapFlags = D3D12_HEAP_FLAG_NONE; - ID3D12ProtectedResourceSession* m_ProtectedSession = NULL; - - bool IsValid() const { return m_List != NULL; } -}; - -class AllocatorPimpl -{ -public: - std::atomic_uint32_t m_RefCount = 1; - CurrentBudgetData m_Budget; - - AllocatorPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc); - HRESULT Init(const ALLOCATOR_DESC& desc); - ~AllocatorPimpl(); - - ID3D12Device* GetDevice() const { return m_Device; } -#ifdef __ID3D12Device4_INTERFACE_DEFINED__ - ID3D12Device4* GetDevice4() const { return m_Device4; } -#endif -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - ID3D12Device8* GetDevice8() const { return m_Device8; } -#endif - // Shortcut for "Allocation Callbacks", because this function is called so often. - const ALLOCATION_CALLBACKS& GetAllocs() const { return m_AllocationCallbacks; } - const D3D12_FEATURE_DATA_D3D12_OPTIONS& GetD3D12Options() const { return m_D3D12Options; } - BOOL IsUMA() const { return m_D3D12Architecture.UMA; } - BOOL IsCacheCoherentUMA() const { return m_D3D12Architecture.CacheCoherentUMA; } - bool SupportsResourceHeapTier2() const { return m_D3D12Options.ResourceHeapTier >= D3D12_RESOURCE_HEAP_TIER_2; } - bool UseMutex() const { return m_UseMutex; } - AllocationObjectAllocator& GetAllocationObjectAllocator() { return m_AllocationObjectAllocator; } - bool HeapFlagsFulfillResourceHeapTier(D3D12_HEAP_FLAGS flags) const; - UINT StandardHeapTypeToMemorySegmentGroup(D3D12_HEAP_TYPE heapType) const; - UINT HeapPropertiesToMemorySegmentGroup(const D3D12_HEAP_PROPERTIES& heapProps) const; - UINT64 GetMemoryCapacity(UINT memorySegmentGroup) const; - - HRESULT CreateResource( - const ALLOCATION_DESC* pAllocDesc, - const D3D12_RESOURCE_DESC* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource); - -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - HRESULT CreateResource2( - const ALLOCATION_DESC* pAllocDesc, - const D3D12_RESOURCE_DESC1* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource); -#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ - - HRESULT AllocateMemory( - const ALLOCATION_DESC* pAllocDesc, - const D3D12_RESOURCE_ALLOCATION_INFO* pAllocInfo, - Allocation** ppAllocation); - - HRESULT CreateAliasingResource( - Allocation* pAllocation, - UINT64 AllocationLocalOffset, - const D3D12_RESOURCE_DESC* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - REFIID riidResource, - void** ppvResource); - - // Unregisters allocation from the collection of dedicated allocations. - // Allocation object must be deleted externally afterwards. - void FreeCommittedMemory(Allocation* allocation); - // Unregisters allocation from the collection of placed allocations. - // Allocation object must be deleted externally afterwards. - void FreePlacedMemory(Allocation* allocation); - // Unregisters allocation from the collection of dedicated allocations and destroys associated heap. - // Allocation object must be deleted externally afterwards. - void FreeHeapMemory(Allocation* allocation); - - void SetCurrentFrameIndex(UINT frameIndex); - - UINT GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); } - - void CalculateStatistics(TotalStatistics& outStats); - - void GetBudget(Budget* outLocalBudget, Budget* outNonLocalBudget); - void GetBudgetForHeapType(Budget& outBudget, D3D12_HEAP_TYPE heapType); - - void BuildStatsString(WCHAR** ppStatsString, BOOL DetailedMap); - - void FreeStatsString(WCHAR* pStatsString); - -private: - friend class Allocator; - friend class Pool; - - /* - Heuristics that decides whether a resource should better be placed in its own, - dedicated allocation (committed resource rather than placed resource). - */ - template - static bool PrefersCommittedAllocation(const D3D12_RESOURCE_DESC_T& resourceDesc); - - const bool m_UseMutex; - const bool m_AlwaysCommitted; - ID3D12Device* m_Device; // AddRef -#ifdef __ID3D12Device4_INTERFACE_DEFINED__ - ID3D12Device4* m_Device4 = NULL; // AddRef, optional -#endif -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - ID3D12Device8* m_Device8 = NULL; // AddRef, optional -#endif - IDXGIAdapter* m_Adapter; // AddRef -#if D3D12MA_DXGI_1_4 - IDXGIAdapter3* m_Adapter3 = NULL; // AddRef, optional -#endif - UINT64 m_PreferredBlockSize; - ALLOCATION_CALLBACKS m_AllocationCallbacks; - D3D12MA_ATOMIC_UINT32 m_CurrentFrameIndex; - DXGI_ADAPTER_DESC m_AdapterDesc; - D3D12_FEATURE_DATA_D3D12_OPTIONS m_D3D12Options; - D3D12_FEATURE_DATA_ARCHITECTURE m_D3D12Architecture; - AllocationObjectAllocator m_AllocationObjectAllocator; - - using PoolList = IntrusiveLinkedList; - PoolList m_Pools[HEAP_TYPE_COUNT]; - D3D12MA_RW_MUTEX m_PoolsMutex[HEAP_TYPE_COUNT]; - - // Default pools. - BlockVector* m_BlockVectors[DEFAULT_POOL_MAX_COUNT]; - - CommittedAllocationList m_CommittedAllocations[STANDARD_HEAP_TYPE_COUNT]; - - // Allocates and registers new committed resource with implicit heap, as dedicated allocation. - // Creates and returns Allocation object and optionally D3D12 resource. - HRESULT AllocateCommittedResource( - const CommittedAllocationParameters& committedAllocParams, - UINT64 resourceSize, bool withinBudget, - const D3D12_RESOURCE_DESC* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, REFIID riidResource, void** ppvResource); - -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - HRESULT AllocateCommittedResource2( - const CommittedAllocationParameters& committedAllocParams, - UINT64 resourceSize, bool withinBudget, - const D3D12_RESOURCE_DESC1* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, REFIID riidResource, void** ppvResource); -#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ - - // Allocates and registers new heap without any resources placed in it, as dedicated allocation. - // Creates and returns Allocation object. - HRESULT AllocateHeap( - const CommittedAllocationParameters& committedAllocParams, - const D3D12_RESOURCE_ALLOCATION_INFO& allocInfo, bool withinBudget, - Allocation** ppAllocation); - - template - HRESULT CalcAllocationParams(const ALLOCATION_DESC& allocDesc, UINT64 allocSize, - const D3D12_RESOURCE_DESC_T* resDesc, // Optional - BlockVector*& outBlockVector, CommittedAllocationParameters& outCommittedAllocationParams, bool& outPreferCommitted); - - /* - If SupportsResourceHeapTier2(): - 0: D3D12_HEAP_TYPE_DEFAULT - 1: D3D12_HEAP_TYPE_UPLOAD - 2: D3D12_HEAP_TYPE_READBACK - else: - 0: D3D12_HEAP_TYPE_DEFAULT + buffer - 1: D3D12_HEAP_TYPE_DEFAULT + texture - 2: D3D12_HEAP_TYPE_DEFAULT + texture RT or DS - 3: D3D12_HEAP_TYPE_UPLOAD + buffer - 4: D3D12_HEAP_TYPE_UPLOAD + texture - 5: D3D12_HEAP_TYPE_UPLOAD + texture RT or DS - 6: D3D12_HEAP_TYPE_READBACK + buffer - 7: D3D12_HEAP_TYPE_READBACK + texture - 8: D3D12_HEAP_TYPE_READBACK + texture RT or DS - */ - UINT CalcDefaultPoolCount() const; - // Returns UINT32_MAX if index cannot be calculcated. - UINT CalcDefaultPoolIndex(const ALLOCATION_DESC& allocDesc, ResourceClass resourceClass) const; - void CalcDefaultPoolParams(D3D12_HEAP_TYPE& outHeapType, D3D12_HEAP_FLAGS& outHeapFlags, UINT index) const; - - // Registers Pool object in m_Pools. - void RegisterPool(Pool* pool, D3D12_HEAP_TYPE heapType); - // Unregisters Pool object from m_Pools. - void UnregisterPool(Pool* pool, D3D12_HEAP_TYPE heapType); - - HRESULT UpdateD3D12Budget(); - - D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfoNative(const D3D12_RESOURCE_DESC& resourceDesc) const; -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ - D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfoNative(const D3D12_RESOURCE_DESC1& resourceDesc) const; -#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ - - template - D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfo(D3D12_RESOURCE_DESC_T& inOutResourceDesc) const; - - bool NewAllocationWithinBudget(D3D12_HEAP_TYPE heapType, UINT64 size); - - // Writes object { } with data of given budget. - static void WriteBudgetToJson(JsonWriter& json, const Budget& budget); -}; - -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata implementation - -BlockMetadata::BlockMetadata(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) : - m_Size(0), - m_IsVirtual(isVirtual), - m_pAllocationCallbacks(allocationCallbacks) -{ - D3D12MA_ASSERT(allocationCallbacks); -} - -void BlockMetadata::PrintDetailedMap_Begin(JsonWriter& json, - UINT64 unusedBytes, size_t allocationCount, size_t unusedRangeCount) const -{ - json.BeginObject(); - - json.WriteString(L"TotalBytes"); - json.WriteNumber(GetSize()); - - json.WriteString(L"UnusedBytes"); - json.WriteNumber(unusedBytes); - - json.WriteString(L"Allocations"); - json.WriteNumber(allocationCount); - - json.WriteString(L"UnusedRanges"); - json.WriteNumber(unusedRangeCount); - - json.WriteString(L"Suballocations"); - json.BeginArray(); -} - -void BlockMetadata::PrintDetailedMap_Allocation(JsonWriter& json, - UINT64 offset, UINT64 size, void* userData) const -{ - json.BeginObject(true); - - json.WriteString(L"Offset"); - json.WriteNumber(offset); - - if (IsVirtual()) - { - json.WriteString(L"Type"); - json.WriteString(L"ALLOCATION"); - json.WriteString(L"Size"); - json.WriteNumber(size); - if (userData) - { - json.WriteString(L"UserData"); - json.WriteNumber((uintptr_t)userData); - } - } - else - { - const Allocation* const alloc = (const Allocation*)userData; - D3D12MA_ASSERT(alloc); - json.AddAllocationToObject(*alloc); - } - json.EndObject(); -} - -void BlockMetadata::PrintDetailedMap_UnusedRange(JsonWriter& json, - UINT64 offset, UINT64 size) const -{ - json.BeginObject(true); - - json.WriteString(L"Offset"); - json.WriteNumber(offset); - - json.WriteString(L"Type"); - json.WriteString(L"FREE"); - - json.WriteString(L"Size"); - json.WriteNumber(size); - - json.EndObject(); -} - -void BlockMetadata::PrintDetailedMap_End(JsonWriter& json) const -{ - json.EndArray(); - json.EndObject(); -} - -#if 0 -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata_Generic implementation - -BlockMetadata_Generic::BlockMetadata_Generic(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) : - BlockMetadata(allocationCallbacks, isVirtual), - m_FreeCount(0), - m_SumFreeSize(0), - m_Suballocations(*allocationCallbacks), - m_FreeSuballocationsBySize(*allocationCallbacks) -{ - D3D12MA_ASSERT(allocationCallbacks); -} - -BlockMetadata_Generic::~BlockMetadata_Generic() -{ -} - -void BlockMetadata_Generic::Init(UINT64 size) -{ - BlockMetadata::Init(size); - m_ZeroInitializedRange.Reset(size); - - m_FreeCount = 1; - m_SumFreeSize = size; - - Suballocation suballoc = {}; - suballoc.offset = 0; - suballoc.size = size; - suballoc.type = SUBALLOCATION_TYPE_FREE; - suballoc.userData = NULL; - - D3D12MA_ASSERT(size > MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); - m_Suballocations.push_back(suballoc); - SuballocationList::iterator suballocItem = m_Suballocations.end(); - --suballocItem; - m_FreeSuballocationsBySize.push_back(suballocItem); -} - -bool BlockMetadata_Generic::Validate() const -{ - D3D12MA_VALIDATE(!m_Suballocations.empty()); - - // Expected offset of new suballocation as calculated from previous ones. - UINT64 calculatedOffset = 0; - // Expected number of free suballocations as calculated from traversing their list. - UINT calculatedFreeCount = 0; - // Expected sum size of free suballocations as calculated from traversing their list. - UINT64 calculatedSumFreeSize = 0; - // Expected number of free suballocations that should be registered in - // m_FreeSuballocationsBySize calculated from traversing their list. - size_t freeSuballocationsToRegister = 0; - // True if previous visited suballocation was free. - bool prevFree = false; - - for(const auto& subAlloc : m_Suballocations) - { - // Actual offset of this suballocation doesn't match expected one. - D3D12MA_VALIDATE(subAlloc.offset == calculatedOffset); - - const bool currFree = (subAlloc.type == SUBALLOCATION_TYPE_FREE); - // Two adjacent free suballocations are invalid. They should be merged. - D3D12MA_VALIDATE(!prevFree || !currFree); - - const Allocation* const alloc = (Allocation*)subAlloc.userData; - if(!IsVirtual()) - { - D3D12MA_VALIDATE(currFree == (alloc == NULL)); - } - - if(currFree) - { - calculatedSumFreeSize += subAlloc.size; - ++calculatedFreeCount; - if(subAlloc.size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) - { - ++freeSuballocationsToRegister; - } - - // Margin required between allocations - every free space must be at least that large. - D3D12MA_VALIDATE(subAlloc.size >= GetDebugMargin()); - } - else - { - if(!IsVirtual()) - { - D3D12MA_VALIDATE(alloc->GetOffset() == subAlloc.offset); - D3D12MA_VALIDATE(alloc->GetSize() == subAlloc.size); - } - - // Margin required between allocations - previous allocation must be free. - D3D12MA_VALIDATE(GetDebugMargin() == 0 || prevFree); - } - - calculatedOffset += subAlloc.size; - prevFree = currFree; - } - - // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't - // match expected one. - D3D12MA_VALIDATE(m_FreeSuballocationsBySize.size() == freeSuballocationsToRegister); - - UINT64 lastSize = 0; - for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i) - { - SuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i]; - - // Only free suballocations can be registered in m_FreeSuballocationsBySize. - D3D12MA_VALIDATE(suballocItem->type == SUBALLOCATION_TYPE_FREE); - // They must be sorted by size ascending. - D3D12MA_VALIDATE(suballocItem->size >= lastSize); - - lastSize = suballocItem->size; - } - - // Check if totals match calculacted values. - D3D12MA_VALIDATE(ValidateFreeSuballocationList()); - D3D12MA_VALIDATE(calculatedOffset == GetSize()); - D3D12MA_VALIDATE(calculatedSumFreeSize == m_SumFreeSize); - D3D12MA_VALIDATE(calculatedFreeCount == m_FreeCount); - - return true; -} - -bool BlockMetadata_Generic::IsEmpty() const -{ - return (m_Suballocations.size() == 1) && (m_FreeCount == 1); -} - -void BlockMetadata_Generic::GetAllocationInfo(AllocHandle allocHandle, VIRTUAL_ALLOCATION_INFO& outInfo) const -{ - Suballocation& suballoc = *FindAtOffset((UINT64)allocHandle - 1).dropConst(); - outInfo.Offset = suballoc.offset; - outInfo.Size = suballoc.size; - outInfo.pUserData = suballoc.userData; -} - -bool BlockMetadata_Generic::CreateAllocationRequest( - UINT64 allocSize, - UINT64 allocAlignment, - bool upperAddress, - AllocationRequest* pAllocationRequest) -{ - D3D12MA_ASSERT(allocSize > 0); - D3D12MA_ASSERT(!upperAddress && "ALLOCATION_FLAG_UPPER_ADDRESS can be used only with linear algorithm."); - D3D12MA_ASSERT(pAllocationRequest != NULL); - D3D12MA_HEAVY_ASSERT(Validate()); - - // There is not enough total free space in this block to fullfill the request: Early return. - if(m_SumFreeSize < allocSize + GetDebugMargin()) - { - return false; - } - - // New algorithm, efficiently searching freeSuballocationsBySize. - const size_t freeSuballocCount = m_FreeSuballocationsBySize.size(); - if(freeSuballocCount > 0) - { - // Find first free suballocation with size not less than allocSize + GetDebugMargin(). - SuballocationList::iterator* const it = BinaryFindFirstNotLess( - m_FreeSuballocationsBySize.data(), - m_FreeSuballocationsBySize.data() + freeSuballocCount, - allocSize + GetDebugMargin(), - SuballocationItemSizeLess()); - size_t index = it - m_FreeSuballocationsBySize.data(); - for(; index < freeSuballocCount; ++index) - { - if(CheckAllocation( - allocSize, - allocAlignment, - m_FreeSuballocationsBySize[index], - &pAllocationRequest->allocHandle, - &pAllocationRequest->sumFreeSize, - &pAllocationRequest->sumItemSize, - &pAllocationRequest->zeroInitialized)) - { - pAllocationRequest->item = m_FreeSuballocationsBySize[index]; - return true; - } - } - } - - return false; -} - -void BlockMetadata_Generic::Alloc( - const AllocationRequest& request, - UINT64 allocSize, - void* userData) -{ - D3D12MA_ASSERT(request.item != m_Suballocations.end()); - Suballocation& suballoc = *request.item; - // Given suballocation is a free block. - D3D12MA_ASSERT(suballoc.type == SUBALLOCATION_TYPE_FREE); - // Given offset is inside this suballocation. - UINT64 offset = (UINT64)request.allocHandle - 1; - D3D12MA_ASSERT(offset >= suballoc.offset); - const UINT64 paddingBegin = offset - suballoc.offset; - D3D12MA_ASSERT(suballoc.size >= paddingBegin + allocSize); - const UINT64 paddingEnd = suballoc.size - paddingBegin - allocSize; - - // Unregister this free suballocation from m_FreeSuballocationsBySize and update - // it to become used. - UnregisterFreeSuballocation(request.item); - - suballoc.offset = offset; - suballoc.size = allocSize; - suballoc.type = SUBALLOCATION_TYPE_ALLOCATION; - suballoc.userData = userData; - - // If there are any free bytes remaining at the end, insert new free suballocation after current one. - if(paddingEnd) - { - Suballocation paddingSuballoc = {}; - paddingSuballoc.offset = offset + allocSize; - paddingSuballoc.size = paddingEnd; - paddingSuballoc.type = SUBALLOCATION_TYPE_FREE; - SuballocationList::iterator next = request.item; - ++next; - const SuballocationList::iterator paddingEndItem = - m_Suballocations.insert(next, paddingSuballoc); - RegisterFreeSuballocation(paddingEndItem); - } - - // If there are any free bytes remaining at the beginning, insert new free suballocation before current one. - if(paddingBegin) - { - Suballocation paddingSuballoc = {}; - paddingSuballoc.offset = offset - paddingBegin; - paddingSuballoc.size = paddingBegin; - paddingSuballoc.type = SUBALLOCATION_TYPE_FREE; - const SuballocationList::iterator paddingBeginItem = - m_Suballocations.insert(request.item, paddingSuballoc); - RegisterFreeSuballocation(paddingBeginItem); - } - - // Update totals. - m_FreeCount = m_FreeCount - 1; - if(paddingBegin > 0) - { - ++m_FreeCount; - } - if(paddingEnd > 0) - { - ++m_FreeCount; - } - m_SumFreeSize -= allocSize; - - m_ZeroInitializedRange.MarkRangeAsUsed(offset, offset + allocSize); -} - -void BlockMetadata_Generic::Free(AllocHandle allocHandle) -{ - FreeSuballocation(FindAtOffset((UINT64)allocHandle - 1).dropConst()); -} - -void BlockMetadata_Generic::Clear() -{ - m_FreeCount = 1; - m_SumFreeSize = GetSize(); - - m_Suballocations.clear(); - Suballocation suballoc = {}; - suballoc.offset = 0; - suballoc.size = GetSize(); - suballoc.type = SUBALLOCATION_TYPE_FREE; - m_Suballocations.push_back(suballoc); - - m_FreeSuballocationsBySize.clear(); - m_FreeSuballocationsBySize.push_back(m_Suballocations.begin()); -} - -SuballocationList::const_iterator BlockMetadata_Generic::FindAtOffset(UINT64 offset) const -{ - const UINT64 last = m_Suballocations.crbegin()->offset; - if (last == offset) - return m_Suballocations.crbegin(); - const UINT64 first = m_Suballocations.cbegin()->offset; - if (first == offset) - return m_Suballocations.cbegin(); - - const size_t suballocCount = m_Suballocations.size(); - const UINT64 step = (last - first + m_Suballocations.cbegin()->size) / suballocCount; - auto findSuballocation = [&](auto begin, auto end) -> SuballocationList::const_iterator - { - for (auto suballocItem = begin; - suballocItem != end; - ++suballocItem) - { - const Suballocation& suballoc = *suballocItem; - if (suballoc.offset == offset) - return suballocItem; - } - D3D12MA_ASSERT(false && "Not found!"); - return m_Suballocations.end(); - }; - // If requested offset is closer to the end of range, search from the end - if ((offset - first) > suballocCount * step / 2) - { - return findSuballocation(m_Suballocations.crbegin(), m_Suballocations.crend()); - } - return findSuballocation(m_Suballocations.cbegin(), m_Suballocations.cend()); -} - -bool BlockMetadata_Generic::ValidateFreeSuballocationList() const -{ - UINT64 lastSize = 0; - for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i) - { - const SuballocationList::iterator it = m_FreeSuballocationsBySize[i]; - - D3D12MA_VALIDATE(it->type == SUBALLOCATION_TYPE_FREE); - D3D12MA_VALIDATE(it->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER); - D3D12MA_VALIDATE(it->size >= lastSize); - lastSize = it->size; - } - return true; -} - -bool BlockMetadata_Generic::CheckAllocation( - UINT64 allocSize, - UINT64 allocAlignment, - SuballocationList::const_iterator suballocItem, - AllocHandle* pAllocHandle, - UINT64* pSumFreeSize, - UINT64* pSumItemSize, - BOOL *pZeroInitialized) const -{ - D3D12MA_ASSERT(allocSize > 0); - D3D12MA_ASSERT(suballocItem != m_Suballocations.cend()); - D3D12MA_ASSERT(pAllocHandle != NULL && pZeroInitialized != NULL); - - *pSumFreeSize = 0; - *pSumItemSize = 0; - *pZeroInitialized = FALSE; - - const Suballocation& suballoc = *suballocItem; - D3D12MA_ASSERT(suballoc.type == SUBALLOCATION_TYPE_FREE); - - *pSumFreeSize = suballoc.size; - - // Size of this suballocation is too small for this request: Early return. - if(suballoc.size < allocSize) - { - return false; - } - - // Start from offset equal to beginning of this suballocation and debug margin of previous allocation if present. - UINT64 offset = suballoc.offset + (suballocItem == m_Suballocations.cbegin() ? 0 : GetDebugMargin()); - - // Apply alignment. - offset = AlignUp(offset, allocAlignment); - - // Calculate padding at the beginning based on current offset. - const UINT64 paddingBegin = offset - suballoc.offset; - - // Fail if requested size plus margin after is bigger than size of this suballocation. - if(paddingBegin + allocSize + GetDebugMargin() > suballoc.size) - { - return false; - } - - // All tests passed: Success. Offset is already filled. - *pZeroInitialized = m_ZeroInitializedRange.IsRangeZeroInitialized(offset, offset + allocSize); - *pAllocHandle = (AllocHandle)(offset + 1); - return true; -} - -void BlockMetadata_Generic::MergeFreeWithNext(SuballocationList::iterator item) -{ - D3D12MA_ASSERT(item != m_Suballocations.end()); - D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); - - SuballocationList::iterator nextItem = item; - ++nextItem; - D3D12MA_ASSERT(nextItem != m_Suballocations.end()); - D3D12MA_ASSERT(nextItem->type == SUBALLOCATION_TYPE_FREE); - - item->size += nextItem->size; - --m_FreeCount; - m_Suballocations.erase(nextItem); -} - -SuballocationList::iterator BlockMetadata_Generic::FreeSuballocation(SuballocationList::iterator suballocItem) -{ - // Change this suballocation to be marked as free. - Suballocation& suballoc = *suballocItem; - suballoc.type = SUBALLOCATION_TYPE_FREE; - suballoc.userData = NULL; - - // Update totals. - ++m_FreeCount; - m_SumFreeSize += suballoc.size; - - // Merge with previous and/or next suballocation if it's also free. - bool mergeWithNext = false; - bool mergeWithPrev = false; - - SuballocationList::iterator nextItem = suballocItem; - ++nextItem; - if((nextItem != m_Suballocations.end()) && (nextItem->type == SUBALLOCATION_TYPE_FREE)) - { - mergeWithNext = true; - } - - SuballocationList::iterator prevItem = suballocItem; - if(suballocItem != m_Suballocations.begin()) - { - --prevItem; - if(prevItem->type == SUBALLOCATION_TYPE_FREE) - { - mergeWithPrev = true; - } - } - - if(mergeWithNext) - { - UnregisterFreeSuballocation(nextItem); - MergeFreeWithNext(suballocItem); - } - - if(mergeWithPrev) - { - UnregisterFreeSuballocation(prevItem); - MergeFreeWithNext(prevItem); - RegisterFreeSuballocation(prevItem); - return prevItem; - } - else - { - RegisterFreeSuballocation(suballocItem); - return suballocItem; - } -} - -void BlockMetadata_Generic::RegisterFreeSuballocation(SuballocationList::iterator item) -{ - D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); - D3D12MA_ASSERT(item->size > 0); - - // You may want to enable this validation at the beginning or at the end of - // this function, depending on what do you want to check. - D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); - - if(item->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) - { - if(m_FreeSuballocationsBySize.empty()) - { - m_FreeSuballocationsBySize.push_back(item); - } - else - { - m_FreeSuballocationsBySize.InsertSorted(item, SuballocationItemSizeLess()); - } - } - - //D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); -} - -void BlockMetadata_Generic::UnregisterFreeSuballocation(SuballocationList::iterator item) -{ - D3D12MA_ASSERT(item->type == SUBALLOCATION_TYPE_FREE); - D3D12MA_ASSERT(item->size > 0); - - // You may want to enable this validation at the beginning or at the end of - // this function, depending on what do you want to check. - D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); - - if(item->size >= MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) - { - SuballocationList::iterator* const it = BinaryFindFirstNotLess( - m_FreeSuballocationsBySize.data(), - m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(), - item, - SuballocationItemSizeLess()); - for(size_t index = it - m_FreeSuballocationsBySize.data(); - index < m_FreeSuballocationsBySize.size(); - ++index) - { - if(m_FreeSuballocationsBySize[index] == item) - { - m_FreeSuballocationsBySize.remove(index); - return; - } - D3D12MA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found."); - } - D3D12MA_ASSERT(0 && "Not found."); - } - - //D3D12MA_HEAVY_ASSERT(ValidateFreeSuballocationList()); -} - -void BlockMetadata_Generic::SetAllocationUserData(AllocHandle allocHandle, void* userData) -{ - Suballocation& suballoc = *FindAtOffset((UINT64)allocHandle - 1).dropConst(); - suballoc.userData = userData; -} - -void BlockMetadata_Generic::AddStatistics(Statistics& inoutStats) const -{ - inoutStats.BlockCount++; - inoutStats.AllocationCount += (UINT)m_Suballocations.size() - m_FreeCount; - inoutStats.BlockBytes += GetSize(); - inoutStats.AllocationBytes += GetSize() - m_SumFreeSize; -} - -void BlockMetadata_Generic::AddDetailedStatistics(DetailedStatistics& inoutStats) const -{ - inoutStats.Stats.BlockCount++; - inoutStats.Stats.BlockBytes += GetSize(); - - for(const auto& suballoc : m_Suballocations) - { - if(suballoc.type == SUBALLOCATION_TYPE_FREE) - AddDetailedStatisticsUnusedRange(inoutStats, suballoc.size); - else - AddDetailedStatisticsAllocation(inoutStats, suballoc.size); - } -} - -void BlockMetadata_Generic::WriteAllocationInfoToJson(JsonWriter& json) const -{ - PrintDetailedMap_Begin(json, GetSumFreeSize(), GetAllocationCount(), m_FreeCount); - for(const auto& suballoc : m_Suballocations) - { - if (suballoc.type == SUBALLOCATION_TYPE_FREE) - PrintDetailedMap_UnusedRange(json, suballoc.offset, suballoc.size); - else - PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.size, suballoc.userData); - } - PrintDetailedMap_End(json); -} -#endif // #if 0 - -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata_Linear implementation - +#ifndef _D3D12MA_BLOCK_METADATA_LINEAR_FUNCTIONS BlockMetadata_Linear::BlockMetadata_Linear(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) : BlockMetadata(allocationCallbacks, isVirtual), m_SumFreeSize(0), @@ -5410,29 +4869,127 @@ bool BlockMetadata_Linear::CreateAllocationRequest_UpperAddress( } return false; } +#endif // _D3D12MA_BLOCK_METADATA_LINEAR_FUNCTIONS +#endif // _D3D12MA_BLOCK_METADATA_LINEAR -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockMetadata_TLSF implementation +#ifndef _D3D12MA_BLOCK_METADATA_TLSF +class BlockMetadata_TLSF : public BlockMetadata +{ +public: + BlockMetadata_TLSF(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual); + virtual ~BlockMetadata_TLSF(); + size_t GetAllocationCount() const override { return m_AllocCount; } + UINT64 GetSumFreeSize() const override { return m_BlocksFreeSize + m_NullBlock->size; } + bool IsEmpty() const override { return m_NullBlock->offset == 0; } + UINT64 GetAllocationOffset(AllocHandle allocHandle) const override { return ((Block*)allocHandle)->offset; }; + + void Init(UINT64 size) override; + bool Validate() const override; + void GetAllocationInfo(AllocHandle allocHandle, VIRTUAL_ALLOCATION_INFO& outInfo) const override; + + bool CreateAllocationRequest( + UINT64 allocSize, + UINT64 allocAlignment, + bool upperAddress, + AllocationRequest* pAllocationRequest) override; + + void Alloc( + const AllocationRequest& request, + UINT64 allocSize, + void* userData) override; + + void Free(AllocHandle allocHandle) override; + void Clear() override; + + void SetAllocationUserData(AllocHandle allocHandle, void* userData) override; + + void AddStatistics(Statistics& inoutStats) const override; + void AddDetailedStatistics(DetailedStatistics& inoutStats) const override; + void WriteAllocationInfoToJson(JsonWriter& json) const override; + +private: + // According to original paper it should be preferable 4 or 5: + // M. Masmano, I. Ripoll, A. Crespo, and J. Real "TLSF: a New Dynamic Memory Allocator for Real-Time Systems" + // http://www.gii.upv.es/tlsf/files/ecrts04_tlsf.pdf + static const UINT8 SECOND_LEVEL_INDEX = 5; + static const UINT16 SMALL_BUFFER_SIZE = 256; + static const UINT INITIAL_BLOCK_ALLOC_COUNT = 16; + static const UINT8 MEMORY_CLASS_SHIFT = 7; + static const UINT8 MAX_MEMORY_CLASSES = 65 - MEMORY_CLASS_SHIFT; + + class Block + { + public: + UINT64 offset; + UINT64 size; + Block* prevPhysical; + Block* nextPhysical; + + void MarkFree() { prevFree = NULL; } + void MarkTaken() { prevFree = this; } + bool IsFree() const { return prevFree != this; } + void*& UserData() { D3D12MA_HEAVY_ASSERT(!IsFree()); return userData; } + Block*& PrevFree() { return prevFree; } + Block*& NextFree() { D3D12MA_HEAVY_ASSERT(IsFree()); return nextFree; } + + private: + Block* prevFree; // Address of the same block here indicates that block is taken + union + { + Block* nextFree; + void* userData; + }; + }; + + size_t m_AllocCount = 0; + // Total number of free blocks besides null block + size_t m_BlocksFreeCount = 0; + // Total size of free blocks excluding null block + UINT64 m_BlocksFreeSize = 0; + UINT32 m_IsFreeBitmap = 0; + UINT8 m_MemoryClasses = 0; + UINT32 m_InnerIsFreeBitmap[MAX_MEMORY_CLASSES]; + UINT32 m_ListsCount = 0; + /* + * 0: 0-3 lists for small buffers + * 1+: 0-(2^SLI-1) lists for normal buffers + */ + Block** m_FreeList = NULL; + PoolAllocator m_BlockAllocator; + Block* m_NullBlock = NULL; + + UINT8 SizeToMemoryClass(UINT64 size) const; + UINT16 SizeToSecondIndex(UINT64 size, UINT8 memoryClass) const; + UINT32 GetListIndex(UINT8 memoryClass, UINT16 secondIndex) const; + UINT32 GetListIndex(UINT64 size) const; + + void RemoveFreeBlock(Block* block); + void InsertFreeBlock(Block* block); + void MergeBlock(Block* block, Block* prev); + + Block* FindFreeBlock(UINT64 size, UINT32& listIndex) const; + bool CheckBlock( + Block& block, + UINT32 listIndex, + UINT64 allocSize, + UINT64 allocAlignment, + AllocationRequest* pAllocationRequest); + + D3D12MA_CLASS_NO_COPY(BlockMetadata_TLSF) +}; + +#ifndef _D3D12MA_BLOCK_METADATA_TLSF_FUNCTIONS BlockMetadata_TLSF::BlockMetadata_TLSF(const ALLOCATION_CALLBACKS* allocationCallbacks, bool isVirtual) : BlockMetadata(allocationCallbacks, isVirtual), - m_AllocCount(0), - m_BlocksFreeCount(0), - m_BlocksFreeSize(0), - m_IsFreeBitmap(0), - m_MemoryClasses(0), - m_ListsCount(0), - m_FreeList(NULL), - m_BlockAllocator(*allocationCallbacks, INITIAL_BLOCK_ALLOC_COUNT), - m_NullBlock(NULL) + m_BlockAllocator(*allocationCallbacks, INITIAL_BLOCK_ALLOC_COUNT) { D3D12MA_ASSERT(allocationCallbacks); } BlockMetadata_TLSF::~BlockMetadata_TLSF() { - if (m_FreeList) - D3D12MA_DELETE_ARRAY(*GetAllocs(), m_FreeList, m_ListsCount); + D3D12MA_DELETE_ARRAY(*GetAllocs(), m_FreeList, m_ListsCount); } void BlockMetadata_TLSF::Init(UINT64 size) @@ -5566,7 +5123,7 @@ bool BlockMetadata_TLSF::CreateAllocationRequest( { D3D12MA_ASSERT(allocSize > 0 && "Cannot allocate empty block!"); D3D12MA_ASSERT(!upperAddress && "ALLOCATION_FLAG_UPPER_ADDRESS can be used only with linear algorithm."); - D3D12MA_ASSERT(pAllocationRequest != NULL); + D3D12MA_ASSERT(pAllocationRequest != NULL); D3D12MA_HEAVY_ASSERT(Validate()); allocSize += GetDebugMargin(); @@ -6018,841 +5575,328 @@ bool BlockMetadata_TLSF::CheckBlock( return true; } +#endif // _D3D12MA_BLOCK_METADATA_TLSF_FUNCTIONS +#endif // _D3D12MA_BLOCK_METADATA_TLSF -//////////////////////////////////////////////////////////////////////////////// -// Private class NormalBlock implementation - -NormalBlock::NormalBlock( - AllocatorPimpl* allocator, - BlockVector* blockVector, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 size, - UINT id) : - MemoryBlock(allocator, heapProps, heapFlags, size, id), - m_pMetadata(NULL), - m_BlockVector(blockVector) +#ifndef _D3D12MA_MEMORY_BLOCK +/* +Represents a single block of device memory (heap). +Base class for inheritance. +Thread-safety: This class must be externally synchronized. +*/ +class MemoryBlock { -} +public: + // Creates the ID3D12Heap. + MemoryBlock( + AllocatorPimpl* allocator, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 size, + UINT id); + virtual ~MemoryBlock(); -NormalBlock::~NormalBlock() + const D3D12_HEAP_PROPERTIES& GetHeapProperties() const { return m_HeapProps; } + D3D12_HEAP_FLAGS GetHeapFlags() const { return m_HeapFlags; } + UINT64 GetSize() const { return m_Size; } + UINT GetId() const { return m_Id; } + ID3D12Heap* GetHeap() const { return m_Heap; } + +protected: + AllocatorPimpl* const m_Allocator; + const D3D12_HEAP_PROPERTIES m_HeapProps; + const D3D12_HEAP_FLAGS m_HeapFlags; + const UINT64 m_Size; + const UINT m_Id; + + HRESULT Init(ID3D12ProtectedResourceSession* pProtectedSession); + +private: + ID3D12Heap* m_Heap = NULL; + + D3D12MA_CLASS_NO_COPY(MemoryBlock) +}; +#endif // _D3D12MA_MEMORY_BLOCK + +#ifndef _D3D12MA_NORMAL_BLOCK +/* +Represents a single block of device memory (heap) with all the data about its +regions (aka suballocations, Allocation), assigned and free. +Thread-safety: This class must be externally synchronized. +*/ +class NormalBlock : public MemoryBlock { - if(m_pMetadata != NULL) +public: + BlockMetadata* m_pMetadata; + + NormalBlock( + AllocatorPimpl* allocator, + BlockVector* blockVector, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 size, + UINT id); + virtual ~NormalBlock(); + + BlockVector* GetBlockVector() const { return m_BlockVector; } + + // 'algorithm' should be one of the *_ALGORITHM_* flags in enums POOL_FLAGS or VIRTUAL_BLOCK_FLAGS + HRESULT Init(UINT32 algorithm, ID3D12ProtectedResourceSession* pProtectedSession); + + // Validates all data structures inside this object. If not valid, returns false. + bool Validate() const; + +private: + BlockVector* m_BlockVector; + + D3D12MA_CLASS_NO_COPY(NormalBlock) +}; +#endif // _D3D12MA_NORMAL_BLOCK + +#ifndef _D3D12MA_COMMITTED_ALLOCATION_LIST_ITEM_TRAITS +struct CommittedAllocationListItemTraits +{ + using ItemType = Allocation; + + static ItemType* GetPrev(const ItemType* item) { - // THIS IS THE MOST IMPORTANT ASSERT IN THE ENTIRE LIBRARY! - // Hitting it means you have some memory leak - unreleased Allocation objects. - D3D12MA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!"); - - D3D12MA_DELETE(m_Allocator->GetAllocs(), m_pMetadata); + D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); + return item->m_Committed.prev; } -} - -HRESULT NormalBlock::Init(UINT32 algorithm, ID3D12ProtectedResourceSession* pProtectedSession) -{ - HRESULT hr = MemoryBlock::Init(pProtectedSession); - if(FAILED(hr)) + static ItemType* GetNext(const ItemType* item) { - return hr; + D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); + return item->m_Committed.next; } + static ItemType*& AccessPrev(ItemType* item) + { + D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); + return item->m_Committed.prev; + } + static ItemType*& AccessNext(ItemType* item) + { + D3D12MA_ASSERT(item->m_PackedData.GetType() == Allocation::TYPE_COMMITTED || item->m_PackedData.GetType() == Allocation::TYPE_HEAP); + return item->m_Committed.next; + } +}; +#endif // _D3D12MA_COMMITTED_ALLOCATION_LIST_ITEM_TRAITS + +#ifndef _D3D12MA_COMMITTED_ALLOCATION_LIST +/* +Stores linked list of Allocation objects that are of TYPE_COMMITTED or TYPE_HEAP. +Thread-safe, synchronized internally. +*/ +class CommittedAllocationList +{ +public: + CommittedAllocationList() = default; + void Init(bool useMutex, D3D12_HEAP_TYPE heapType, PoolPimpl* pool); + ~CommittedAllocationList(); + + D3D12_HEAP_TYPE GetHeapType() const { return m_HeapType; } + PoolPimpl* GetPool() const { return m_Pool; } + UINT GetMemorySegmentGroup(AllocatorPimpl* allocator) const; - switch (algorithm) - { - case POOL_FLAG_ALGORITHM_LINEAR: - m_pMetadata = D3D12MA_NEW(m_Allocator->GetAllocs(), BlockMetadata_Linear)(&m_Allocator->GetAllocs(), false); - break; - default: - D3D12MA_ASSERT(0); - case 0: - m_pMetadata = D3D12MA_NEW(m_Allocator->GetAllocs(), BlockMetadata_TLSF)(&m_Allocator->GetAllocs(), false); - break; - } - m_pMetadata->Init(m_Size); + void AddStatistics(Statistics& inoutStats); + void AddDetailedStatistics(DetailedStatistics& inoutStats); + // Writes JSON array with the list of allocations. + void BuildStatsString(JsonWriter& json); - return hr; -} + void Register(Allocation* alloc); + void Unregister(Allocation* alloc); -bool NormalBlock::Validate() const +private: + using CommittedAllocationLinkedList = IntrusiveLinkedList; + + bool m_UseMutex = true; + D3D12_HEAP_TYPE m_HeapType = D3D12_HEAP_TYPE_CUSTOM; + PoolPimpl* m_Pool = NULL; + + D3D12MA_RW_MUTEX m_Mutex; + CommittedAllocationLinkedList m_AllocationList; +}; +#endif // _D3D12MA_COMMITTED_ALLOCATION_LIST + +#ifndef _D3D12M_COMMITTED_ALLOCATION_PARAMETERS +struct CommittedAllocationParameters { - D3D12MA_VALIDATE(GetHeap() && - m_pMetadata && - m_pMetadata->GetSize() != 0 && - m_pMetadata->GetSize() == GetSize()); - return m_pMetadata->Validate(); -} + CommittedAllocationList* m_List = NULL; + D3D12_HEAP_PROPERTIES m_HeapProperties = {}; + D3D12_HEAP_FLAGS m_HeapFlags = D3D12_HEAP_FLAG_NONE; + ID3D12ProtectedResourceSession* m_ProtectedSession = NULL; -//////////////////////////////////////////////////////////////////////////////// -// Private class MemoryBlock definition + bool IsValid() const { return m_List != NULL; } +}; +#endif // _D3D12M_COMMITTED_ALLOCATION_PARAMETERS -MemoryBlock::MemoryBlock( - AllocatorPimpl* allocator, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 size, - UINT id) : - m_Allocator(allocator), - m_HeapProps(heapProps), - m_HeapFlags(heapFlags), - m_Size(size), - m_Id(id) +#ifndef _D3D12MA_BLOCK_VECTOR +/* +Sequence of NormalBlock. Represents memory blocks allocated for a specific +heap type and possibly resource type (if only Tier 1 is supported). + +Synchronized internally with a mutex. +*/ +class BlockVector { -} + D3D12MA_CLASS_NO_COPY(BlockVector) +public: + BlockVector( + AllocatorPimpl* hAllocator, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 preferredBlockSize, + size_t minBlockCount, + size_t maxBlockCount, + bool explicitBlockSize, + UINT64 minAllocationAlignment, + UINT32 algorithm, + ID3D12ProtectedResourceSession* pProtectedSession); + ~BlockVector(); -MemoryBlock::~MemoryBlock() -{ - if(m_Heap) - { - m_Heap->Release(); - m_Allocator->m_Budget.RemoveBlock( - m_Allocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), m_Size); - } -} + const D3D12_HEAP_PROPERTIES& GetHeapProperties() const { return m_HeapProps; } + UINT64 GetPreferredBlockSize() const { return m_PreferredBlockSize; } -HRESULT MemoryBlock::Init(ID3D12ProtectedResourceSession* pProtectedSession) -{ - D3D12MA_ASSERT(m_Heap == NULL && m_Size > 0); + HRESULT CreateMinBlocks(); + bool IsEmpty(); - D3D12_HEAP_DESC heapDesc = {}; - heapDesc.SizeInBytes = m_Size; - heapDesc.Properties = m_HeapProps; - heapDesc.Alignment = HeapFlagsToAlignment(m_HeapFlags); - heapDesc.Flags = m_HeapFlags; + HRESULT Allocate( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + size_t allocationCount, + Allocation** pAllocations); - HRESULT hr; -#ifdef __ID3D12Device4_INTERFACE_DEFINED__ - ID3D12Device4* const device4 = m_Allocator->GetDevice4(); - if(device4) - hr = m_Allocator->GetDevice4()->CreateHeap1(&heapDesc, pProtectedSession, D3D12MA_IID_PPV_ARGS(&m_Heap)); - else -#endif - { - if(pProtectedSession == NULL) - hr = m_Allocator->GetDevice()->CreateHeap(&heapDesc, D3D12MA_IID_PPV_ARGS(&m_Heap)); - else - hr = E_NOINTERFACE; - } - - if(SUCCEEDED(hr)) - { - m_Allocator->m_Budget.AddBlock( - m_Allocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), m_Size); - } - return hr; -} + void Free( + Allocation* hAllocation); -//////////////////////////////////////////////////////////////////////////////// -// Private class CommittedAllocationList implementation - -CommittedAllocationList::CommittedAllocationList() -{ -} - -void CommittedAllocationList::Init(bool useMutex, D3D12_HEAP_TYPE heapType, PoolPimpl* pool) -{ - m_UseMutex = useMutex; - m_HeapType = heapType; - m_Pool = pool; -} - -CommittedAllocationList::~CommittedAllocationList() -{ - if(!m_AllocationList.IsEmpty()) - { - D3D12MA_ASSERT(0 && "Unfreed committed allocations found!"); - } -} - -UINT CommittedAllocationList::GetMemorySegmentGroup(AllocatorPimpl* allocator) const -{ - if(m_Pool) - return allocator->HeapPropertiesToMemorySegmentGroup(m_Pool->GetDesc().HeapProperties); - else - return allocator->StandardHeapTypeToMemorySegmentGroup(m_HeapType); -} - -void CommittedAllocationList::AddStatistics(Statistics& inoutStats) -{ - MutexLockRead lock(m_Mutex, m_UseMutex); - - for(Allocation* alloc = m_AllocationList.Front(); - alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) - { - const UINT64 size = alloc->GetSize(); - inoutStats.BlockCount++; - inoutStats.AllocationCount++; - inoutStats.BlockBytes += size; - inoutStats.AllocationBytes += size; - } -} - -void CommittedAllocationList::AddDetailedStatistics(DetailedStatistics& inoutStats) -{ - MutexLockRead lock(m_Mutex, m_UseMutex); - - for(Allocation* alloc = m_AllocationList.Front(); - alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) - { - const UINT64 size = alloc->GetSize(); - inoutStats.Stats.BlockCount++; - inoutStats.Stats.BlockBytes += size; - AddDetailedStatisticsAllocation(inoutStats, size); - } -} - -void CommittedAllocationList::BuildStatsString(JsonWriter& json) -{ - MutexLockRead lock(m_Mutex, m_UseMutex); - - json.BeginArray(); - for(Allocation* alloc = m_AllocationList.Front(); - alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) - { - json.BeginObject(true); - json.AddAllocationToObject(*alloc); - json.EndObject(); - } - json.EndArray(); -} - -void CommittedAllocationList::Register(Allocation* alloc) -{ - MutexLockWrite lock(m_Mutex, m_UseMutex); - m_AllocationList.PushBack(alloc); -} - -void CommittedAllocationList::Unregister(Allocation* alloc) -{ - MutexLockWrite lock(m_Mutex, m_UseMutex); - m_AllocationList.Remove(alloc); -} - -//////////////////////////////////////////////////////////////////////////////// -// Private class BlockVector implementation - -BlockVector::BlockVector( - AllocatorPimpl* hAllocator, - const D3D12_HEAP_PROPERTIES& heapProps, - D3D12_HEAP_FLAGS heapFlags, - UINT64 preferredBlockSize, - size_t minBlockCount, - size_t maxBlockCount, - bool explicitBlockSize, - UINT64 minAllocationAlignment, - UINT32 algorithm, - ID3D12ProtectedResourceSession* pProtectedSession) : - m_hAllocator(hAllocator), - m_HeapProps(heapProps), - m_HeapFlags(heapFlags), - m_PreferredBlockSize(preferredBlockSize), - m_MinBlockCount(minBlockCount), - m_MaxBlockCount(maxBlockCount), - m_ExplicitBlockSize(explicitBlockSize), - m_MinAllocationAlignment(minAllocationAlignment), - m_Algorithm(algorithm), - m_ProtectedSession(pProtectedSession), - m_HasEmptyBlock(false), - m_Blocks(hAllocator->GetAllocs()), - m_NextBlockId(0) -{ -} - -BlockVector::~BlockVector() -{ - for(size_t i = m_Blocks.size(); i--; ) - { - D3D12MA_DELETE(m_hAllocator->GetAllocs(), m_Blocks[i]); - } -} - -HRESULT BlockVector::CreateMinBlocks() -{ - for(size_t i = 0; i < m_MinBlockCount; ++i) - { - HRESULT hr = CreateBlock(m_PreferredBlockSize, NULL); - if(FAILED(hr)) - { - return hr; - } - } - return S_OK; -} - -bool BlockVector::IsEmpty() -{ - MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); - return m_Blocks.empty(); -} - -HRESULT BlockVector::Allocate( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - size_t allocationCount, - Allocation** pAllocations) -{ - size_t allocIndex; - HRESULT hr = S_OK; - - { - MutexLockWrite lock(m_Mutex, m_hAllocator->UseMutex()); - for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) - { - hr = AllocatePage( - size, - alignment, - allocDesc, - pAllocations + allocIndex); - if(FAILED(hr)) - { - break; - } - } - } - - if(FAILED(hr)) - { - // Free all already created allocations. - while(allocIndex--) - { - Free(pAllocations[allocIndex]); - } - ZeroMemory(pAllocations, sizeof(Allocation*) * allocationCount); - } - - return hr; -} - -HRESULT BlockVector::AllocatePage( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - Allocation** pAllocation) -{ - // Early reject: requested allocation size is larger that maximum block size for this block vector. - if(size + D3D12MA_DEBUG_MARGIN > m_PreferredBlockSize) - { - return E_OUTOFMEMORY; - } - - UINT64 freeMemory = UINT64_MAX; - if(IsHeapTypeStandard(m_HeapProps.Type)) - { - Budget budget = {}; - m_hAllocator->GetBudgetForHeapType(budget, m_HeapProps.Type); - freeMemory = (budget.UsageBytes < budget.BudgetBytes) ? (budget.BudgetBytes - budget.UsageBytes) : 0; - } - - const bool canCreateNewBlock = - ((allocDesc.Flags & ALLOCATION_FLAG_NEVER_ALLOCATE) == 0) && - (m_Blocks.size() < m_MaxBlockCount) && - // Even if we don't have to stay within budget with this allocation, when the - // budget would be exceeded, we don't want to allocate new blocks, but always - // create resources as committed. - freeMemory >= size; - - // 1. Search existing allocations - { - // Forward order in m_Blocks - prefer blocks with smallest amount of free space. - for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) - { - NormalBlock* const pCurrBlock = m_Blocks[blockIndex]; - D3D12MA_ASSERT(pCurrBlock); - HRESULT hr = AllocateFromBlock( - pCurrBlock, - size, - alignment, - allocDesc.Flags, - pAllocation); - if(SUCCEEDED(hr)) - { - return hr; - } - } - } - - // 2. Try to create new block. - if(canCreateNewBlock) - { - // Calculate optimal size for new block. - UINT64 newBlockSize = m_PreferredBlockSize; - UINT newBlockSizeShift = 0; - - if(!m_ExplicitBlockSize) - { - // Allocate 1/8, 1/4, 1/2 as first blocks. - const UINT64 maxExistingBlockSize = CalcMaxBlockSize(); - for(UINT i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i) - { - const UINT64 smallerNewBlockSize = newBlockSize / 2; - if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2) - { - newBlockSize = smallerNewBlockSize; - ++newBlockSizeShift; - } - else - { - break; - } - } - } - - size_t newBlockIndex = 0; - HRESULT hr = newBlockSize <= freeMemory ? - CreateBlock(newBlockSize, &newBlockIndex) : E_OUTOFMEMORY; - // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize. - if(!m_ExplicitBlockSize) - { - while(FAILED(hr) && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX) - { - const UINT64 smallerNewBlockSize = newBlockSize / 2; - if(smallerNewBlockSize >= size) - { - newBlockSize = smallerNewBlockSize; - ++newBlockSizeShift; - hr = newBlockSize <= freeMemory ? - CreateBlock(newBlockSize, &newBlockIndex) : E_OUTOFMEMORY; - } - else - { - break; - } - } - } - - if(SUCCEEDED(hr)) - { - NormalBlock* const pBlock = m_Blocks[newBlockIndex]; - D3D12MA_ASSERT(pBlock->m_pMetadata->GetSize() >= size); - - hr = AllocateFromBlock( - pBlock, - size, - alignment, - allocDesc.Flags, - pAllocation); - if(SUCCEEDED(hr)) - { - return hr; - } - else - { - // Allocation from new block failed, possibly due to D3D12MA_DEBUG_MARGIN or alignment. - return E_OUTOFMEMORY; - } - } - } - - return E_OUTOFMEMORY; -} - -void BlockVector::Free(Allocation* hAllocation) -{ - NormalBlock* pBlockToDelete = NULL; - - bool budgetExceeded = false; - if(IsHeapTypeStandard(m_HeapProps.Type)) - { - Budget budget = {}; - m_hAllocator->GetBudgetForHeapType(budget, m_HeapProps.Type); - budgetExceeded = budget.UsageBytes >= budget.BudgetBytes; - } - - // Scope for lock. - { - MutexLockWrite lock(m_Mutex, m_hAllocator->UseMutex()); - - NormalBlock* pBlock = hAllocation->m_Placed.block; - - pBlock->m_pMetadata->Free(hAllocation->GetAllocHandle()); - D3D12MA_HEAVY_ASSERT(pBlock->Validate()); - - const size_t blockCount = m_Blocks.size(); - // pBlock became empty after this deallocation. - if(pBlock->m_pMetadata->IsEmpty()) - { - // Already has empty Allocation. We don't want to have two, so delete this one. - if((m_HasEmptyBlock || budgetExceeded) && - blockCount > m_MinBlockCount) - { - pBlockToDelete = pBlock; - Remove(pBlock); - } - // We now have first empty block. - else - { - m_HasEmptyBlock = true; - } - } - // pBlock didn't become empty, but we have another empty block - find and free that one. - // (This is optional, heuristics.) - else if(m_HasEmptyBlock && blockCount > m_MinBlockCount) - { - NormalBlock* pLastBlock = m_Blocks.back(); - if(pLastBlock->m_pMetadata->IsEmpty()) - { - pBlockToDelete = pLastBlock; - m_Blocks.pop_back(); - m_HasEmptyBlock = false; - } - } - - IncrementallySortBlocks(); - } - - // Destruction of a free Allocation. Deferred until this point, outside of mutex - // lock, for performance reason. - if(pBlockToDelete != NULL) - { - D3D12MA_DELETE(m_hAllocator->GetAllocs(), pBlockToDelete); - } -} - -HRESULT BlockVector::CreateResource( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - const D3D12_RESOURCE_DESC& resourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource) -{ - HRESULT hr = Allocate(size, alignment, allocDesc, 1, ppAllocation); - if(SUCCEEDED(hr)) - { - ID3D12Resource* res = NULL; - hr = m_hAllocator->GetDevice()->CreatePlacedResource( - (*ppAllocation)->m_Placed.block->GetHeap(), - (*ppAllocation)->GetOffset(), - &resourceDesc, - InitialResourceState, - pOptimizedClearValue, - D3D12MA_IID_PPV_ARGS(&res)); - if(SUCCEEDED(hr)) - { - if(ppvResource != NULL) - { - hr = res->QueryInterface(riidResource, ppvResource); - } - if(SUCCEEDED(hr)) - { - (*ppAllocation)->SetResource(res, &resourceDesc); - } - else - { - res->Release(); - SAFE_RELEASE(*ppAllocation); - } - } - else - { - SAFE_RELEASE(*ppAllocation); - } - } - return hr; -} + HRESULT CreateResource( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + const D3D12_RESOURCE_DESC& resourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource); #ifdef __ID3D12Device8_INTERFACE_DEFINED__ -HRESULT BlockVector::CreateResource2( - UINT64 size, - UINT64 alignment, - const ALLOCATION_DESC& allocDesc, - const D3D12_RESOURCE_DESC1& resourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, - REFIID riidResource, - void** ppvResource) -{ - ID3D12Device8* const device8 = m_hAllocator->GetDevice8(); - if(device8 == NULL) - { - return E_NOINTERFACE; - } - - HRESULT hr = Allocate(size, alignment, allocDesc, 1, ppAllocation); - if(SUCCEEDED(hr)) - { - ID3D12Resource* res = NULL; - hr = device8->CreatePlacedResource1( - (*ppAllocation)->m_Placed.block->GetHeap(), - (*ppAllocation)->GetOffset(), - &resourceDesc, - InitialResourceState, - pOptimizedClearValue, - D3D12MA_IID_PPV_ARGS(&res)); - if(SUCCEEDED(hr)) - { - if(ppvResource != NULL) - { - hr = res->QueryInterface(riidResource, ppvResource); - } - if(SUCCEEDED(hr)) - { - (*ppAllocation)->SetResource(res, &resourceDesc); - } - else - { - res->Release(); - SAFE_RELEASE(*ppAllocation); - } - } - else - { - SAFE_RELEASE(*ppAllocation); - } - } - return hr; -} + HRESULT CreateResource2( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + const D3D12_RESOURCE_DESC1& resourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource); #endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ -UINT64 BlockVector::CalcSumBlockSize() const + void AddStatistics(Statistics& inoutStats); + void AddDetailedStatistics(DetailedStatistics& inoutStats); + + void WriteBlockInfoToJson(JsonWriter& json); + +private: + AllocatorPimpl* const m_hAllocator; + const D3D12_HEAP_PROPERTIES m_HeapProps; + const D3D12_HEAP_FLAGS m_HeapFlags; + const UINT64 m_PreferredBlockSize; + const size_t m_MinBlockCount; + const size_t m_MaxBlockCount; + const bool m_ExplicitBlockSize; + const UINT64 m_MinAllocationAlignment; + const UINT32 m_Algorithm; + ID3D12ProtectedResourceSession* const m_ProtectedSession; + /* There can be at most one allocation that is completely empty - a + hysteresis to avoid pessimistic case of alternating creation and destruction + of a VkDeviceMemory. */ + bool m_HasEmptyBlock; + D3D12MA_RW_MUTEX m_Mutex; + // Incrementally sorted by sumFreeSize, ascending. + Vector m_Blocks; + UINT m_NextBlockId; + + UINT64 CalcSumBlockSize() const; + UINT64 CalcMaxBlockSize() const; + + // Finds and removes given block from vector. + void Remove(NormalBlock* pBlock); + + // Performs single step in sorting m_Blocks. They may not be fully sorted + // after this call. + void IncrementallySortBlocks(); + + HRESULT AllocatePage( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + Allocation** pAllocation); + + HRESULT AllocateFromBlock( + NormalBlock* pBlock, + UINT64 size, + UINT64 alignment, + ALLOCATION_FLAGS allocFlags, + Allocation** pAllocation); + + HRESULT CreateBlock( + UINT64 blockSize, + size_t* pNewBlockIndex); +}; +#endif // _D3D12MA_BLOCK_VECTOR + +#ifndef _D3D12MA_CURRENT_BUDGET_DATA +class CurrentBudgetData { - UINT64 result = 0; - for(size_t i = m_Blocks.size(); i--; ) - { - result += m_Blocks[i]->m_pMetadata->GetSize(); - } - return result; -} +public: + bool ShouldUpdateBudget() const { return m_OperationsSinceBudgetFetch >= 30; } -UINT64 BlockVector::CalcMaxBlockSize() const -{ - UINT64 result = 0; - for(size_t i = m_Blocks.size(); i--; ) - { - result = D3D12MA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize()); - if(result >= m_PreferredBlockSize) - { - break; - } - } - return result; -} + void GetStatistics(Statistics& outStats, UINT group) const; + void GetBudget(bool useMutex, + UINT64* outLocalUsage, UINT64* outLocalBudget, + UINT64* outNonLocalUsage, UINT64* outNonLocalBudget); -void BlockVector::Remove(NormalBlock* pBlock) -{ - for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) - { - if(m_Blocks[blockIndex] == pBlock) - { - m_Blocks.remove(blockIndex); - return; - } - } - D3D12MA_ASSERT(0); -} - -void BlockVector::IncrementallySortBlocks() -{ - // Bubble sort only until first swap. - for(size_t i = 1; i < m_Blocks.size(); ++i) - { - if(m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize()) - { - D3D12MA_SWAP(m_Blocks[i - 1], m_Blocks[i]); - return; - } - } -} - -HRESULT BlockVector::AllocateFromBlock( - NormalBlock* pBlock, - UINT64 size, - UINT64 alignment, - ALLOCATION_FLAGS allocFlags, - Allocation** pAllocation) -{ - alignment = D3D12MA_MAX(alignment, m_MinAllocationAlignment); - - AllocationRequest currRequest = {}; - if(pBlock->m_pMetadata->CreateAllocationRequest( - size, - alignment, - allocFlags & ALLOCATION_FLAG_UPPER_ADDRESS, - &currRequest)) - { - // We no longer have an empty Allocation. - if(pBlock->m_pMetadata->IsEmpty()) - { - m_HasEmptyBlock = false; - } - - *pAllocation = m_hAllocator->GetAllocationObjectAllocator().Allocate(m_hAllocator, size, currRequest.zeroInitialized); - pBlock->m_pMetadata->Alloc(currRequest, size, *pAllocation); - (*pAllocation)->InitPlaced(currRequest.allocHandle, alignment, pBlock); - D3D12MA_HEAVY_ASSERT(pBlock->Validate()); - m_hAllocator->m_Budget.AddAllocation(m_hAllocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), size); - return S_OK; - } - return E_OUTOFMEMORY; -} - -HRESULT BlockVector::CreateBlock( - UINT64 blockSize, - size_t* pNewBlockIndex) -{ - NormalBlock* const pBlock = D3D12MA_NEW(m_hAllocator->GetAllocs(), NormalBlock)( - m_hAllocator, - this, - m_HeapProps, - m_HeapFlags, - blockSize, - m_NextBlockId++); - HRESULT hr = pBlock->Init(m_Algorithm, m_ProtectedSession); - if(FAILED(hr)) - { - D3D12MA_DELETE(m_hAllocator->GetAllocs(), pBlock); - return hr; - } - - m_Blocks.push_back(pBlock); - if(pNewBlockIndex != NULL) - { - *pNewBlockIndex = m_Blocks.size() - 1; - } - - return hr; -} - -void BlockVector::AddStatistics(Statistics& inoutStats) -{ - MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); - - for(size_t i = 0; i < m_Blocks.size(); ++i) - { - const NormalBlock* const pBlock = m_Blocks[i]; - D3D12MA_ASSERT(pBlock); - D3D12MA_HEAVY_ASSERT(pBlock->Validate()); - pBlock->m_pMetadata->AddStatistics(inoutStats); - } -} - -void BlockVector::AddDetailedStatistics(DetailedStatistics& inoutStats) -{ - MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); - - for(size_t i = 0; i < m_Blocks.size(); ++i) - { - const NormalBlock* const pBlock = m_Blocks[i]; - D3D12MA_ASSERT(pBlock); - D3D12MA_HEAVY_ASSERT(pBlock->Validate()); - pBlock->m_pMetadata->AddDetailedStatistics(inoutStats); - } -} - -void BlockVector::WriteBlockInfoToJson(JsonWriter& json) -{ - MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); - - json.BeginObject(); - - for (size_t i = 0, count = m_Blocks.size(); i < count; ++i) - { - const NormalBlock* const pBlock = m_Blocks[i]; - D3D12MA_ASSERT(pBlock); - D3D12MA_HEAVY_ASSERT(pBlock->Validate()); - json.BeginString(); - json.ContinueString(pBlock->GetId()); - json.EndString(); - - pBlock->m_pMetadata->WriteAllocationInfoToJson(json); - } - - json.EndObject(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Private class PoolPimpl - -PoolPimpl::PoolPimpl(AllocatorPimpl* allocator, const POOL_DESC& desc) : - m_Allocator(allocator), - m_Desc(desc), - m_BlockVector(NULL), - m_Name(NULL) -{ - const bool explicitBlockSize = desc.BlockSize != 0; - const UINT64 preferredBlockSize = explicitBlockSize ? desc.BlockSize : D3D12MA_DEFAULT_BLOCK_SIZE; - UINT maxBlockCount = desc.MaxBlockCount != 0 ? desc.MaxBlockCount : UINT_MAX; - -#ifndef __ID3D12Device4_INTERFACE_DEFINED__ - D3D12MA_ASSERT(m_Desc.pProtectedSession == NULL); +#if D3D12MA_DXGI_1_4 + HRESULT UpdateBudget(IDXGIAdapter3* adapter3, bool useMutex); #endif - m_BlockVector = D3D12MA_NEW(allocator->GetAllocs(), BlockVector)( - allocator, desc.HeapProperties, desc.HeapFlags, - preferredBlockSize, - desc.MinBlockCount, maxBlockCount, - explicitBlockSize, - D3D12MA_MAX(desc.MinAllocationAlignment, (UINT64)D3D12MA_DEBUG_ALIGNMENT), - desc.Flags & POOL_FLAG_ALGORITHM_MASK, - desc.pProtectedSession); -} + void AddAllocation(UINT group, UINT64 allocationBytes); + void RemoveAllocation(UINT group, UINT64 allocationBytes); -HRESULT PoolPimpl::Init() + void AddBlock(UINT group, UINT64 blockBytes); + void RemoveBlock(UINT group, UINT64 blockBytes); + +private: + D3D12MA_ATOMIC_UINT32 m_BlockCount[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + D3D12MA_ATOMIC_UINT32 m_AllocationCount[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + D3D12MA_ATOMIC_UINT64 m_BlockBytes[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + D3D12MA_ATOMIC_UINT64 m_AllocationBytes[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + + D3D12MA_ATOMIC_UINT32 m_OperationsSinceBudgetFetch = 0; + D3D12MA_RW_MUTEX m_BudgetMutex; + UINT64 m_D3D12Usage[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + UINT64 m_D3D12Budget[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; + UINT64 m_BlockBytesAtD3D12Fetch[DXGI_MEMORY_SEGMENT_GROUP_COUNT] = {}; +}; + +#ifndef _D3D12MA_CURRENT_BUDGET_DATA_FUNCTIONS +void CurrentBudgetData::GetStatistics(Statistics& outStats, UINT group) const { - m_CommittedAllocations.Init(m_Allocator->UseMutex(), m_Desc.HeapProperties.Type, this); - return m_BlockVector->CreateMinBlocks(); + outStats.BlockCount = m_BlockCount[group]; + outStats.AllocationCount = m_AllocationCount[group]; + outStats.BlockBytes = m_BlockBytes[group]; + outStats.AllocationBytes = m_AllocationBytes[group]; } -PoolPimpl::~PoolPimpl() -{ - D3D12MA_ASSERT(m_PrevPool == NULL && m_NextPool == NULL); - FreeName(); - D3D12MA_DELETE(m_Allocator->GetAllocs(), m_BlockVector); -} - -void PoolPimpl::GetStatistics(Statistics& outStats) -{ - ClearStatistics(outStats); - m_BlockVector->AddStatistics(outStats); - m_CommittedAllocations.AddStatistics(outStats); -} - -void PoolPimpl::CalculateStatistics(DetailedStatistics& outStats) -{ - ClearDetailedStatistics(outStats); - AddDetailedStatistics(outStats); -} - -void PoolPimpl::AddDetailedStatistics(DetailedStatistics& inoutStats) -{ - m_BlockVector->AddDetailedStatistics(inoutStats); - m_CommittedAllocations.AddDetailedStatistics(inoutStats); -} - -void PoolPimpl::SetName(LPCWSTR Name) -{ - FreeName(); - - if(Name) - { - const size_t nameCharCount = wcslen(Name) + 1; - m_Name = D3D12MA_NEW_ARRAY(m_Allocator->GetAllocs(), WCHAR, nameCharCount); - memcpy(m_Name, Name, nameCharCount * sizeof(WCHAR)); - } -} - -void PoolPimpl::FreeName() -{ - if(m_Name) - { - const size_t nameCharCount = wcslen(m_Name) + 1; - D3D12MA_DELETE_ARRAY(m_Allocator->GetAllocs(), m_Name, nameCharCount); - m_Name = NULL; - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Private class CurrentBudgetData implementation - void CurrentBudgetData::GetBudget(bool useMutex, UINT64* outLocalUsage, UINT64* outLocalBudget, UINT64* outNonLocalUsage, UINT64* outNonLocalBudget) { MutexLockRead lockRead(m_BudgetMutex, useMutex); - if(outLocalUsage) + if (outLocalUsage) { const UINT64 D3D12Usage = m_D3D12Usage[DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY]; const UINT64 blockBytes = m_BlockBytes[DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY]; @@ -6860,10 +5904,10 @@ void CurrentBudgetData::GetBudget(bool useMutex, *outLocalUsage = D3D12Usage + blockBytes > blockBytesAtD3D12Fetch ? D3D12Usage + blockBytes - blockBytesAtD3D12Fetch : 0; } - if(outLocalBudget) + if (outLocalBudget) *outLocalBudget = m_D3D12Budget[DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY]; - if(outNonLocalUsage) + if (outNonLocalUsage) { const UINT64 D3D12Usage = m_D3D12Usage[DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY]; const UINT64 blockBytes = m_BlockBytes[DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY]; @@ -6871,7 +5915,7 @@ void CurrentBudgetData::GetBudget(bool useMutex, *outNonLocalUsage = D3D12Usage + blockBytes > blockBytesAtD3D12Fetch ? D3D12Usage + blockBytes - blockBytesAtD3D12Fetch : 0; } - if(outNonLocalBudget) + if (outNonLocalBudget) *outNonLocalBudget = m_D3D12Budget[DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY]; } @@ -6885,16 +5929,16 @@ HRESULT CurrentBudgetData::UpdateBudget(IDXGIAdapter3* adapter3, bool useMutex) const HRESULT hrLocal = adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &infoLocal); const HRESULT hrNonLocal = adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL, &infoNonLocal); - if(SUCCEEDED(hrLocal) || SUCCEEDED(hrNonLocal)) + if (SUCCEEDED(hrLocal) || SUCCEEDED(hrNonLocal)) { MutexLockWrite lockWrite(m_BudgetMutex, useMutex); - if(SUCCEEDED(hrLocal)) + if (SUCCEEDED(hrLocal)) { m_D3D12Usage[0] = infoLocal.CurrentUsage; m_D3D12Budget[0] = infoLocal.Budget; } - if(SUCCEEDED(hrNonLocal)) + if (SUCCEEDED(hrNonLocal)) { m_D3D12Usage[1] = infoNonLocal.CurrentUsage; m_D3D12Budget[1] = infoNonLocal.Budget; @@ -6909,66 +5953,287 @@ HRESULT CurrentBudgetData::UpdateBudget(IDXGIAdapter3* adapter3, bool useMutex) } #endif // #if D3D12MA_DXGI_1_4 -//////////////////////////////////////////////////////////////////////////////// -// Public class Pool implementation - -void Pool::ReleaseThis() +void CurrentBudgetData::AddAllocation(UINT group, UINT64 allocationBytes) { - if(this == NULL) - { - return; - } - - D3D12MA_DELETE(m_Pimpl->GetAllocator()->GetAllocs(), this); + ++m_AllocationCount[group]; + m_AllocationBytes[group] += allocationBytes; + ++m_OperationsSinceBudgetFetch; } -POOL_DESC Pool::GetDesc() const +void CurrentBudgetData::RemoveAllocation(UINT group, UINT64 allocationBytes) { - return m_Pimpl->GetDesc(); + D3D12MA_ASSERT(m_AllocationBytes[group] >= allocationBytes); + D3D12MA_ASSERT(m_AllocationCount[group] > 0); + m_AllocationBytes[group] -= allocationBytes; + --m_AllocationCount[group]; + ++m_OperationsSinceBudgetFetch; } -void Pool::GetStatistics(Statistics* pStats) +void CurrentBudgetData::AddBlock(UINT group, UINT64 blockBytes) { - D3D12MA_ASSERT(pStats); - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->GetStatistics(*pStats); + ++m_BlockCount[group]; + m_BlockBytes[group] += blockBytes; + ++m_OperationsSinceBudgetFetch; } -void Pool::CalculateStatistics(DetailedStatistics* pStats) +void CurrentBudgetData::RemoveBlock(UINT group, UINT64 blockBytes) { - D3D12MA_ASSERT(pStats); - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->CalculateStatistics(*pStats); + D3D12MA_ASSERT(m_BlockBytes[group] >= blockBytes); + D3D12MA_ASSERT(m_BlockCount[group] > 0); + m_BlockBytes[group] -= blockBytes; + --m_BlockCount[group]; + ++m_OperationsSinceBudgetFetch; } +#endif // _D3D12MA_CURRENT_BUDGET_DATA_FUNCTIONS +#endif // _D3D12MA_CURRENT_BUDGET_DATA -void Pool::SetName(LPCWSTR Name) +#ifndef _D3D12MA_POOL_PIMPL +class PoolPimpl { - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->SetName(Name); -} + friend class Allocator; + friend struct PoolListItemTraits; +public: + PoolPimpl(AllocatorPimpl* allocator, const POOL_DESC& desc); + ~PoolPimpl(); -LPCWSTR Pool::GetName() const + AllocatorPimpl* GetAllocator() const { return m_Allocator; } + const POOL_DESC& GetDesc() const { return m_Desc; } + bool SupportsCommittedAllocations() const { return m_Desc.BlockSize == 0; } + LPCWSTR GetName() const { return m_Name; } + + BlockVector* GetBlockVector() { return m_BlockVector; } + CommittedAllocationList* GetCommittedAllocationList() { return SupportsCommittedAllocations() ? &m_CommittedAllocations : NULL; } + + HRESULT Init(); + void GetStatistics(Statistics& outStats); + void CalculateStatistics(DetailedStatistics& outStats); + void AddDetailedStatistics(DetailedStatistics& inoutStats); + void SetName(LPCWSTR Name); + +private: + AllocatorPimpl* m_Allocator; // Externally owned object. + POOL_DESC m_Desc; + BlockVector* m_BlockVector; // Owned object. + CommittedAllocationList m_CommittedAllocations; + wchar_t* m_Name; + PoolPimpl* m_PrevPool = NULL; + PoolPimpl* m_NextPool = NULL; + + void FreeName(); +}; + +struct PoolListItemTraits { - return m_Pimpl->GetName(); -} + using ItemType = PoolPimpl; + static ItemType* GetPrev(const ItemType* item) { return item->m_PrevPool; } + static ItemType* GetNext(const ItemType* item) { return item->m_NextPool; } + static ItemType*& AccessPrev(ItemType* item) { return item->m_PrevPool; } + static ItemType*& AccessNext(ItemType* item) { return item->m_NextPool; } +}; +#endif // _D3D12MA_POOL_PIMPL -Pool::Pool(Allocator* allocator, const POOL_DESC &desc) : - m_Pimpl(D3D12MA_NEW(allocator->m_Pimpl->GetAllocs(), PoolPimpl)(allocator->m_Pimpl, desc)) + +#ifndef _D3D12MA_ALLOCATOR_PIMPL +class AllocatorPimpl { -} + friend class Allocator; + friend class Pool; +public: + std::atomic_uint32_t m_RefCount = 1; + CurrentBudgetData m_Budget; -Pool::~Pool() -{ - m_Pimpl->GetAllocator()->UnregisterPool(this, m_Pimpl->GetDesc().HeapProperties.Type); + AllocatorPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc); + ~AllocatorPimpl(); - D3D12MA_DELETE(m_Pimpl->GetAllocator()->GetAllocs(), m_Pimpl); -} + ID3D12Device* GetDevice() const { return m_Device; } +#ifdef __ID3D12Device4_INTERFACE_DEFINED__ + ID3D12Device4* GetDevice4() const { return m_Device4; } +#endif +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ + ID3D12Device8* GetDevice8() const { return m_Device8; } +#endif + // Shortcut for "Allocation Callbacks", because this function is called so often. + const ALLOCATION_CALLBACKS& GetAllocs() const { return m_AllocationCallbacks; } + const D3D12_FEATURE_DATA_D3D12_OPTIONS& GetD3D12Options() const { return m_D3D12Options; } + BOOL IsUMA() const { return m_D3D12Architecture.UMA; } + BOOL IsCacheCoherentUMA() const { return m_D3D12Architecture.CacheCoherentUMA; } + bool SupportsResourceHeapTier2() const { return m_D3D12Options.ResourceHeapTier >= D3D12_RESOURCE_HEAP_TIER_2; } + bool UseMutex() const { return m_UseMutex; } + AllocationObjectAllocator& GetAllocationObjectAllocator() { return m_AllocationObjectAllocator; } + UINT GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); } -//////////////////////////////////////////////////////////////////////////////// -// Private class AllocatorPimpl implementation + HRESULT Init(const ALLOCATOR_DESC& desc); + bool HeapFlagsFulfillResourceHeapTier(D3D12_HEAP_FLAGS flags) const; + UINT StandardHeapTypeToMemorySegmentGroup(D3D12_HEAP_TYPE heapType) const; + UINT HeapPropertiesToMemorySegmentGroup(const D3D12_HEAP_PROPERTIES& heapProps) const; + UINT64 GetMemoryCapacity(UINT memorySegmentGroup) const; -AllocatorPimpl::AllocatorPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc) : - m_UseMutex((desc.Flags & ALLOCATOR_FLAG_SINGLETHREADED) == 0), + HRESULT CreateResource( + const ALLOCATION_DESC* pAllocDesc, + const D3D12_RESOURCE_DESC* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource); + +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ + HRESULT CreateResource2( + const ALLOCATION_DESC* pAllocDesc, + const D3D12_RESOURCE_DESC1* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource); +#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ + + HRESULT CreateAliasingResource( + Allocation* pAllocation, + UINT64 AllocationLocalOffset, + const D3D12_RESOURCE_DESC* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE *pOptimizedClearValue, + REFIID riidResource, + void** ppvResource); + + HRESULT AllocateMemory( + const ALLOCATION_DESC* pAllocDesc, + const D3D12_RESOURCE_ALLOCATION_INFO* pAllocInfo, + Allocation** ppAllocation); + + // Unregisters allocation from the collection of dedicated allocations. + // Allocation object must be deleted externally afterwards. + void FreeCommittedMemory(Allocation* allocation); + // Unregisters allocation from the collection of placed allocations. + // Allocation object must be deleted externally afterwards. + void FreePlacedMemory(Allocation* allocation); + // Unregisters allocation from the collection of dedicated allocations and destroys associated heap. + // Allocation object must be deleted externally afterwards. + void FreeHeapMemory(Allocation* allocation); + + void SetCurrentFrameIndex(UINT frameIndex); + + void CalculateStatistics(TotalStatistics& outStats); + + void GetBudget(Budget* outLocalBudget, Budget* outNonLocalBudget); + void GetBudgetForHeapType(Budget& outBudget, D3D12_HEAP_TYPE heapType); + + void BuildStatsString(WCHAR** ppStatsString, BOOL DetailedMap); + void FreeStatsString(WCHAR* pStatsString); + +private: + using PoolList = IntrusiveLinkedList; + + const bool m_UseMutex; + const bool m_AlwaysCommitted; + ID3D12Device* m_Device; // AddRef +#ifdef __ID3D12Device4_INTERFACE_DEFINED__ + ID3D12Device4* m_Device4 = NULL; // AddRef, optional +#endif +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ + ID3D12Device8* m_Device8 = NULL; // AddRef, optional +#endif + IDXGIAdapter* m_Adapter; // AddRef +#if D3D12MA_DXGI_1_4 + IDXGIAdapter3* m_Adapter3 = NULL; // AddRef, optional +#endif + UINT64 m_PreferredBlockSize; + ALLOCATION_CALLBACKS m_AllocationCallbacks; + D3D12MA_ATOMIC_UINT32 m_CurrentFrameIndex; + DXGI_ADAPTER_DESC m_AdapterDesc; + D3D12_FEATURE_DATA_D3D12_OPTIONS m_D3D12Options; + D3D12_FEATURE_DATA_ARCHITECTURE m_D3D12Architecture; + AllocationObjectAllocator m_AllocationObjectAllocator; + + D3D12MA_RW_MUTEX m_PoolsMutex[HEAP_TYPE_COUNT]; + PoolList m_Pools[HEAP_TYPE_COUNT]; + // Default pools. + BlockVector* m_BlockVectors[DEFAULT_POOL_MAX_COUNT]; + CommittedAllocationList m_CommittedAllocations[STANDARD_HEAP_TYPE_COUNT]; + + /* + Heuristics that decides whether a resource should better be placed in its own, + dedicated allocation (committed resource rather than placed resource). + */ + template + static bool PrefersCommittedAllocation(const D3D12_RESOURCE_DESC_T& resourceDesc); + + // Allocates and registers new committed resource with implicit heap, as dedicated allocation. + // Creates and returns Allocation object and optionally D3D12 resource. + HRESULT AllocateCommittedResource( + const CommittedAllocationParameters& committedAllocParams, + UINT64 resourceSize, bool withinBudget, + const D3D12_RESOURCE_DESC* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, REFIID riidResource, void** ppvResource); + +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ + HRESULT AllocateCommittedResource2( + const CommittedAllocationParameters& committedAllocParams, + UINT64 resourceSize, bool withinBudget, + const D3D12_RESOURCE_DESC1* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, + Allocation** ppAllocation, REFIID riidResource, void** ppvResource); +#endif + + // Allocates and registers new heap without any resources placed in it, as dedicated allocation. + // Creates and returns Allocation object. + HRESULT AllocateHeap( + const CommittedAllocationParameters& committedAllocParams, + const D3D12_RESOURCE_ALLOCATION_INFO& allocInfo, bool withinBudget, + Allocation** ppAllocation); + + template + HRESULT CalcAllocationParams(const ALLOCATION_DESC& allocDesc, UINT64 allocSize, + const D3D12_RESOURCE_DESC_T* resDesc, // Optional + BlockVector*& outBlockVector, CommittedAllocationParameters& outCommittedAllocationParams, bool& outPreferCommitted); + + /* + If SupportsResourceHeapTier2(): + 0: D3D12_HEAP_TYPE_DEFAULT + 1: D3D12_HEAP_TYPE_UPLOAD + 2: D3D12_HEAP_TYPE_READBACK + else: + 0: D3D12_HEAP_TYPE_DEFAULT + buffer + 1: D3D12_HEAP_TYPE_DEFAULT + texture + 2: D3D12_HEAP_TYPE_DEFAULT + texture RT or DS + 3: D3D12_HEAP_TYPE_UPLOAD + buffer + 4: D3D12_HEAP_TYPE_UPLOAD + texture + 5: D3D12_HEAP_TYPE_UPLOAD + texture RT or DS + 6: D3D12_HEAP_TYPE_READBACK + buffer + 7: D3D12_HEAP_TYPE_READBACK + texture + 8: D3D12_HEAP_TYPE_READBACK + texture RT or DS + */ + UINT CalcDefaultPoolCount() const; + // Returns UINT32_MAX if index cannot be calculcated. + UINT CalcDefaultPoolIndex(const ALLOCATION_DESC& allocDesc, ResourceClass resourceClass) const; + void CalcDefaultPoolParams(D3D12_HEAP_TYPE& outHeapType, D3D12_HEAP_FLAGS& outHeapFlags, UINT index) const; + + // Registers Pool object in m_Pools. + void RegisterPool(Pool* pool, D3D12_HEAP_TYPE heapType); + // Unregisters Pool object from m_Pools. + void UnregisterPool(Pool* pool, D3D12_HEAP_TYPE heapType); + + HRESULT UpdateD3D12Budget(); + + D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfoNative(const D3D12_RESOURCE_DESC& resourceDesc) const; +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ + D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfoNative(const D3D12_RESOURCE_DESC1& resourceDesc) const; +#endif + + template + D3D12_RESOURCE_ALLOCATION_INFO GetResourceAllocationInfo(D3D12_RESOURCE_DESC_T& inOutResourceDesc) const; + + bool NewAllocationWithinBudget(D3D12_HEAP_TYPE heapType, UINT64 size); + + // Writes object { } with data of given budget. + static void WriteBudgetToJson(JsonWriter& json, const Budget& budget); +}; + +#ifndef _D3D12MA_ALLOCATOR_PIMPL_FUNCTINOS +AllocatorPimpl::AllocatorPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc) + : m_UseMutex((desc.Flags & ALLOCATOR_FLAG_SINGLETHREADED) == 0), m_AlwaysCommitted((desc.Flags & ALLOCATOR_FLAG_ALWAYS_COMMITTED) != 0), m_Device(desc.pDevice), m_Adapter(desc.pAdapter), @@ -6984,7 +6249,7 @@ AllocatorPimpl::AllocatorPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, ZeroMemory(m_BlockVectors, sizeof(m_BlockVectors)); - for(UINT i = 0; i < STANDARD_HEAP_TYPE_COUNT; ++i) + for (UINT i = 0; i < STANDARD_HEAP_TYPE_COUNT; ++i) { m_CommittedAllocations[i].Init( m_UseMutex, @@ -7011,13 +6276,13 @@ HRESULT AllocatorPimpl::Init(const ALLOCATOR_DESC& desc) #endif HRESULT hr = m_Adapter->GetDesc(&m_AdapterDesc); - if(FAILED(hr)) + if (FAILED(hr)) { return hr; } hr = m_Device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS, &m_D3D12Options, sizeof(m_D3D12Options)); - if(FAILED(hr)) + if (FAILED(hr)) { return hr; } @@ -7026,7 +6291,7 @@ HRESULT AllocatorPimpl::Init(const ALLOCATOR_DESC& desc) #endif hr = m_Device->CheckFeatureSupport(D3D12_FEATURE_ARCHITECTURE, &m_D3D12Architecture, sizeof(m_D3D12Architecture)); - if(FAILED(hr)) + if (FAILED(hr)) { m_D3D12Architecture.UMA = FALSE; m_D3D12Architecture.CacheCoherentUMA = FALSE; @@ -7034,7 +6299,7 @@ HRESULT AllocatorPimpl::Init(const ALLOCATOR_DESC& desc) D3D12_HEAP_PROPERTIES heapProps = {}; const UINT defaultPoolCount = CalcDefaultPoolCount(); - for(UINT i = 0; i < defaultPoolCount; ++i) + for (UINT i = 0; i < defaultPoolCount; ++i) { D3D12_HEAP_FLAGS heapFlags; CalcDefaultPoolParams(heapProps.Type, heapFlags, i); @@ -7074,14 +6339,14 @@ AllocatorPimpl::~AllocatorPimpl() SAFE_RELEASE(m_Adapter); SAFE_RELEASE(m_Device); - for(UINT i = DEFAULT_POOL_MAX_COUNT; i--; ) + for (UINT i = DEFAULT_POOL_MAX_COUNT; i--; ) { D3D12MA_DELETE(GetAllocs(), m_BlockVectors[i]); } - for(UINT i = HEAP_TYPE_COUNT; i--; ) + for (UINT i = HEAP_TYPE_COUNT; i--; ) { - if(!m_Pools[i].IsEmpty()) + if (!m_Pools[i].IsEmpty()) { D3D12MA_ASSERT(0 && "Unfreed pools found!"); } @@ -7090,25 +6355,59 @@ AllocatorPimpl::~AllocatorPimpl() bool AllocatorPimpl::HeapFlagsFulfillResourceHeapTier(D3D12_HEAP_FLAGS flags) const { - if(SupportsResourceHeapTier2()) + if (SupportsResourceHeapTier2()) { return true; } else { - const bool allowBuffers = (flags & D3D12_HEAP_FLAG_DENY_BUFFERS ) == 0; - const bool allowRtDsTextures = (flags & D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES ) == 0; + const bool allowBuffers = (flags & D3D12_HEAP_FLAG_DENY_BUFFERS) == 0; + const bool allowRtDsTextures = (flags & D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES) == 0; const bool allowNonRtDsTextures = (flags & D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES) == 0; const uint8_t allowedGroupCount = (allowBuffers ? 1 : 0) + (allowRtDsTextures ? 1 : 0) + (allowNonRtDsTextures ? 1 : 0); return allowedGroupCount == 1; } } +UINT AllocatorPimpl::StandardHeapTypeToMemorySegmentGroup(D3D12_HEAP_TYPE heapType) const +{ + D3D12MA_ASSERT(IsHeapTypeStandard(heapType)); + if (IsUMA()) + return DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY; + return heapType == D3D12_HEAP_TYPE_DEFAULT ? + DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY : DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY; +} + +UINT AllocatorPimpl::HeapPropertiesToMemorySegmentGroup(const D3D12_HEAP_PROPERTIES& heapProps) const +{ + if (IsUMA()) + return DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY; + if (heapProps.MemoryPoolPreference == D3D12_MEMORY_POOL_UNKNOWN) + return StandardHeapTypeToMemorySegmentGroup(heapProps.Type); + return heapProps.MemoryPoolPreference == D3D12_MEMORY_POOL_L1 ? + DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY : DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY; +} + +UINT64 AllocatorPimpl::GetMemoryCapacity(UINT memorySegmentGroup) const +{ + switch (memorySegmentGroup) + { + case DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY: + return IsUMA() ? + m_AdapterDesc.DedicatedVideoMemory + m_AdapterDesc.SharedSystemMemory : m_AdapterDesc.DedicatedVideoMemory; + case DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY: + return IsUMA() ? 0 : m_AdapterDesc.SharedSystemMemory; + default: + D3D12MA_ASSERT(0); + return UINT64_MAX; + } +} + HRESULT AllocatorPimpl::CreateResource( const ALLOCATION_DESC* pAllocDesc, const D3D12_RESOURCE_DESC* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, Allocation** ppAllocation, REFIID riidResource, void** ppvResource) @@ -7116,7 +6415,7 @@ HRESULT AllocatorPimpl::CreateResource( D3D12MA_ASSERT(pAllocDesc && pResourceDesc && ppAllocation); *ppAllocation = NULL; - if(ppvResource) + if (ppvResource) { *ppvResource = NULL; } @@ -7132,36 +6431,36 @@ HRESULT AllocatorPimpl::CreateResource( HRESULT hr = CalcAllocationParams(*pAllocDesc, resAllocInfo.SizeInBytes, pResourceDesc, blockVector, committedAllocationParams, preferCommitted); - if(FAILED(hr)) + if (FAILED(hr)) return hr; const bool withinBudget = (pAllocDesc->Flags & ALLOCATION_FLAG_WITHIN_BUDGET) != 0; hr = E_INVALIDARG; - if(committedAllocationParams.IsValid() && preferCommitted) + if (committedAllocationParams.IsValid() && preferCommitted) { hr = AllocateCommittedResource(committedAllocationParams, resAllocInfo.SizeInBytes, withinBudget, &finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(blockVector != NULL) + if (blockVector != NULL) { hr = blockVector->CreateResource(resAllocInfo.SizeInBytes, resAllocInfo.Alignment, *pAllocDesc, finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(committedAllocationParams.IsValid() && !preferCommitted) + if (committedAllocationParams.IsValid() && !preferCommitted) { hr = AllocateCommittedResource(committedAllocationParams, resAllocInfo.SizeInBytes, withinBudget, &finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } return hr; @@ -7172,7 +6471,7 @@ HRESULT AllocatorPimpl::CreateResource2( const ALLOCATION_DESC* pAllocDesc, const D3D12_RESOURCE_DESC1* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, Allocation** ppAllocation, REFIID riidResource, void** ppvResource) @@ -7180,11 +6479,11 @@ HRESULT AllocatorPimpl::CreateResource2( D3D12MA_ASSERT(pAllocDesc && pResourceDesc && ppAllocation); *ppAllocation = NULL; - if(ppvResource) + if (ppvResource) { *ppvResource = NULL; } - if(m_Device8 == NULL) + if (m_Device8 == NULL) { return E_NOINTERFACE; } @@ -7200,36 +6499,36 @@ HRESULT AllocatorPimpl::CreateResource2( HRESULT hr = CalcAllocationParams(*pAllocDesc, resAllocInfo.SizeInBytes, pResourceDesc, blockVector, committedAllocationParams, preferCommitted); - if(FAILED(hr)) + if (FAILED(hr)) return hr; - + const bool withinBudget = (pAllocDesc->Flags & ALLOCATION_FLAG_WITHIN_BUDGET) != 0; hr = E_INVALIDARG; - if(committedAllocationParams.IsValid() && preferCommitted) + if (committedAllocationParams.IsValid() && preferCommitted) { hr = AllocateCommittedResource2(committedAllocationParams, - resAllocInfo.SizeInBytes, withinBudget,&finalResourceDesc, + resAllocInfo.SizeInBytes, withinBudget, &finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(blockVector != NULL) + if (blockVector != NULL) { hr = blockVector->CreateResource2(resAllocInfo.SizeInBytes, resAllocInfo.Alignment, *pAllocDesc, finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(committedAllocationParams.IsValid() && !preferCommitted) + if (committedAllocationParams.IsValid() && !preferCommitted) { hr = AllocateCommittedResource2(committedAllocationParams, - resAllocInfo.SizeInBytes, withinBudget,&finalResourceDesc, + resAllocInfo.SizeInBytes, withinBudget, &finalResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } return hr; @@ -7249,28 +6548,28 @@ HRESULT AllocatorPimpl::AllocateMemory( HRESULT hr = CalcAllocationParams(*pAllocDesc, pAllocInfo->SizeInBytes, NULL, // pResDesc blockVector, committedAllocationParams, preferCommitted); - if(FAILED(hr)) + if (FAILED(hr)) return hr; - + const bool withinBudget = (pAllocDesc->Flags & ALLOCATION_FLAG_WITHIN_BUDGET) != 0; hr = E_INVALIDARG; - if(committedAllocationParams.IsValid() && preferCommitted) + if (committedAllocationParams.IsValid() && preferCommitted) { hr = AllocateHeap(committedAllocationParams, *pAllocInfo, withinBudget, ppAllocation); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(blockVector != NULL) + if (blockVector != NULL) { hr = blockVector->Allocate(pAllocInfo->SizeInBytes, pAllocInfo->Alignment, *pAllocDesc, 1, (Allocation**)ppAllocation); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } - if(committedAllocationParams.IsValid() && !preferCommitted) + if (committedAllocationParams.IsValid() && !preferCommitted) { hr = AllocateHeap(committedAllocationParams, *pAllocInfo, withinBudget, ppAllocation); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) return hr; } return hr; @@ -7281,7 +6580,7 @@ HRESULT AllocatorPimpl::CreateAliasingResource( UINT64 AllocationLocalOffset, const D3D12_RESOURCE_DESC* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, REFIID riidResource, void** ppvResource) { @@ -7297,7 +6596,7 @@ HRESULT AllocatorPimpl::CreateAliasingResource( const UINT64 existingSize = pAllocation->GetSize(); const UINT64 newOffset = existingOffset + AllocationLocalOffset; - if(existingHeap == NULL || + if (existingHeap == NULL || AllocationLocalOffset + resAllocInfo.SizeInBytes > existingSize || newOffset % resAllocInfo.Alignment != 0) { @@ -7314,375 +6613,6 @@ HRESULT AllocatorPimpl::CreateAliasingResource( ppvResource); } -template -bool AllocatorPimpl::PrefersCommittedAllocation(const D3D12_RESOURCE_DESC_T& resourceDesc) -{ - // Intentional. It may change in the future. - return false; -} - -HRESULT AllocatorPimpl::AllocateCommittedResource( - const CommittedAllocationParameters& committedAllocParams, - UINT64 resourceSize, bool withinBudget, - const D3D12_RESOURCE_DESC* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, REFIID riidResource, void** ppvResource) -{ - D3D12MA_ASSERT(committedAllocParams.IsValid()); - - if(withinBudget && - !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, resourceSize)) - { - return E_OUTOFMEMORY; - } - - ID3D12Resource* res = NULL; - /* D3D12 ERROR: - * ID3D12Device::CreateCommittedResource: - * When creating a committed resource, D3D12_HEAP_FLAGS must not have either - * D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES, - * D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES, - * nor D3D12_HEAP_FLAG_DENY_BUFFERS set. - * These flags will be set automatically to correspond with the committed resource type. - * - * [ STATE_CREATION ERROR #640: CREATERESOURCEANDHEAP_INVALIDHEAPMISCFLAGS] - */ - HRESULT hr; -#ifdef __ID3D12Device4_INTERFACE_DEFINED__ - if(m_Device4) - { - hr = m_Device4->CreateCommittedResource1( - &committedAllocParams.m_HeapProperties, - committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, - pResourceDesc, InitialResourceState, - pOptimizedClearValue, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&res)); - } - else -#endif - { - if(committedAllocParams.m_ProtectedSession == NULL) - { - hr = m_Device->CreateCommittedResource( - &committedAllocParams.m_HeapProperties, - committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, - pResourceDesc, InitialResourceState, - pOptimizedClearValue, D3D12MA_IID_PPV_ARGS(&res)); - } - else - hr = E_NOINTERFACE; - } - - if(SUCCEEDED(hr)) - { - if(ppvResource != NULL) - { - hr = res->QueryInterface(riidResource, ppvResource); - } - if(SUCCEEDED(hr)) - { - const BOOL wasZeroInitialized = TRUE; - Allocation* alloc = m_AllocationObjectAllocator.Allocate(this, resourceSize, wasZeroInitialized); - alloc->InitCommitted(committedAllocParams.m_List); - alloc->SetResource(res, pResourceDesc); - - *ppAllocation = alloc; - - committedAllocParams.m_List->Register(alloc); - - const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); - m_Budget.AddBlock(memSegmentGroup, resourceSize); - m_Budget.AddAllocation(memSegmentGroup, resourceSize); - } - else - { - res->Release(); - } - } - return hr; -} - -#ifdef __ID3D12Device8_INTERFACE_DEFINED__ -HRESULT AllocatorPimpl::AllocateCommittedResource2( - const CommittedAllocationParameters& committedAllocParams, - UINT64 resourceSize, bool withinBudget, - const D3D12_RESOURCE_DESC1* pResourceDesc, - D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE *pOptimizedClearValue, - Allocation** ppAllocation, REFIID riidResource, void** ppvResource) -{ - D3D12MA_ASSERT(committedAllocParams.IsValid()); - - if(m_Device8 == NULL) - { - return E_NOINTERFACE; - } - - if(withinBudget && - !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, resourceSize)) - { - return E_OUTOFMEMORY; - } - - ID3D12Resource* res = NULL; - HRESULT hr = m_Device8->CreateCommittedResource2( - &committedAllocParams.m_HeapProperties, - committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, // D3D12 ERROR: ID3D12Device::CreateCommittedResource: When creating a committed resource, D3D12_HEAP_FLAGS must not have either D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES, D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES, nor D3D12_HEAP_FLAG_DENY_BUFFERS set. These flags will be set automatically to correspond with the committed resource type. [ STATE_CREATION ERROR #640: CREATERESOURCEANDHEAP_INVALIDHEAPMISCFLAGS] - pResourceDesc, InitialResourceState, - pOptimizedClearValue, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&res)); - if(SUCCEEDED(hr)) - { - if(ppvResource != NULL) - { - hr = res->QueryInterface(riidResource, ppvResource); - } - if(SUCCEEDED(hr)) - { - const BOOL wasZeroInitialized = TRUE; - Allocation* alloc = m_AllocationObjectAllocator.Allocate(this, resourceSize, wasZeroInitialized); - alloc->InitCommitted(committedAllocParams.m_List); - alloc->SetResource(res, pResourceDesc); - - *ppAllocation = alloc; - - committedAllocParams.m_List->Register(alloc); - - const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); - m_Budget.AddBlock(memSegmentGroup, resourceSize); - m_Budget.AddAllocation(memSegmentGroup, resourceSize); - } - else - { - res->Release(); - } - } - return hr; -} -#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ - -HRESULT AllocatorPimpl::AllocateHeap( - const CommittedAllocationParameters& committedAllocParams, - const D3D12_RESOURCE_ALLOCATION_INFO& allocInfo, bool withinBudget, - Allocation** ppAllocation) -{ - D3D12MA_ASSERT(committedAllocParams.IsValid()); - - *ppAllocation = nullptr; - - if(withinBudget && - !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, allocInfo.SizeInBytes)) - { - return E_OUTOFMEMORY; - } - - D3D12_HEAP_DESC heapDesc = {}; - heapDesc.SizeInBytes = allocInfo.SizeInBytes; - heapDesc.Properties = committedAllocParams.m_HeapProperties; - heapDesc.Alignment = allocInfo.Alignment; - heapDesc.Flags = committedAllocParams.m_HeapFlags; - - HRESULT hr; - ID3D12Heap* heap = nullptr; -#ifdef __ID3D12Device4_INTERFACE_DEFINED__ - if(m_Device4) - hr = m_Device4->CreateHeap1(&heapDesc, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&heap)); - else -#endif - { - if(committedAllocParams.m_ProtectedSession == NULL) - hr = m_Device->CreateHeap(&heapDesc, D3D12MA_IID_PPV_ARGS(&heap)); - else - hr = E_NOINTERFACE; - } - - if(SUCCEEDED(hr)) - { - const BOOL wasZeroInitialized = TRUE; - (*ppAllocation) = m_AllocationObjectAllocator.Allocate(this, allocInfo.SizeInBytes, wasZeroInitialized); - (*ppAllocation)->InitHeap(committedAllocParams.m_List, heap); - committedAllocParams.m_List->Register(*ppAllocation); - - const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); - m_Budget.AddBlock(memSegmentGroup, allocInfo.SizeInBytes); - m_Budget.AddAllocation(memSegmentGroup, allocInfo.SizeInBytes); - } - return hr; -} - -template -HRESULT AllocatorPimpl::CalcAllocationParams(const ALLOCATION_DESC& allocDesc, UINT64 allocSize, - const D3D12_RESOURCE_DESC_T* resDesc, - BlockVector*& outBlockVector, CommittedAllocationParameters& outCommittedAllocationParams, bool& outPreferCommitted) -{ - outBlockVector = NULL; - outCommittedAllocationParams = CommittedAllocationParameters(); - outPreferCommitted = false; - - if(allocDesc.CustomPool != NULL) - { - PoolPimpl* const pool = allocDesc.CustomPool->m_Pimpl; - - outBlockVector = pool->GetBlockVector(); - - outCommittedAllocationParams.m_ProtectedSession = pool->GetDesc().pProtectedSession; - outCommittedAllocationParams.m_HeapProperties = pool->GetDesc().HeapProperties; - outCommittedAllocationParams.m_HeapFlags = pool->GetDesc().HeapFlags; - outCommittedAllocationParams.m_List = pool->GetCommittedAllocationList(); - } - else - { - if(!IsHeapTypeStandard(allocDesc.HeapType)) - { - return E_INVALIDARG; - } - - outCommittedAllocationParams.m_HeapProperties = StandardHeapTypeToHeapProperties(allocDesc.HeapType); - outCommittedAllocationParams.m_HeapFlags = allocDesc.ExtraHeapFlags; - outCommittedAllocationParams.m_List = &m_CommittedAllocations[HeapTypeToIndex(allocDesc.HeapType)]; - - const ResourceClass resourceClass = (resDesc != NULL) ? - ResourceDescToResourceClass(*resDesc) : HeapFlagsToResourceClass(allocDesc.ExtraHeapFlags); - const UINT defaultPoolIndex = CalcDefaultPoolIndex(allocDesc, resourceClass); - if(defaultPoolIndex != UINT32_MAX) - { - outBlockVector = m_BlockVectors[defaultPoolIndex]; - const UINT64 preferredBlockSize = outBlockVector->GetPreferredBlockSize(); - if(allocSize > preferredBlockSize) - { - outBlockVector = NULL; - } - else if(allocSize > preferredBlockSize / 2) - { - // Heuristics: Allocate committed memory if requested size if greater than half of preferred block size. - outPreferCommitted = true; - } - } - - const D3D12_HEAP_FLAGS extraHeapFlags = allocDesc.ExtraHeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS; - if(outBlockVector != NULL && extraHeapFlags != 0) - { - outBlockVector = NULL; - } - } - - if((allocDesc.Flags & ALLOCATION_FLAG_COMMITTED) != 0 || - m_AlwaysCommitted) - { - outBlockVector = NULL; - } - if((allocDesc.Flags & ALLOCATION_FLAG_NEVER_ALLOCATE) != 0) - { - outCommittedAllocationParams.m_List = NULL; - } - - if(resDesc != NULL && !outPreferCommitted && PrefersCommittedAllocation(*resDesc)) - { - outPreferCommitted = true; - } - - return (outBlockVector != NULL || outCommittedAllocationParams.m_List != NULL) ? S_OK : E_INVALIDARG; -} - -UINT AllocatorPimpl::CalcDefaultPoolCount() const -{ - if(SupportsResourceHeapTier2()) - { - return 3; - } - else - { - return 9; - } -} - -UINT AllocatorPimpl::CalcDefaultPoolIndex(const ALLOCATION_DESC& allocDesc, ResourceClass resourceClass) const -{ - const D3D12_HEAP_FLAGS extraHeapFlags = allocDesc.ExtraHeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS; - if(extraHeapFlags != 0) - { - return UINT32_MAX; - } - - UINT poolIndex = UINT_MAX; - switch(allocDesc.HeapType) - { - case D3D12_HEAP_TYPE_DEFAULT: poolIndex = 0; break; - case D3D12_HEAP_TYPE_UPLOAD: poolIndex = 1; break; - case D3D12_HEAP_TYPE_READBACK: poolIndex = 2; break; - default: D3D12MA_ASSERT(0); - } - - if(SupportsResourceHeapTier2()) - return poolIndex; - else - { - switch(resourceClass) - { - case ResourceClass::Buffer: - return poolIndex * 3; - case ResourceClass::Non_RT_DS_Texture: - return poolIndex * 3 + 1; - case ResourceClass::RT_DS_Texture: - return poolIndex * 3 + 2; - default: - return UINT32_MAX; - } - } -} - -void AllocatorPimpl::CalcDefaultPoolParams(D3D12_HEAP_TYPE& outHeapType, D3D12_HEAP_FLAGS& outHeapFlags, UINT index) const -{ - outHeapType = D3D12_HEAP_TYPE_DEFAULT; - outHeapFlags = D3D12_HEAP_FLAG_NONE; - - if(!SupportsResourceHeapTier2()) - { - switch(index % 3) - { - case 0: - outHeapFlags = D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; - break; - case 1: - outHeapFlags = D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES; - break; - case 2: - outHeapFlags = D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; - break; - } - - index /= 3; - } - - switch(index) - { - case 0: - outHeapType = D3D12_HEAP_TYPE_DEFAULT; - break; - case 1: - outHeapType = D3D12_HEAP_TYPE_UPLOAD; - break; - case 2: - outHeapType = D3D12_HEAP_TYPE_READBACK; - break; - default: - D3D12MA_ASSERT(0); - } -} - -void AllocatorPimpl::RegisterPool(Pool* pool, D3D12_HEAP_TYPE heapType) -{ - const UINT heapTypeIndex = HeapTypeToIndex(heapType); - - MutexLockWrite lock(m_PoolsMutex[heapTypeIndex], m_UseMutex); - m_Pools[heapTypeIndex].PushBack(pool->m_Pimpl); -} - -void AllocatorPimpl::UnregisterPool(Pool* pool, D3D12_HEAP_TYPE heapType) -{ - const UINT heapTypeIndex = HeapTypeToIndex(heapType); - - MutexLockWrite lock(m_PoolsMutex[heapTypeIndex], m_UseMutex); - m_Pools[heapTypeIndex].Remove(pool->m_Pimpl); -} - void AllocatorPimpl::FreeCommittedMemory(Allocation* allocation) { D3D12MA_ASSERT(allocation && allocation->m_PackedData.GetType() == Allocation::TYPE_COMMITTED); @@ -7734,17 +6664,17 @@ void AllocatorPimpl::SetCurrentFrameIndex(UINT frameIndex) void AllocatorPimpl::CalculateStatistics(TotalStatistics& outStats) { // Init stats - for(size_t i = 0; i < HEAP_TYPE_COUNT; i++) + for (size_t i = 0; i < HEAP_TYPE_COUNT; i++) ClearDetailedStatistics(outStats.HeapType[i]); - for(size_t i = 0; i < DXGI_MEMORY_SEGMENT_GROUP_COUNT; i++) + for (size_t i = 0; i < DXGI_MEMORY_SEGMENT_GROUP_COUNT; i++) ClearDetailedStatistics(outStats.MemorySegmentGroup[i]); ClearDetailedStatistics(outStats.Total); // Process default pools. 3 standard heap types only. Add them to outStats.HeapType[i]. - if(SupportsResourceHeapTier2()) + if (SupportsResourceHeapTier2()) { // DEFAULT, UPLOAD, READBACK. - for(size_t heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) + for (size_t heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) { BlockVector* const pBlockVector = m_BlockVectors[heapTypeIndex]; D3D12MA_ASSERT(pBlockVector); @@ -7754,9 +6684,9 @@ void AllocatorPimpl::CalculateStatistics(TotalStatistics& outStats) else { // DEFAULT, UPLOAD, READBACK. - for(size_t heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) + for (size_t heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) { - for(size_t heapSubType = 0; heapSubType < 3; ++heapSubType) + for (size_t heapSubType = 0; heapSubType < 3; ++heapSubType) { BlockVector* const pBlockVector = m_BlockVectors[heapTypeIndex * 3 + heapSubType]; D3D12MA_ASSERT(pBlockVector); @@ -7778,11 +6708,11 @@ void AllocatorPimpl::CalculateStatistics(TotalStatistics& outStats) // Process custom pools. DetailedStatistics tmpStats; - for(size_t heapTypeIndex = 0; heapTypeIndex < HEAP_TYPE_COUNT; ++heapTypeIndex) + for (size_t heapTypeIndex = 0; heapTypeIndex < HEAP_TYPE_COUNT; ++heapTypeIndex) { MutexLockRead lock(m_PoolsMutex[heapTypeIndex], m_UseMutex); PoolList& poolList = m_Pools[heapTypeIndex]; - for(PoolPimpl* pool = poolList.Front(); pool != NULL; pool = poolList.GetNext(pool)) + for (PoolPimpl* pool = poolList.Front(); pool != NULL; pool = poolList.GetNext(pool)) { const D3D12_HEAP_PROPERTIES& poolHeapProps = pool->GetDesc().HeapProperties; ClearDetailedStatistics(tmpStats); @@ -7795,7 +6725,7 @@ void AllocatorPimpl::CalculateStatistics(TotalStatistics& outStats) } // Process committed allocations. 3 standard heap types only. - for(UINT heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) + for (UINT heapTypeIndex = 0; heapTypeIndex < STANDARD_HEAP_TYPE_COUNT; ++heapTypeIndex) { ClearDetailedStatistics(tmpStats); m_CommittedAllocations[heapTypeIndex].AddDetailedStatistics(tmpStats); @@ -7839,15 +6769,15 @@ void AllocatorPimpl::CalculateStatistics(TotalStatistics& outStats) void AllocatorPimpl::GetBudget(Budget* outLocalBudget, Budget* outNonLocalBudget) { - if(outLocalBudget) + if (outLocalBudget) m_Budget.GetStatistics(outLocalBudget->Stats, DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY); - if(outNonLocalBudget) + if (outNonLocalBudget) m_Budget.GetStatistics(outNonLocalBudget->Stats, DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY); #if D3D12MA_DXGI_1_4 - if(m_Adapter3) + if (m_Adapter3) { - if(!m_Budget.ShouldUpdateBudget()) + if (!m_Budget.ShouldUpdateBudget()) { m_Budget.GetBudget(m_UseMutex, outLocalBudget ? &outLocalBudget->UsageBytes : NULL, @@ -7864,12 +6794,12 @@ void AllocatorPimpl::GetBudget(Budget* outLocalBudget, Budget* outNonLocalBudget else #endif { - if(outLocalBudget) + if (outLocalBudget) { outLocalBudget->UsageBytes = outLocalBudget->Stats.BlockBytes; outLocalBudget->BudgetBytes = GetMemoryCapacity(DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY) * 8 / 10; // 80% heuristics. } - if(outNonLocalBudget) + if (outNonLocalBudget) { outNonLocalBudget->UsageBytes = outNonLocalBudget->Stats.BlockBytes; outNonLocalBudget->BudgetBytes = GetMemoryCapacity(DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY) * 8 / 10; // 80% heuristics. @@ -7879,7 +6809,7 @@ void AllocatorPimpl::GetBudget(Budget* outLocalBudget, Budget* outNonLocalBudget void AllocatorPimpl::GetBudgetForHeapType(Budget& outBudget, D3D12_HEAP_TYPE heapType) { - switch(heapType) + switch (heapType) { case D3D12_HEAP_TYPE_DEFAULT: GetBudget(&outBudget, NULL); @@ -7892,39 +6822,6 @@ void AllocatorPimpl::GetBudgetForHeapType(Budget& outBudget, D3D12_HEAP_TYPE hea } } -static void AddDetailedStatisticsInfoToJson(JsonWriter& json, const DetailedStatistics& stats) -{ - json.BeginObject(); - json.WriteString(L"BlockCount"); - json.WriteNumber(stats.Stats.BlockCount); - json.WriteString(L"AllocationCount"); - json.WriteNumber(stats.Stats.AllocationCount); - json.WriteString(L"UnusedRangeCount"); - json.WriteNumber(stats.UnusedRangeCount); - json.WriteString(L"BlockBytes"); - json.WriteNumber(stats.Stats.BlockBytes); - json.WriteString(L"AllocationBytes"); - json.WriteNumber(stats.Stats.AllocationBytes); - - json.WriteString(L"AllocationSize"); - json.BeginObject(true); - json.WriteString(L"Min"); - json.WriteNumber(stats.AllocationSizeMin); - json.WriteString(L"Max"); - json.WriteNumber(stats.AllocationSizeMax); - json.EndObject(); - - json.WriteString(L"UnusedRangeSize"); - json.BeginObject(true); - json.WriteString(L"Min"); - json.WriteNumber(stats.UnusedRangeSizeMin); - json.WriteString(L"Max"); - json.WriteNumber(stats.UnusedRangeSizeMax); - json.EndObject(); - - json.EndObject(); -} - void AllocatorPimpl::BuildStatsString(WCHAR** ppStatsString, BOOL DetailedMap) { StringBuilder sb(GetAllocs()); @@ -7938,13 +6835,13 @@ void AllocatorPimpl::BuildStatsString(WCHAR** ppStatsString, BOOL DetailedMap) CalculateStatistics(stats); json.BeginObject(); - + json.WriteString(L"Total"); - AddDetailedStatisticsInfoToJson(json, stats.Total); + json.AddDetailedStatisticsInfoObject(stats.Total); for (size_t heapType = 0; heapType < HEAP_TYPE_COUNT; ++heapType) { json.WriteString(HeapTypeNames[heapType]); - AddDetailedStatisticsInfoToJson(json, stats.HeapType[heapType]); + json.AddDetailedStatisticsInfoObject(stats.HeapType[heapType]); } json.WriteString(L"Budget"); @@ -8077,10 +6974,379 @@ void AllocatorPimpl::FreeStatsString(WCHAR* pStatsString) Free(GetAllocs(), pStatsString); } +template +bool AllocatorPimpl::PrefersCommittedAllocation(const D3D12_RESOURCE_DESC_T& resourceDesc) +{ + // Intentional. It may change in the future. + return false; +} + +HRESULT AllocatorPimpl::AllocateCommittedResource( + const CommittedAllocationParameters& committedAllocParams, + UINT64 resourceSize, bool withinBudget, + const D3D12_RESOURCE_DESC* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE* pOptimizedClearValue, + Allocation** ppAllocation, REFIID riidResource, void** ppvResource) +{ + D3D12MA_ASSERT(committedAllocParams.IsValid()); + + if (withinBudget && + !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, resourceSize)) + { + return E_OUTOFMEMORY; + } + + ID3D12Resource* res = NULL; + /* D3D12 ERROR: + * ID3D12Device::CreateCommittedResource: + * When creating a committed resource, D3D12_HEAP_FLAGS must not have either + * D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES, + * D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES, + * nor D3D12_HEAP_FLAG_DENY_BUFFERS set. + * These flags will be set automatically to correspond with the committed resource type. + * + * [ STATE_CREATION ERROR #640: CREATERESOURCEANDHEAP_INVALIDHEAPMISCFLAGS] + */ + HRESULT hr; +#ifdef __ID3D12Device4_INTERFACE_DEFINED__ + if (m_Device4) + { + hr = m_Device4->CreateCommittedResource1( + &committedAllocParams.m_HeapProperties, + committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, + pResourceDesc, InitialResourceState, + pOptimizedClearValue, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&res)); + } + else +#endif + { + if (committedAllocParams.m_ProtectedSession == NULL) + { + hr = m_Device->CreateCommittedResource( + &committedAllocParams.m_HeapProperties, + committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, + pResourceDesc, InitialResourceState, + pOptimizedClearValue, D3D12MA_IID_PPV_ARGS(&res)); + } + else + hr = E_NOINTERFACE; + } + + if (SUCCEEDED(hr)) + { + if (ppvResource != NULL) + { + hr = res->QueryInterface(riidResource, ppvResource); + } + if (SUCCEEDED(hr)) + { + const BOOL wasZeroInitialized = TRUE; + Allocation* alloc = m_AllocationObjectAllocator.Allocate(this, resourceSize, wasZeroInitialized); + alloc->InitCommitted(committedAllocParams.m_List); + alloc->SetResource(res, pResourceDesc); + + *ppAllocation = alloc; + + committedAllocParams.m_List->Register(alloc); + + const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); + m_Budget.AddBlock(memSegmentGroup, resourceSize); + m_Budget.AddAllocation(memSegmentGroup, resourceSize); + } + else + { + res->Release(); + } + } + return hr; +} + +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ +HRESULT AllocatorPimpl::AllocateCommittedResource2( + const CommittedAllocationParameters& committedAllocParams, + UINT64 resourceSize, bool withinBudget, + const D3D12_RESOURCE_DESC1* pResourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, const D3D12_CLEAR_VALUE* pOptimizedClearValue, + Allocation** ppAllocation, REFIID riidResource, void** ppvResource) +{ + D3D12MA_ASSERT(committedAllocParams.IsValid()); + + if (m_Device8 == NULL) + { + return E_NOINTERFACE; + } + + if (withinBudget && + !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, resourceSize)) + { + return E_OUTOFMEMORY; + } + + ID3D12Resource* res = NULL; + HRESULT hr = m_Device8->CreateCommittedResource2( + &committedAllocParams.m_HeapProperties, + committedAllocParams.m_HeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS, // D3D12 ERROR: ID3D12Device::CreateCommittedResource: When creating a committed resource, D3D12_HEAP_FLAGS must not have either D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES, D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES, nor D3D12_HEAP_FLAG_DENY_BUFFERS set. These flags will be set automatically to correspond with the committed resource type. [ STATE_CREATION ERROR #640: CREATERESOURCEANDHEAP_INVALIDHEAPMISCFLAGS] + pResourceDesc, InitialResourceState, + pOptimizedClearValue, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&res)); + if (SUCCEEDED(hr)) + { + if (ppvResource != NULL) + { + hr = res->QueryInterface(riidResource, ppvResource); + } + if (SUCCEEDED(hr)) + { + const BOOL wasZeroInitialized = TRUE; + Allocation* alloc = m_AllocationObjectAllocator.Allocate(this, resourceSize, wasZeroInitialized); + alloc->InitCommitted(committedAllocParams.m_List); + alloc->SetResource(res, pResourceDesc); + + *ppAllocation = alloc; + + committedAllocParams.m_List->Register(alloc); + + const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); + m_Budget.AddBlock(memSegmentGroup, resourceSize); + m_Budget.AddAllocation(memSegmentGroup, resourceSize); + } + else + { + res->Release(); + } + } + return hr; +} +#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ + +HRESULT AllocatorPimpl::AllocateHeap( + const CommittedAllocationParameters& committedAllocParams, + const D3D12_RESOURCE_ALLOCATION_INFO& allocInfo, bool withinBudget, + Allocation** ppAllocation) +{ + D3D12MA_ASSERT(committedAllocParams.IsValid()); + + *ppAllocation = nullptr; + + if (withinBudget && + !NewAllocationWithinBudget(committedAllocParams.m_HeapProperties.Type, allocInfo.SizeInBytes)) + { + return E_OUTOFMEMORY; + } + + D3D12_HEAP_DESC heapDesc = {}; + heapDesc.SizeInBytes = allocInfo.SizeInBytes; + heapDesc.Properties = committedAllocParams.m_HeapProperties; + heapDesc.Alignment = allocInfo.Alignment; + heapDesc.Flags = committedAllocParams.m_HeapFlags; + + HRESULT hr; + ID3D12Heap* heap = nullptr; +#ifdef __ID3D12Device4_INTERFACE_DEFINED__ + if (m_Device4) + hr = m_Device4->CreateHeap1(&heapDesc, committedAllocParams.m_ProtectedSession, D3D12MA_IID_PPV_ARGS(&heap)); + else +#endif + { + if (committedAllocParams.m_ProtectedSession == NULL) + hr = m_Device->CreateHeap(&heapDesc, D3D12MA_IID_PPV_ARGS(&heap)); + else + hr = E_NOINTERFACE; + } + + if (SUCCEEDED(hr)) + { + const BOOL wasZeroInitialized = TRUE; + (*ppAllocation) = m_AllocationObjectAllocator.Allocate(this, allocInfo.SizeInBytes, wasZeroInitialized); + (*ppAllocation)->InitHeap(committedAllocParams.m_List, heap); + committedAllocParams.m_List->Register(*ppAllocation); + + const UINT memSegmentGroup = HeapPropertiesToMemorySegmentGroup(committedAllocParams.m_HeapProperties); + m_Budget.AddBlock(memSegmentGroup, allocInfo.SizeInBytes); + m_Budget.AddAllocation(memSegmentGroup, allocInfo.SizeInBytes); + } + return hr; +} + +template +HRESULT AllocatorPimpl::CalcAllocationParams(const ALLOCATION_DESC& allocDesc, UINT64 allocSize, + const D3D12_RESOURCE_DESC_T* resDesc, + BlockVector*& outBlockVector, CommittedAllocationParameters& outCommittedAllocationParams, bool& outPreferCommitted) +{ + outBlockVector = NULL; + outCommittedAllocationParams = CommittedAllocationParameters(); + outPreferCommitted = false; + + if (allocDesc.CustomPool != NULL) + { + PoolPimpl* const pool = allocDesc.CustomPool->m_Pimpl; + + outBlockVector = pool->GetBlockVector(); + + outCommittedAllocationParams.m_ProtectedSession = pool->GetDesc().pProtectedSession; + outCommittedAllocationParams.m_HeapProperties = pool->GetDesc().HeapProperties; + outCommittedAllocationParams.m_HeapFlags = pool->GetDesc().HeapFlags; + outCommittedAllocationParams.m_List = pool->GetCommittedAllocationList(); + } + else + { + if (!IsHeapTypeStandard(allocDesc.HeapType)) + { + return E_INVALIDARG; + } + + outCommittedAllocationParams.m_HeapProperties = StandardHeapTypeToHeapProperties(allocDesc.HeapType); + outCommittedAllocationParams.m_HeapFlags = allocDesc.ExtraHeapFlags; + outCommittedAllocationParams.m_List = &m_CommittedAllocations[HeapTypeToIndex(allocDesc.HeapType)]; + + const ResourceClass resourceClass = (resDesc != NULL) ? + ResourceDescToResourceClass(*resDesc) : HeapFlagsToResourceClass(allocDesc.ExtraHeapFlags); + const UINT defaultPoolIndex = CalcDefaultPoolIndex(allocDesc, resourceClass); + if (defaultPoolIndex != UINT32_MAX) + { + outBlockVector = m_BlockVectors[defaultPoolIndex]; + const UINT64 preferredBlockSize = outBlockVector->GetPreferredBlockSize(); + if (allocSize > preferredBlockSize) + { + outBlockVector = NULL; + } + else if (allocSize > preferredBlockSize / 2) + { + // Heuristics: Allocate committed memory if requested size if greater than half of preferred block size. + outPreferCommitted = true; + } + } + + const D3D12_HEAP_FLAGS extraHeapFlags = allocDesc.ExtraHeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS; + if (outBlockVector != NULL && extraHeapFlags != 0) + { + outBlockVector = NULL; + } + } + + if ((allocDesc.Flags & ALLOCATION_FLAG_COMMITTED) != 0 || + m_AlwaysCommitted) + { + outBlockVector = NULL; + } + if ((allocDesc.Flags & ALLOCATION_FLAG_NEVER_ALLOCATE) != 0) + { + outCommittedAllocationParams.m_List = NULL; + } + + if (resDesc != NULL && !outPreferCommitted && PrefersCommittedAllocation(*resDesc)) + { + outPreferCommitted = true; + } + + return (outBlockVector != NULL || outCommittedAllocationParams.m_List != NULL) ? S_OK : E_INVALIDARG; +} + +UINT AllocatorPimpl::CalcDefaultPoolCount() const +{ + if (SupportsResourceHeapTier2()) + { + return 3; + } + else + { + return 9; + } +} + +UINT AllocatorPimpl::CalcDefaultPoolIndex(const ALLOCATION_DESC& allocDesc, ResourceClass resourceClass) const +{ + const D3D12_HEAP_FLAGS extraHeapFlags = allocDesc.ExtraHeapFlags & ~RESOURCE_CLASS_HEAP_FLAGS; + if (extraHeapFlags != 0) + { + return UINT32_MAX; + } + + UINT poolIndex = UINT_MAX; + switch (allocDesc.HeapType) + { + case D3D12_HEAP_TYPE_DEFAULT: poolIndex = 0; break; + case D3D12_HEAP_TYPE_UPLOAD: poolIndex = 1; break; + case D3D12_HEAP_TYPE_READBACK: poolIndex = 2; break; + default: D3D12MA_ASSERT(0); + } + + if (SupportsResourceHeapTier2()) + return poolIndex; + else + { + switch (resourceClass) + { + case ResourceClass::Buffer: + return poolIndex * 3; + case ResourceClass::Non_RT_DS_Texture: + return poolIndex * 3 + 1; + case ResourceClass::RT_DS_Texture: + return poolIndex * 3 + 2; + default: + return UINT32_MAX; + } + } +} + +void AllocatorPimpl::CalcDefaultPoolParams(D3D12_HEAP_TYPE& outHeapType, D3D12_HEAP_FLAGS& outHeapFlags, UINT index) const +{ + outHeapType = D3D12_HEAP_TYPE_DEFAULT; + outHeapFlags = D3D12_HEAP_FLAG_NONE; + + if (!SupportsResourceHeapTier2()) + { + switch (index % 3) + { + case 0: + outHeapFlags = D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; + break; + case 1: + outHeapFlags = D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_RT_DS_TEXTURES; + break; + case 2: + outHeapFlags = D3D12_HEAP_FLAG_DENY_BUFFERS | D3D12_HEAP_FLAG_DENY_NON_RT_DS_TEXTURES; + break; + } + + index /= 3; + } + + switch (index) + { + case 0: + outHeapType = D3D12_HEAP_TYPE_DEFAULT; + break; + case 1: + outHeapType = D3D12_HEAP_TYPE_UPLOAD; + break; + case 2: + outHeapType = D3D12_HEAP_TYPE_READBACK; + break; + default: + D3D12MA_ASSERT(0); + } +} + +void AllocatorPimpl::RegisterPool(Pool* pool, D3D12_HEAP_TYPE heapType) +{ + const UINT heapTypeIndex = HeapTypeToIndex(heapType); + + MutexLockWrite lock(m_PoolsMutex[heapTypeIndex], m_UseMutex); + m_Pools[heapTypeIndex].PushBack(pool->m_Pimpl); +} + +void AllocatorPimpl::UnregisterPool(Pool* pool, D3D12_HEAP_TYPE heapType) +{ + const UINT heapTypeIndex = HeapTypeToIndex(heapType); + + MutexLockWrite lock(m_PoolsMutex[heapTypeIndex], m_UseMutex); + m_Pools[heapTypeIndex].Remove(pool->m_Pimpl); +} + HRESULT AllocatorPimpl::UpdateD3D12Budget() { #if D3D12MA_DXGI_1_4 - if(m_Adapter3) + if (m_Adapter3) return m_Budget.UpdateBudget(m_Adapter3, m_UseMutex); else return E_NOINTERFACE; @@ -8108,22 +7374,22 @@ D3D12_RESOURCE_ALLOCATION_INFO AllocatorPimpl::GetResourceAllocationInfo(D3D12_R { /* Optional optimization: Microsoft documentation says: https://docs.microsoft.com/en-us/windows/win32/api/d3d12/nf-d3d12-id3d12device-getresourceallocationinfo - + Your application can forgo using GetResourceAllocationInfo for buffer resources (D3D12_RESOURCE_DIMENSION_BUFFER). Buffers have the same size on all adapters, which is merely the smallest multiple of 64KB that's greater or equal to D3D12_RESOURCE_DESC::Width. */ - if(inOutResourceDesc.Alignment == 0 && + if (inOutResourceDesc.Alignment == 0 && inOutResourceDesc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER) { return { AlignUp(inOutResourceDesc.Width, D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT), // SizeInBytes - D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT}; // Alignment + D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT }; // Alignment } #if D3D12MA_USE_SMALL_RESOURCE_PLACEMENT_ALIGNMENT - if(inOutResourceDesc.Alignment == 0 && + if (inOutResourceDesc.Alignment == 0 && inOutResourceDesc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D && (inOutResourceDesc.Flags & (D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL)) == 0 #if D3D12MA_USE_SMALL_RESOURCE_PLACEMENT_ALIGNMENT == 1 @@ -8141,7 +7407,7 @@ D3D12_RESOURCE_ALLOCATION_INFO AllocatorPimpl::GetResourceAllocationInfo(D3D12_R inOutResourceDesc.Alignment = smallAlignmentToTry; const D3D12_RESOURCE_ALLOCATION_INFO smallAllocInfo = GetResourceAllocationInfoNative(inOutResourceDesc); // Check if alignment requested has been granted. - if(smallAllocInfo.Alignment == smallAlignmentToTry) + if (smallAllocInfo.Alignment == smallAlignmentToTry) { return smallAllocInfo; } @@ -8178,49 +7444,905 @@ void AllocatorPimpl::WriteBudgetToJson(JsonWriter& json, const Budget& budget) } json.EndObject(); } +#endif // _D3D12MA_ALLOCATOR_PIMPL +#endif // _D3D12MA_ALLOCATOR_PIMPL -UINT AllocatorPimpl::StandardHeapTypeToMemorySegmentGroup(D3D12_HEAP_TYPE heapType) const +#ifndef _D3D12MA_VIRTUAL_BLOCK_PIMPL +class VirtualBlockPimpl { - D3D12MA_ASSERT(IsHeapTypeStandard(heapType)); - if(IsUMA()) - return DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY; - return heapType == D3D12_HEAP_TYPE_DEFAULT ? - DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY : DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY; -} +public: + const ALLOCATION_CALLBACKS m_AllocationCallbacks; + const UINT64 m_Size; + BlockMetadata* m_Metadata; -UINT AllocatorPimpl::HeapPropertiesToMemorySegmentGroup(const D3D12_HEAP_PROPERTIES& heapProps) const -{ - if(IsUMA()) - return DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY; - if(heapProps.MemoryPoolPreference == D3D12_MEMORY_POOL_UNKNOWN) - return StandardHeapTypeToMemorySegmentGroup(heapProps.Type); - return heapProps.MemoryPoolPreference == D3D12_MEMORY_POOL_L1 ? - DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY : DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY; -} + VirtualBlockPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc); + ~VirtualBlockPimpl(); +}; -UINT64 AllocatorPimpl::GetMemoryCapacity(UINT memorySegmentGroup) const +#ifndef _D3D12MA_VIRTUAL_BLOCK_PIMPL_FUNCTIONS +VirtualBlockPimpl::VirtualBlockPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc) + : m_AllocationCallbacks(allocationCallbacks), m_Size(desc.Size) { - switch(memorySegmentGroup) + switch (desc.Flags & VIRTUAL_BLOCK_FLAG_ALGORITHM_MASK) { - case DXGI_MEMORY_SEGMENT_GROUP_LOCAL_COPY: - return IsUMA() ? - m_AdapterDesc.DedicatedVideoMemory + m_AdapterDesc.SharedSystemMemory : m_AdapterDesc.DedicatedVideoMemory; - case DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL_COPY: - return IsUMA() ? 0 : m_AdapterDesc.SharedSystemMemory; + case VIRTUAL_BLOCK_FLAG_ALGORITHM_LINEAR: + m_Metadata = D3D12MA_NEW(allocationCallbacks, BlockMetadata_Linear)(&m_AllocationCallbacks, true); + break; default: D3D12MA_ASSERT(0); - return UINT64_MAX; + case 0: + m_Metadata = D3D12MA_NEW(allocationCallbacks, BlockMetadata_TLSF)(&m_AllocationCallbacks, true); + break; + } + m_Metadata->Init(m_Size); +} + +VirtualBlockPimpl::~VirtualBlockPimpl() +{ + D3D12MA_DELETE(m_AllocationCallbacks, m_Metadata); +} +#endif // _D3D12MA_VIRTUAL_BLOCK_PIMPL_FUNCTIONS +#endif // _D3D12MA_VIRTUAL_BLOCK_PIMPL + + +#ifndef _D3D12MA_MEMORY_BLOCK_FUNCTIONS +MemoryBlock::MemoryBlock( + AllocatorPimpl* allocator, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 size, + UINT id) + : m_Allocator(allocator), + m_HeapProps(heapProps), + m_HeapFlags(heapFlags), + m_Size(size), + m_Id(id) {} + +MemoryBlock::~MemoryBlock() +{ + if (m_Heap) + { + m_Heap->Release(); + m_Allocator->m_Budget.RemoveBlock( + m_Allocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), m_Size); } } -//////////////////////////////////////////////////////////////////////////////// -// Public but internal class IUnknownImpl implementation +HRESULT MemoryBlock::Init(ID3D12ProtectedResourceSession* pProtectedSession) +{ + D3D12MA_ASSERT(m_Heap == NULL && m_Size > 0); + D3D12_HEAP_DESC heapDesc = {}; + heapDesc.SizeInBytes = m_Size; + heapDesc.Properties = m_HeapProps; + heapDesc.Alignment = HeapFlagsToAlignment(m_HeapFlags); + heapDesc.Flags = m_HeapFlags; + + HRESULT hr; +#ifdef __ID3D12Device4_INTERFACE_DEFINED__ + ID3D12Device4* const device4 = m_Allocator->GetDevice4(); + if (device4) + hr = m_Allocator->GetDevice4()->CreateHeap1(&heapDesc, pProtectedSession, D3D12MA_IID_PPV_ARGS(&m_Heap)); + else +#endif + { + if (pProtectedSession == NULL) + hr = m_Allocator->GetDevice()->CreateHeap(&heapDesc, D3D12MA_IID_PPV_ARGS(&m_Heap)); + else + hr = E_NOINTERFACE; + } + + if (SUCCEEDED(hr)) + { + m_Allocator->m_Budget.AddBlock( + m_Allocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), m_Size); + } + return hr; +} +#endif // _D3D12MA_MEMORY_BLOCK_FUNCTIONS + +#ifndef _D3D12MA_NORMAL_BLOCK_FUNCTIONS +NormalBlock::NormalBlock( + AllocatorPimpl* allocator, + BlockVector* blockVector, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 size, + UINT id) + : MemoryBlock(allocator, heapProps, heapFlags, size, id), + m_pMetadata(NULL), + m_BlockVector(blockVector) {} + +NormalBlock::~NormalBlock() +{ + if (m_pMetadata != NULL) + { + // THIS IS THE MOST IMPORTANT ASSERT IN THE ENTIRE LIBRARY! + // Hitting it means you have some memory leak - unreleased Allocation objects. + D3D12MA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!"); + + D3D12MA_DELETE(m_Allocator->GetAllocs(), m_pMetadata); + } +} + +HRESULT NormalBlock::Init(UINT32 algorithm, ID3D12ProtectedResourceSession* pProtectedSession) +{ + HRESULT hr = MemoryBlock::Init(pProtectedSession); + if (FAILED(hr)) + { + return hr; + } + + switch (algorithm) + { + case POOL_FLAG_ALGORITHM_LINEAR: + m_pMetadata = D3D12MA_NEW(m_Allocator->GetAllocs(), BlockMetadata_Linear)(&m_Allocator->GetAllocs(), false); + break; + default: + D3D12MA_ASSERT(0); + case 0: + m_pMetadata = D3D12MA_NEW(m_Allocator->GetAllocs(), BlockMetadata_TLSF)(&m_Allocator->GetAllocs(), false); + break; + } + m_pMetadata->Init(m_Size); + + return hr; +} + +bool NormalBlock::Validate() const +{ + D3D12MA_VALIDATE(GetHeap() && + m_pMetadata && + m_pMetadata->GetSize() != 0 && + m_pMetadata->GetSize() == GetSize()); + return m_pMetadata->Validate(); +} +#endif // _D3D12MA_NORMAL_BLOCK_FUNCTIONS + +#ifndef _D3D12MA_COMMITTED_ALLOCATION_LIST_FUNCTIONS +void CommittedAllocationList::Init(bool useMutex, D3D12_HEAP_TYPE heapType, PoolPimpl* pool) +{ + m_UseMutex = useMutex; + m_HeapType = heapType; + m_Pool = pool; +} + +CommittedAllocationList::~CommittedAllocationList() +{ + if (!m_AllocationList.IsEmpty()) + { + D3D12MA_ASSERT(0 && "Unfreed committed allocations found!"); + } +} + +UINT CommittedAllocationList::GetMemorySegmentGroup(AllocatorPimpl* allocator) const +{ + if (m_Pool) + return allocator->HeapPropertiesToMemorySegmentGroup(m_Pool->GetDesc().HeapProperties); + else + return allocator->StandardHeapTypeToMemorySegmentGroup(m_HeapType); +} + +void CommittedAllocationList::AddStatistics(Statistics& inoutStats) +{ + MutexLockRead lock(m_Mutex, m_UseMutex); + + for (Allocation* alloc = m_AllocationList.Front(); + alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) + { + const UINT64 size = alloc->GetSize(); + inoutStats.BlockCount++; + inoutStats.AllocationCount++; + inoutStats.BlockBytes += size; + inoutStats.AllocationBytes += size; + } +} + +void CommittedAllocationList::AddDetailedStatistics(DetailedStatistics& inoutStats) +{ + MutexLockRead lock(m_Mutex, m_UseMutex); + + for (Allocation* alloc = m_AllocationList.Front(); + alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) + { + const UINT64 size = alloc->GetSize(); + inoutStats.Stats.BlockCount++; + inoutStats.Stats.BlockBytes += size; + AddDetailedStatisticsAllocation(inoutStats, size); + } +} + +void CommittedAllocationList::BuildStatsString(JsonWriter& json) +{ + MutexLockRead lock(m_Mutex, m_UseMutex); + + json.BeginArray(); + for (Allocation* alloc = m_AllocationList.Front(); + alloc != NULL; alloc = m_AllocationList.GetNext(alloc)) + { + json.BeginObject(true); + json.AddAllocationToObject(*alloc); + json.EndObject(); + } + json.EndArray(); +} + +void CommittedAllocationList::Register(Allocation* alloc) +{ + MutexLockWrite lock(m_Mutex, m_UseMutex); + m_AllocationList.PushBack(alloc); +} + +void CommittedAllocationList::Unregister(Allocation* alloc) +{ + MutexLockWrite lock(m_Mutex, m_UseMutex); + m_AllocationList.Remove(alloc); +} +#endif // _D3D12MA_COMMITTED_ALLOCATION_LIST_FUNCTIONS + +#ifndef _D3D12MA_BLOCK_VECTOR_FUNCTIONS +BlockVector::BlockVector( + AllocatorPimpl* hAllocator, + const D3D12_HEAP_PROPERTIES& heapProps, + D3D12_HEAP_FLAGS heapFlags, + UINT64 preferredBlockSize, + size_t minBlockCount, + size_t maxBlockCount, + bool explicitBlockSize, + UINT64 minAllocationAlignment, + UINT32 algorithm, + ID3D12ProtectedResourceSession* pProtectedSession) + : m_hAllocator(hAllocator), + m_HeapProps(heapProps), + m_HeapFlags(heapFlags), + m_PreferredBlockSize(preferredBlockSize), + m_MinBlockCount(minBlockCount), + m_MaxBlockCount(maxBlockCount), + m_ExplicitBlockSize(explicitBlockSize), + m_MinAllocationAlignment(minAllocationAlignment), + m_Algorithm(algorithm), + m_ProtectedSession(pProtectedSession), + m_HasEmptyBlock(false), + m_Blocks(hAllocator->GetAllocs()), + m_NextBlockId(0) {} + +BlockVector::~BlockVector() +{ + for (size_t i = m_Blocks.size(); i--; ) + { + D3D12MA_DELETE(m_hAllocator->GetAllocs(), m_Blocks[i]); + } +} + +HRESULT BlockVector::CreateMinBlocks() +{ + for (size_t i = 0; i < m_MinBlockCount; ++i) + { + HRESULT hr = CreateBlock(m_PreferredBlockSize, NULL); + if (FAILED(hr)) + { + return hr; + } + } + return S_OK; +} + +bool BlockVector::IsEmpty() +{ + MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); + return m_Blocks.empty(); +} + +HRESULT BlockVector::Allocate( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + size_t allocationCount, + Allocation** pAllocations) +{ + size_t allocIndex; + HRESULT hr = S_OK; + + { + MutexLockWrite lock(m_Mutex, m_hAllocator->UseMutex()); + for (allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + hr = AllocatePage( + size, + alignment, + allocDesc, + pAllocations + allocIndex); + if (FAILED(hr)) + { + break; + } + } + } + + if (FAILED(hr)) + { + // Free all already created allocations. + while (allocIndex--) + { + Free(pAllocations[allocIndex]); + } + ZeroMemory(pAllocations, sizeof(Allocation*) * allocationCount); + } + + return hr; +} + +void BlockVector::Free(Allocation* hAllocation) +{ + NormalBlock* pBlockToDelete = NULL; + + bool budgetExceeded = false; + if (IsHeapTypeStandard(m_HeapProps.Type)) + { + Budget budget = {}; + m_hAllocator->GetBudgetForHeapType(budget, m_HeapProps.Type); + budgetExceeded = budget.UsageBytes >= budget.BudgetBytes; + } + + // Scope for lock. + { + MutexLockWrite lock(m_Mutex, m_hAllocator->UseMutex()); + + NormalBlock* pBlock = hAllocation->m_Placed.block; + + pBlock->m_pMetadata->Free(hAllocation->GetAllocHandle()); + D3D12MA_HEAVY_ASSERT(pBlock->Validate()); + + const size_t blockCount = m_Blocks.size(); + // pBlock became empty after this deallocation. + if (pBlock->m_pMetadata->IsEmpty()) + { + // Already has empty Allocation. We don't want to have two, so delete this one. + if ((m_HasEmptyBlock || budgetExceeded) && + blockCount > m_MinBlockCount) + { + pBlockToDelete = pBlock; + Remove(pBlock); + } + // We now have first empty block. + else + { + m_HasEmptyBlock = true; + } + } + // pBlock didn't become empty, but we have another empty block - find and free that one. + // (This is optional, heuristics.) + else if (m_HasEmptyBlock && blockCount > m_MinBlockCount) + { + NormalBlock* pLastBlock = m_Blocks.back(); + if (pLastBlock->m_pMetadata->IsEmpty()) + { + pBlockToDelete = pLastBlock; + m_Blocks.pop_back(); + m_HasEmptyBlock = false; + } + } + + IncrementallySortBlocks(); + } + + // Destruction of a free Allocation. Deferred until this point, outside of mutex + // lock, for performance reason. + if (pBlockToDelete != NULL) + { + D3D12MA_DELETE(m_hAllocator->GetAllocs(), pBlockToDelete); + } +} + +HRESULT BlockVector::CreateResource( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + const D3D12_RESOURCE_DESC& resourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource) +{ + HRESULT hr = Allocate(size, alignment, allocDesc, 1, ppAllocation); + if (SUCCEEDED(hr)) + { + ID3D12Resource* res = NULL; + hr = m_hAllocator->GetDevice()->CreatePlacedResource( + (*ppAllocation)->m_Placed.block->GetHeap(), + (*ppAllocation)->GetOffset(), + &resourceDesc, + InitialResourceState, + pOptimizedClearValue, + D3D12MA_IID_PPV_ARGS(&res)); + if (SUCCEEDED(hr)) + { + if (ppvResource != NULL) + { + hr = res->QueryInterface(riidResource, ppvResource); + } + if (SUCCEEDED(hr)) + { + (*ppAllocation)->SetResource(res, &resourceDesc); + } + else + { + res->Release(); + SAFE_RELEASE(*ppAllocation); + } + } + else + { + SAFE_RELEASE(*ppAllocation); + } + } + return hr; +} + +#ifdef __ID3D12Device8_INTERFACE_DEFINED__ +HRESULT BlockVector::CreateResource2( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + const D3D12_RESOURCE_DESC1& resourceDesc, + D3D12_RESOURCE_STATES InitialResourceState, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, + Allocation** ppAllocation, + REFIID riidResource, + void** ppvResource) +{ + ID3D12Device8* const device8 = m_hAllocator->GetDevice8(); + if (device8 == NULL) + { + return E_NOINTERFACE; + } + + HRESULT hr = Allocate(size, alignment, allocDesc, 1, ppAllocation); + if (SUCCEEDED(hr)) + { + ID3D12Resource* res = NULL; + hr = device8->CreatePlacedResource1( + (*ppAllocation)->m_Placed.block->GetHeap(), + (*ppAllocation)->GetOffset(), + &resourceDesc, + InitialResourceState, + pOptimizedClearValue, + D3D12MA_IID_PPV_ARGS(&res)); + if (SUCCEEDED(hr)) + { + if (ppvResource != NULL) + { + hr = res->QueryInterface(riidResource, ppvResource); + } + if (SUCCEEDED(hr)) + { + (*ppAllocation)->SetResource(res, &resourceDesc); + } + else + { + res->Release(); + SAFE_RELEASE(*ppAllocation); + } + } + else + { + SAFE_RELEASE(*ppAllocation); + } + } + return hr; +} +#endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ + +void BlockVector::AddStatistics(Statistics& inoutStats) +{ + MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); + + for (size_t i = 0; i < m_Blocks.size(); ++i) + { + const NormalBlock* const pBlock = m_Blocks[i]; + D3D12MA_ASSERT(pBlock); + D3D12MA_HEAVY_ASSERT(pBlock->Validate()); + pBlock->m_pMetadata->AddStatistics(inoutStats); + } +} + +void BlockVector::AddDetailedStatistics(DetailedStatistics& inoutStats) +{ + MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); + + for (size_t i = 0; i < m_Blocks.size(); ++i) + { + const NormalBlock* const pBlock = m_Blocks[i]; + D3D12MA_ASSERT(pBlock); + D3D12MA_HEAVY_ASSERT(pBlock->Validate()); + pBlock->m_pMetadata->AddDetailedStatistics(inoutStats); + } +} + +void BlockVector::WriteBlockInfoToJson(JsonWriter& json) +{ + MutexLockRead lock(m_Mutex, m_hAllocator->UseMutex()); + + json.BeginObject(); + + for (size_t i = 0, count = m_Blocks.size(); i < count; ++i) + { + const NormalBlock* const pBlock = m_Blocks[i]; + D3D12MA_ASSERT(pBlock); + D3D12MA_HEAVY_ASSERT(pBlock->Validate()); + json.BeginString(); + json.ContinueString(pBlock->GetId()); + json.EndString(); + + pBlock->m_pMetadata->WriteAllocationInfoToJson(json); + } + + json.EndObject(); +} + +UINT64 BlockVector::CalcSumBlockSize() const +{ + UINT64 result = 0; + for (size_t i = m_Blocks.size(); i--; ) + { + result += m_Blocks[i]->m_pMetadata->GetSize(); + } + return result; +} + +UINT64 BlockVector::CalcMaxBlockSize() const +{ + UINT64 result = 0; + for (size_t i = m_Blocks.size(); i--; ) + { + result = D3D12MA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize()); + if (result >= m_PreferredBlockSize) + { + break; + } + } + return result; +} + +void BlockVector::Remove(NormalBlock* pBlock) +{ + for (size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + { + if (m_Blocks[blockIndex] == pBlock) + { + m_Blocks.remove(blockIndex); + return; + } + } + D3D12MA_ASSERT(0); +} + +void BlockVector::IncrementallySortBlocks() +{ + // Bubble sort only until first swap. + for (size_t i = 1; i < m_Blocks.size(); ++i) + { + if (m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize()) + { + D3D12MA_SWAP(m_Blocks[i - 1], m_Blocks[i]); + return; + } + } +} + +HRESULT BlockVector::AllocatePage( + UINT64 size, + UINT64 alignment, + const ALLOCATION_DESC& allocDesc, + Allocation** pAllocation) +{ + // Early reject: requested allocation size is larger that maximum block size for this block vector. + if (size + D3D12MA_DEBUG_MARGIN > m_PreferredBlockSize) + { + return E_OUTOFMEMORY; + } + + UINT64 freeMemory = UINT64_MAX; + if (IsHeapTypeStandard(m_HeapProps.Type)) + { + Budget budget = {}; + m_hAllocator->GetBudgetForHeapType(budget, m_HeapProps.Type); + freeMemory = (budget.UsageBytes < budget.BudgetBytes) ? (budget.BudgetBytes - budget.UsageBytes) : 0; + } + + const bool canCreateNewBlock = + ((allocDesc.Flags & ALLOCATION_FLAG_NEVER_ALLOCATE) == 0) && + (m_Blocks.size() < m_MaxBlockCount) && + // Even if we don't have to stay within budget with this allocation, when the + // budget would be exceeded, we don't want to allocate new blocks, but always + // create resources as committed. + freeMemory >= size; + + // 1. Search existing allocations + { + // Forward order in m_Blocks - prefer blocks with smallest amount of free space. + for (size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + { + NormalBlock* const pCurrBlock = m_Blocks[blockIndex]; + D3D12MA_ASSERT(pCurrBlock); + HRESULT hr = AllocateFromBlock( + pCurrBlock, + size, + alignment, + allocDesc.Flags, + pAllocation); + if (SUCCEEDED(hr)) + { + return hr; + } + } + } + + // 2. Try to create new block. + if (canCreateNewBlock) + { + // Calculate optimal size for new block. + UINT64 newBlockSize = m_PreferredBlockSize; + UINT newBlockSizeShift = 0; + + if (!m_ExplicitBlockSize) + { + // Allocate 1/8, 1/4, 1/2 as first blocks. + const UINT64 maxExistingBlockSize = CalcMaxBlockSize(); + for (UINT i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i) + { + const UINT64 smallerNewBlockSize = newBlockSize / 2; + if (smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2) + { + newBlockSize = smallerNewBlockSize; + ++newBlockSizeShift; + } + else + { + break; + } + } + } + + size_t newBlockIndex = 0; + HRESULT hr = newBlockSize <= freeMemory ? + CreateBlock(newBlockSize, &newBlockIndex) : E_OUTOFMEMORY; + // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize. + if (!m_ExplicitBlockSize) + { + while (FAILED(hr) && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX) + { + const UINT64 smallerNewBlockSize = newBlockSize / 2; + if (smallerNewBlockSize >= size) + { + newBlockSize = smallerNewBlockSize; + ++newBlockSizeShift; + hr = newBlockSize <= freeMemory ? + CreateBlock(newBlockSize, &newBlockIndex) : E_OUTOFMEMORY; + } + else + { + break; + } + } + } + + if (SUCCEEDED(hr)) + { + NormalBlock* const pBlock = m_Blocks[newBlockIndex]; + D3D12MA_ASSERT(pBlock->m_pMetadata->GetSize() >= size); + + hr = AllocateFromBlock( + pBlock, + size, + alignment, + allocDesc.Flags, + pAllocation); + if (SUCCEEDED(hr)) + { + return hr; + } + else + { + // Allocation from new block failed, possibly due to D3D12MA_DEBUG_MARGIN or alignment. + return E_OUTOFMEMORY; + } + } + } + + return E_OUTOFMEMORY; +} + +HRESULT BlockVector::AllocateFromBlock( + NormalBlock* pBlock, + UINT64 size, + UINT64 alignment, + ALLOCATION_FLAGS allocFlags, + Allocation** pAllocation) +{ + alignment = D3D12MA_MAX(alignment, m_MinAllocationAlignment); + + AllocationRequest currRequest = {}; + if (pBlock->m_pMetadata->CreateAllocationRequest( + size, + alignment, + allocFlags & ALLOCATION_FLAG_UPPER_ADDRESS, + &currRequest)) + { + // We no longer have an empty Allocation. + if (pBlock->m_pMetadata->IsEmpty()) + { + m_HasEmptyBlock = false; + } + + *pAllocation = m_hAllocator->GetAllocationObjectAllocator().Allocate(m_hAllocator, size, currRequest.zeroInitialized); + pBlock->m_pMetadata->Alloc(currRequest, size, *pAllocation); + (*pAllocation)->InitPlaced(currRequest.allocHandle, alignment, pBlock); + D3D12MA_HEAVY_ASSERT(pBlock->Validate()); + m_hAllocator->m_Budget.AddAllocation(m_hAllocator->HeapPropertiesToMemorySegmentGroup(m_HeapProps), size); + return S_OK; + } + return E_OUTOFMEMORY; +} + +HRESULT BlockVector::CreateBlock( + UINT64 blockSize, + size_t* pNewBlockIndex) +{ + NormalBlock* const pBlock = D3D12MA_NEW(m_hAllocator->GetAllocs(), NormalBlock)( + m_hAllocator, + this, + m_HeapProps, + m_HeapFlags, + blockSize, + m_NextBlockId++); + HRESULT hr = pBlock->Init(m_Algorithm, m_ProtectedSession); + if (FAILED(hr)) + { + D3D12MA_DELETE(m_hAllocator->GetAllocs(), pBlock); + return hr; + } + + m_Blocks.push_back(pBlock); + if (pNewBlockIndex != NULL) + { + *pNewBlockIndex = m_Blocks.size() - 1; + } + + return hr; +} +#endif // _D3D12MA_BLOCK_VECTOR_FUNCTIONS + +#ifndef _D3D12MA_POOL_PIMPL_FUNCTIONS +PoolPimpl::PoolPimpl(AllocatorPimpl* allocator, const POOL_DESC& desc) + : m_Allocator(allocator), + m_Desc(desc), + m_BlockVector(NULL), + m_Name(NULL) +{ + const bool explicitBlockSize = desc.BlockSize != 0; + const UINT64 preferredBlockSize = explicitBlockSize ? desc.BlockSize : D3D12MA_DEFAULT_BLOCK_SIZE; + UINT maxBlockCount = desc.MaxBlockCount != 0 ? desc.MaxBlockCount : UINT_MAX; + +#ifndef __ID3D12Device4_INTERFACE_DEFINED__ + D3D12MA_ASSERT(m_Desc.pProtectedSession == NULL); +#endif + + m_BlockVector = D3D12MA_NEW(allocator->GetAllocs(), BlockVector)( + allocator, desc.HeapProperties, desc.HeapFlags, + preferredBlockSize, + desc.MinBlockCount, maxBlockCount, + explicitBlockSize, + D3D12MA_MAX(desc.MinAllocationAlignment, (UINT64)D3D12MA_DEBUG_ALIGNMENT), + desc.Flags & POOL_FLAG_ALGORITHM_MASK, + desc.pProtectedSession); +} + +PoolPimpl::~PoolPimpl() +{ + D3D12MA_ASSERT(m_PrevPool == NULL && m_NextPool == NULL); + FreeName(); + D3D12MA_DELETE(m_Allocator->GetAllocs(), m_BlockVector); +} + +HRESULT PoolPimpl::Init() +{ + m_CommittedAllocations.Init(m_Allocator->UseMutex(), m_Desc.HeapProperties.Type, this); + return m_BlockVector->CreateMinBlocks(); +} + +void PoolPimpl::GetStatistics(Statistics& outStats) +{ + ClearStatistics(outStats); + m_BlockVector->AddStatistics(outStats); + m_CommittedAllocations.AddStatistics(outStats); +} + +void PoolPimpl::CalculateStatistics(DetailedStatistics& outStats) +{ + ClearDetailedStatistics(outStats); + AddDetailedStatistics(outStats); +} + +void PoolPimpl::AddDetailedStatistics(DetailedStatistics& inoutStats) +{ + m_BlockVector->AddDetailedStatistics(inoutStats); + m_CommittedAllocations.AddDetailedStatistics(inoutStats); +} + +void PoolPimpl::SetName(LPCWSTR Name) +{ + FreeName(); + + if (Name) + { + const size_t nameCharCount = wcslen(Name) + 1; + m_Name = D3D12MA_NEW_ARRAY(m_Allocator->GetAllocs(), WCHAR, nameCharCount); + memcpy(m_Name, Name, nameCharCount * sizeof(WCHAR)); + } +} + +void PoolPimpl::FreeName() +{ + if (m_Name) + { + const size_t nameCharCount = wcslen(m_Name) + 1; + D3D12MA_DELETE_ARRAY(m_Allocator->GetAllocs(), m_Name, nameCharCount); + m_Name = NULL; + } +} +#endif // _D3D12MA_POOL_PIMPL_FUNCTIONS + + +#ifndef _D3D12MA_PUBLIC_FUNCTIONS +HRESULT CreateAllocator(const ALLOCATOR_DESC* pDesc, Allocator** ppAllocator) +{ + if (!pDesc || !ppAllocator || !pDesc->pDevice || !pDesc->pAdapter || + !(pDesc->PreferredBlockSize == 0 || (pDesc->PreferredBlockSize >= 16 && pDesc->PreferredBlockSize < 0x10000000000ull))) + { + D3D12MA_ASSERT(0 && "Invalid arguments passed to CreateAllocator."); + return E_INVALIDARG; + } + + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + + ALLOCATION_CALLBACKS allocationCallbacks; + SetupAllocationCallbacks(allocationCallbacks, pDesc->pAllocationCallbacks); + + *ppAllocator = D3D12MA_NEW(allocationCallbacks, Allocator)(allocationCallbacks, *pDesc); + HRESULT hr = (*ppAllocator)->m_Pimpl->Init(*pDesc); + if (FAILED(hr)) + { + D3D12MA_DELETE(allocationCallbacks, *ppAllocator); + *ppAllocator = NULL; + } + return hr; +} + +HRESULT CreateVirtualBlock(const VIRTUAL_BLOCK_DESC* pDesc, VirtualBlock** ppVirtualBlock) +{ + if (!pDesc || !ppVirtualBlock) + { + D3D12MA_ASSERT(0 && "Invalid arguments passed to CreateVirtualBlock."); + return E_INVALIDARG; + } + + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + + ALLOCATION_CALLBACKS allocationCallbacks; + SetupAllocationCallbacks(allocationCallbacks, pDesc->pAllocationCallbacks); + + *ppVirtualBlock = D3D12MA_NEW(allocationCallbacks, VirtualBlock)(allocationCallbacks, *pDesc); + return S_OK; +} + +#ifndef _D3D12MA_IUNKNOWN_IMPL_FUNCTIONS HRESULT STDMETHODCALLTYPE IUnknownImpl::QueryInterface(REFIID riid, void** ppvObject) { - if(ppvObject == NULL) + if (ppvObject == NULL) return E_POINTER; - if(riid == IID_IUnknown) + if (riid == IID_IUnknown) { ++m_RefCount; *ppvObject = this; @@ -8239,15 +8361,14 @@ ULONG STDMETHODCALLTYPE IUnknownImpl::Release() { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - const uint32_t newRefCount = --m_RefCount; - if(newRefCount == 0) + const uint32_t newRefCount = --m_RefCount; + if (newRefCount == 0) ReleaseThis(); return newRefCount; } +#endif // _D3D12MA_IUNKNOWN_IMPL_FUNCTIONS -//////////////////////////////////////////////////////////////////////////////// -// Public class Allocation implementation - +#ifndef _D3D12MA_ALLOCATION_FUNCTIONS void Allocation::PackedData::SetType(Type type) { const UINT u = (UINT)type; @@ -8276,36 +8397,9 @@ void Allocation::PackedData::SetTextureLayout(D3D12_TEXTURE_LAYOUT textureLayout m_TextureLayout = u; } -void Allocation::ReleaseThis() -{ - if(this == NULL) - { - return; - } - - SAFE_RELEASE(m_Resource); - - switch(m_PackedData.GetType()) - { - case TYPE_COMMITTED: - m_Allocator->FreeCommittedMemory(this); - break; - case TYPE_PLACED: - m_Allocator->FreePlacedMemory(this); - break; - case TYPE_HEAP: - m_Allocator->FreeHeapMemory(this); - break; - } - - FreeName(); - - m_Allocator->GetAllocationObjectAllocator().Free(this); -} - UINT64 Allocation::GetOffset() const { - switch(m_PackedData.GetType()) + switch (m_PackedData.GetType()) { case TYPE_COMMITTED: case TYPE_HEAP: @@ -8320,7 +8414,7 @@ UINT64 Allocation::GetOffset() const ID3D12Heap* Allocation::GetHeap() const { - switch(m_PackedData.GetType()) + switch (m_PackedData.GetType()) { case TYPE_COMMITTED: return NULL; @@ -8338,7 +8432,7 @@ void Allocation::SetName(LPCWSTR Name) { FreeName(); - if(Name) + if (Name) { const size_t nameCharCount = wcslen(Name) + 1; m_Name = D3D12MA_NEW_ARRAY(m_Allocator->GetAllocs(), WCHAR, nameCharCount); @@ -8346,12 +8440,39 @@ void Allocation::SetName(LPCWSTR Name) } } -Allocation::Allocation(AllocatorPimpl* allocator, UINT64 size, BOOL wasZeroInitialized) : - m_Allocator{allocator}, - m_Size{size}, - m_Resource{NULL}, - m_CreationFrameIndex{allocator->GetCurrentFrameIndex()}, - m_Name{NULL} +void Allocation::ReleaseThis() +{ + if (this == NULL) + { + return; + } + + SAFE_RELEASE(m_Resource); + + switch (m_PackedData.GetType()) + { + case TYPE_COMMITTED: + m_Allocator->FreeCommittedMemory(this); + break; + case TYPE_PLACED: + m_Allocator->FreePlacedMemory(this); + break; + case TYPE_HEAP: + m_Allocator->FreeHeapMemory(this); + break; + } + + FreeName(); + + m_Allocator->GetAllocationObjectAllocator().Free(this); +} + +Allocation::Allocation(AllocatorPimpl* allocator, UINT64 size, BOOL wasZeroInitialized) + : m_Allocator{ allocator }, + m_Size{ size }, + m_Resource{ NULL }, + m_CreationFrameIndex{ allocator->GetCurrentFrameIndex() }, + m_Name{ NULL } { D3D12MA_ASSERT(allocator); @@ -8413,62 +8534,78 @@ void Allocation::SetResource(ID3D12Resource* resource, const D3D12_RESOURCE_DESC void Allocation::FreeName() { - if(m_Name) + if (m_Name) { const size_t nameCharCount = wcslen(m_Name) + 1; D3D12MA_DELETE_ARRAY(m_Allocator->GetAllocs(), m_Name, nameCharCount); m_Name = NULL; } } +#endif // _D3D12MA_ALLOCATION_FUNCTIONS -//////////////////////////////////////////////////////////////////////////////// -// Private class AllocationObjectAllocator implementation - -AllocationObjectAllocator::AllocationObjectAllocator(const ALLOCATION_CALLBACKS& allocationCallbacks) : - m_Allocator(allocationCallbacks, 1024) +#ifndef _D3D12MA_POOL_FUNCTIONS +POOL_DESC Pool::GetDesc() const { + return m_Pimpl->GetDesc(); } -template Allocation* AllocationObjectAllocator::Allocate(Types... args) +void Pool::GetStatistics(Statistics* pStats) { - MutexLock mutexLock(m_Mutex); - return m_Allocator.Alloc(std::forward(args)...); + D3D12MA_ASSERT(pStats); + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + m_Pimpl->GetStatistics(*pStats); } -void AllocationObjectAllocator::Free(Allocation* alloc) +void Pool::CalculateStatistics(DetailedStatistics* pStats) { - MutexLock mutexLock(m_Mutex); - m_Allocator.Free(alloc); + D3D12MA_ASSERT(pStats); + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + m_Pimpl->CalculateStatistics(*pStats); } -//////////////////////////////////////////////////////////////////////////////// -// Public class Allocator implementation - -Allocator::Allocator(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc) : - m_Pimpl(D3D12MA_NEW(allocationCallbacks, AllocatorPimpl)(allocationCallbacks, desc)) +void Pool::SetName(LPCWSTR Name) { + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + m_Pimpl->SetName(Name); } -Allocator::~Allocator() +LPCWSTR Pool::GetName() const { - D3D12MA_DELETE(m_Pimpl->GetAllocs(), m_Pimpl); + return m_Pimpl->GetName(); } -void Allocator::ReleaseThis() +void Pool::ReleaseThis() { - // Copy is needed because otherwise we would call destructor and invalidate the structure with callbacks before using it to free memory. - const ALLOCATION_CALLBACKS allocationCallbacksCopy = m_Pimpl->GetAllocs(); - D3D12MA_DELETE(allocationCallbacksCopy, this); + if (this == NULL) + { + return; + } + + D3D12MA_DELETE(m_Pimpl->GetAllocator()->GetAllocs(), this); } +Pool::Pool(Allocator* allocator, const POOL_DESC& desc) + : m_Pimpl(D3D12MA_NEW(allocator->m_Pimpl->GetAllocs(), PoolPimpl)(allocator->m_Pimpl, desc)) {} + +Pool::~Pool() +{ + m_Pimpl->GetAllocator()->UnregisterPool(this, m_Pimpl->GetDesc().HeapProperties.Type); + + D3D12MA_DELETE(m_Pimpl->GetAllocator()->GetAllocs(), m_Pimpl); +} +#endif // _D3D12MA_POOL_FUNCTIONS + +#ifndef _D3D12MA_ALLOCATOR_FUNCTIONS const D3D12_FEATURE_DATA_D3D12_OPTIONS& Allocator::GetD3D12Options() const { return m_Pimpl->GetD3D12Options(); } + BOOL Allocator::IsUMA() const { return m_Pimpl->IsUMA(); } + BOOL Allocator::IsCacheCoherentUMA() const { return m_Pimpl->IsCacheCoherentUMA(); @@ -8483,18 +8620,18 @@ HRESULT Allocator::CreateResource( const ALLOCATION_DESC* pAllocDesc, const D3D12_RESOURCE_DESC* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, Allocation** ppAllocation, REFIID riidResource, void** ppvResource) { - if(!pAllocDesc || !pResourceDesc || !ppAllocation) + if (!pAllocDesc || !pResourceDesc || !ppAllocation) { D3D12MA_ASSERT(0 && "Invalid arguments passed to Allocator::CreateResource."); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - return m_Pimpl->CreateResource(pAllocDesc, pResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); + return m_Pimpl->CreateResource(pAllocDesc, pResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); } #ifdef __ID3D12Device8_INTERFACE_DEFINED__ @@ -8502,48 +8639,33 @@ HRESULT Allocator::CreateResource2( const ALLOCATION_DESC* pAllocDesc, const D3D12_RESOURCE_DESC1* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, Allocation** ppAllocation, REFIID riidResource, void** ppvResource) { - if(!pAllocDesc || !pResourceDesc || !ppAllocation) + if (!pAllocDesc || !pResourceDesc || !ppAllocation) { D3D12MA_ASSERT(0 && "Invalid arguments passed to Allocator::CreateResource2."); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - return m_Pimpl->CreateResource2(pAllocDesc, pResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); + return m_Pimpl->CreateResource2(pAllocDesc, pResourceDesc, InitialResourceState, pOptimizedClearValue, ppAllocation, riidResource, ppvResource); } #endif // #ifdef __ID3D12Device8_INTERFACE_DEFINED__ -static inline bool ValidateAllocateMemoryParameters( - const ALLOCATION_DESC* pAllocDesc, - const D3D12_RESOURCE_ALLOCATION_INFO* pAllocInfo, - Allocation** ppAllocation) -{ - return pAllocDesc && - pAllocInfo && - ppAllocation && - (pAllocInfo->Alignment == 0 || - pAllocInfo->Alignment == D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT || - pAllocInfo->Alignment == D3D12_DEFAULT_MSAA_RESOURCE_PLACEMENT_ALIGNMENT) && - pAllocInfo->SizeInBytes != 0 && - pAllocInfo->SizeInBytes % (64ull * 1024) == 0; -} - HRESULT Allocator::AllocateMemory( const ALLOCATION_DESC* pAllocDesc, const D3D12_RESOURCE_ALLOCATION_INFO* pAllocInfo, Allocation** ppAllocation) { - if(!ValidateAllocateMemoryParameters(pAllocDesc, pAllocInfo, ppAllocation)) + if (!ValidateAllocateMemoryParameters(pAllocDesc, pAllocInfo, ppAllocation)) { D3D12MA_ASSERT(0 && "Invalid arguments passed to Allocator::AllocateMemory."); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - return m_Pimpl->AllocateMemory(pAllocDesc, pAllocInfo, ppAllocation); + return m_Pimpl->AllocateMemory(pAllocDesc, pAllocInfo, ppAllocation); } HRESULT Allocator::CreateAliasingResource( @@ -8551,39 +8673,39 @@ HRESULT Allocator::CreateAliasingResource( UINT64 AllocationLocalOffset, const D3D12_RESOURCE_DESC* pResourceDesc, D3D12_RESOURCE_STATES InitialResourceState, - const D3D12_CLEAR_VALUE *pOptimizedClearValue, + const D3D12_CLEAR_VALUE* pOptimizedClearValue, REFIID riidResource, void** ppvResource) { - if(!pAllocation || !pResourceDesc || !ppvResource) + if (!pAllocation || !pResourceDesc || !ppvResource) { D3D12MA_ASSERT(0 && "Invalid arguments passed to Allocator::CreateAliasingResource."); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - return m_Pimpl->CreateAliasingResource(pAllocation, AllocationLocalOffset, pResourceDesc, InitialResourceState, pOptimizedClearValue, riidResource, ppvResource); + return m_Pimpl->CreateAliasingResource(pAllocation, AllocationLocalOffset, pResourceDesc, InitialResourceState, pOptimizedClearValue, riidResource, ppvResource); } HRESULT Allocator::CreatePool( const POOL_DESC* pPoolDesc, Pool** ppPool) { - if(!pPoolDesc || !ppPool || + if (!pPoolDesc || !ppPool || (pPoolDesc->MaxBlockCount > 0 && pPoolDesc->MaxBlockCount < pPoolDesc->MinBlockCount) || (pPoolDesc->MinAllocationAlignment > 0 && !IsPow2(pPoolDesc->MinAllocationAlignment))) { D3D12MA_ASSERT(0 && "Invalid arguments passed to Allocator::CreatePool."); return E_INVALIDARG; } - if(!m_Pimpl->HeapFlagsFulfillResourceHeapTier(pPoolDesc->HeapFlags)) + if (!m_Pimpl->HeapFlagsFulfillResourceHeapTier(pPoolDesc->HeapFlags)) { D3D12MA_ASSERT(0 && "Invalid pPoolDesc->HeapFlags passed to Allocator::CreatePool. Did you forget to handle ResourceHeapTier=1?"); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - *ppPool = D3D12MA_NEW(m_Pimpl->GetAllocs(), Pool)(this, *pPoolDesc); + * ppPool = D3D12MA_NEW(m_Pimpl->GetAllocs(), Pool)(this, *pPoolDesc); HRESULT hr = (*ppPool)->m_Pimpl->Init(); - if(SUCCEEDED(hr)) + if (SUCCEEDED(hr)) { m_Pimpl->RegisterPool(*ppPool, pPoolDesc->HeapProperties.Type); } @@ -8598,31 +8720,31 @@ HRESULT Allocator::CreatePool( void Allocator::SetCurrentFrameIndex(UINT frameIndex) { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->SetCurrentFrameIndex(frameIndex); + m_Pimpl->SetCurrentFrameIndex(frameIndex); +} + +void Allocator::GetBudget(Budget* pLocalBudget, Budget* pNonLocalBudget) +{ + if (pLocalBudget == NULL && pNonLocalBudget == NULL) + { + return; + } + D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK + m_Pimpl->GetBudget(pLocalBudget, pNonLocalBudget); } void Allocator::CalculateStatistics(TotalStatistics* pStats) { D3D12MA_ASSERT(pStats); D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->CalculateStatistics(*pStats); -} - -void Allocator::GetBudget(Budget* pLocalBudget, Budget* pNonLocalBudget) -{ - if(pLocalBudget == NULL && pNonLocalBudget == NULL) - { - return; - } - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->GetBudget(pLocalBudget, pNonLocalBudget); + m_Pimpl->CalculateStatistics(*pStats); } void Allocator::BuildStatsString(WCHAR** ppStatsString, BOOL DetailedMap) const { D3D12MA_ASSERT(ppStatsString); D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->BuildStatsString(ppStatsString, DetailedMap); + m_Pimpl->BuildStatsString(ppStatsString, DetailedMap); } void Allocator::FreeStatsString(WCHAR* pStatsString) const @@ -8630,76 +8752,32 @@ void Allocator::FreeStatsString(WCHAR* pStatsString) const if (pStatsString != NULL) { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->FreeStatsString(pStatsString); + m_Pimpl->FreeStatsString(pStatsString); } } -//////////////////////////////////////////////////////////////////////////////// -// Private class VirtualBlockPimpl definition - -class VirtualBlockPimpl -{ -public: - const ALLOCATION_CALLBACKS m_AllocationCallbacks; - const UINT64 m_Size; - BlockMetadata* m_Metadata; - - VirtualBlockPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc); - ~VirtualBlockPimpl(); -}; - -VirtualBlockPimpl::VirtualBlockPimpl(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc) : - m_AllocationCallbacks(allocationCallbacks), - m_Size(desc.Size) -{ - switch (desc.Flags & VIRTUAL_BLOCK_FLAG_ALGORITHM_MASK) - { - case VIRTUAL_BLOCK_FLAG_ALGORITHM_LINEAR: - m_Metadata = D3D12MA_NEW(allocationCallbacks, BlockMetadata_Linear)(&m_AllocationCallbacks, true); - break; - default: - D3D12MA_ASSERT(0); - case 0: - m_Metadata = D3D12MA_NEW(allocationCallbacks, BlockMetadata_TLSF)(&m_AllocationCallbacks, true); - break; - } - m_Metadata->Init(m_Size); -} - -VirtualBlockPimpl::~VirtualBlockPimpl() -{ - D3D12MA_DELETE(m_AllocationCallbacks, m_Metadata); -} - -//////////////////////////////////////////////////////////////////////////////// -// Public class VirtualBlock implementation - -VirtualBlock::VirtualBlock(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc) : - m_Pimpl(D3D12MA_NEW(allocationCallbacks, VirtualBlockPimpl)(allocationCallbacks, desc)) -{ -} - -VirtualBlock::~VirtualBlock() -{ - // THIS IS AN IMPORTANT ASSERT! - // Hitting it means you have some memory leak - unreleased allocations in this virtual block. - D3D12MA_ASSERT(m_Pimpl->m_Metadata->IsEmpty() && "Some allocations were not freed before destruction of this virtual block!"); - - D3D12MA_DELETE(m_Pimpl->m_AllocationCallbacks, m_Pimpl); -} - -void VirtualBlock::ReleaseThis() +void Allocator::ReleaseThis() { // Copy is needed because otherwise we would call destructor and invalidate the structure with callbacks before using it to free memory. - const ALLOCATION_CALLBACKS allocationCallbacksCopy = m_Pimpl->m_AllocationCallbacks; + const ALLOCATION_CALLBACKS allocationCallbacksCopy = m_Pimpl->GetAllocs(); D3D12MA_DELETE(allocationCallbacksCopy, this); } +Allocator::Allocator(const ALLOCATION_CALLBACKS& allocationCallbacks, const ALLOCATOR_DESC& desc) + : m_Pimpl(D3D12MA_NEW(allocationCallbacks, AllocatorPimpl)(allocationCallbacks, desc)) {} + +Allocator::~Allocator() +{ + D3D12MA_DELETE(m_Pimpl->GetAllocs(), m_Pimpl); +} +#endif // _D3D12MA_ALLOCATOR_FUNCTIONS + +#ifndef _D3D12MA_VIRTUAL_BLOCK_FUNCTIONS BOOL VirtualBlock::IsEmpty() const { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - return m_Pimpl->m_Metadata->IsEmpty() ? TRUE : FALSE; + return m_Pimpl->m_Metadata->IsEmpty() ? TRUE : FALSE; } void VirtualBlock::GetAllocationInfo(VirtualAllocation allocation, VIRTUAL_ALLOCATION_INFO* pInfo) const @@ -8708,22 +8786,22 @@ void VirtualBlock::GetAllocationInfo(VirtualAllocation allocation, VIRTUAL_ALLOC D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->m_Metadata->GetAllocationInfo(allocation.AllocHandle, *pInfo); + m_Pimpl->m_Metadata->GetAllocationInfo(allocation.AllocHandle, *pInfo); } HRESULT VirtualBlock::Allocate(const VIRTUAL_ALLOCATION_DESC* pDesc, VirtualAllocation* pAllocation, UINT64* pOffset) { - if(!pDesc || !pAllocation || pDesc->Size == 0 || !IsPow2(pDesc->Alignment)) + if (!pDesc || !pAllocation || pDesc->Size == 0 || !IsPow2(pDesc->Alignment)) { D3D12MA_ASSERT(0 && "Invalid arguments passed to VirtualBlock::Allocate."); return E_INVALIDARG; } D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - - const UINT64 alignment = pDesc->Alignment != 0 ? pDesc->Alignment : 1; + + const UINT64 alignment = pDesc->Alignment != 0 ? pDesc->Alignment : 1; AllocationRequest allocRequest = {}; - if(m_Pimpl->m_Metadata->CreateAllocationRequest( + if (m_Pimpl->m_Metadata->CreateAllocationRequest( pDesc->Size, alignment, pDesc->Flags & VIRTUAL_ALLOCATION_FLAG_UPPER_ADDRESS, @@ -8747,12 +8825,12 @@ HRESULT VirtualBlock::Allocate(const VIRTUAL_ALLOCATION_DESC* pDesc, VirtualAllo void VirtualBlock::FreeAllocation(VirtualAllocation allocation) { - if(allocation.AllocHandle == (AllocHandle)0) + if (allocation.AllocHandle == (AllocHandle)0) return; D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->m_Metadata->Free(allocation.AllocHandle); + m_Pimpl->m_Metadata->Free(allocation.AllocHandle); D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); } @@ -8760,7 +8838,7 @@ void VirtualBlock::Clear() { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->m_Metadata->Clear(); + m_Pimpl->m_Metadata->Clear(); D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); } @@ -8770,14 +8848,14 @@ void VirtualBlock::SetAllocationUserData(VirtualAllocation allocation, void* pUs D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - m_Pimpl->m_Metadata->SetAllocationUserData(allocation.AllocHandle, pUserData); + m_Pimpl->m_Metadata->SetAllocationUserData(allocation.AllocHandle, pUserData); } void VirtualBlock::GetStatistics(Statistics* pStats) const { D3D12MA_ASSERT(pStats); D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); + D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); ClearStatistics(*pStats); m_Pimpl->m_Metadata->AddStatistics(*pStats); } @@ -8786,7 +8864,7 @@ void VirtualBlock::CalculateStatistics(DetailedStatistics* pStats) const { D3D12MA_ASSERT(pStats); D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); + D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); ClearDetailedStatistics(*pStats); m_Pimpl->m_Metadata->AddDetailedStatistics(*pStats); } @@ -8797,7 +8875,7 @@ void VirtualBlock::BuildStatsString(WCHAR** ppStatsString) const D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - StringBuilder sb(m_Pimpl->m_AllocationCallbacks); + StringBuilder sb(m_Pimpl->m_AllocationCallbacks); { JsonWriter json(m_Pimpl->m_AllocationCallbacks, sb); D3D12MA_HEAVY_ASSERT(m_Pimpl->m_Metadata->Validate()); @@ -8816,53 +8894,28 @@ void VirtualBlock::FreeStatsString(WCHAR* pStatsString) const if (pStatsString != NULL) { D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - D3D12MA::Free(m_Pimpl->m_AllocationCallbacks, pStatsString); + D3D12MA::Free(m_Pimpl->m_AllocationCallbacks, pStatsString); } } - -//////////////////////////////////////////////////////////////////////////////// -// Public global functions - -HRESULT CreateAllocator(const ALLOCATOR_DESC* pDesc, Allocator** ppAllocator) +void VirtualBlock::ReleaseThis() { - if(!pDesc || !ppAllocator || !pDesc->pDevice || !pDesc->pAdapter || - !(pDesc->PreferredBlockSize == 0 || (pDesc->PreferredBlockSize >= 16 && pDesc->PreferredBlockSize < 0x10000000000ull))) - { - D3D12MA_ASSERT(0 && "Invalid arguments passed to CreateAllocator."); - return E_INVALIDARG; - } - - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - - ALLOCATION_CALLBACKS allocationCallbacks; - SetupAllocationCallbacks(allocationCallbacks, pDesc->pAllocationCallbacks); - - *ppAllocator = D3D12MA_NEW(allocationCallbacks, Allocator)(allocationCallbacks, *pDesc); - HRESULT hr = (*ppAllocator)->m_Pimpl->Init(*pDesc); - if(FAILED(hr)) - { - D3D12MA_DELETE(allocationCallbacks, *ppAllocator); - *ppAllocator = NULL; - } - return hr; + // Copy is needed because otherwise we would call destructor and invalidate the structure with callbacks before using it to free memory. + const ALLOCATION_CALLBACKS allocationCallbacksCopy = m_Pimpl->m_AllocationCallbacks; + D3D12MA_DELETE(allocationCallbacksCopy, this); } -HRESULT CreateVirtualBlock(const VIRTUAL_BLOCK_DESC* pDesc, VirtualBlock** ppVirtualBlock) +VirtualBlock::VirtualBlock(const ALLOCATION_CALLBACKS& allocationCallbacks, const VIRTUAL_BLOCK_DESC& desc) + : m_Pimpl(D3D12MA_NEW(allocationCallbacks, VirtualBlockPimpl)(allocationCallbacks, desc)) {} + +VirtualBlock::~VirtualBlock() { - if(!pDesc || !ppVirtualBlock) - { - D3D12MA_ASSERT(0 && "Invalid arguments passed to CreateVirtualBlock."); - return E_INVALIDARG; - } + // THIS IS AN IMPORTANT ASSERT! + // Hitting it means you have some memory leak - unreleased allocations in this virtual block. + D3D12MA_ASSERT(m_Pimpl->m_Metadata->IsEmpty() && "Some allocations were not freed before destruction of this virtual block!"); - D3D12MA_DEBUG_GLOBAL_MUTEX_LOCK - - ALLOCATION_CALLBACKS allocationCallbacks; - SetupAllocationCallbacks(allocationCallbacks, pDesc->pAllocationCallbacks); - - *ppVirtualBlock = D3D12MA_NEW(allocationCallbacks, VirtualBlock)(allocationCallbacks, *pDesc); - return S_OK; + D3D12MA_DELETE(m_Pimpl->m_AllocationCallbacks, m_Pimpl); } - +#endif // _D3D12MA_VIRTUAL_BLOCK_FUNCTIONS +#endif // _D3D12MA_PUBLIC_FUNCTIONS } // namespace D3D12MA