diff --git a/bin/VmaReplay_Release_vs2015.exe b/bin/VmaReplay_Release_vs2015.exe index 05a6da4..1e08a58 100644 Binary files a/bin/VmaReplay_Release_vs2015.exe and b/bin/VmaReplay_Release_vs2015.exe differ diff --git a/bin/VulkanSample_Release_vs2015.exe b/bin/VulkanSample_Release_vs2015.exe index 05ae82b..3958098 100644 Binary files a/bin/VulkanSample_Release_vs2015.exe and b/bin/VulkanSample_Release_vs2015.exe differ diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 8f7e034..ea6d4ef 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,7 +62,7 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
1258 #include <vulkan/vulkan.h>
1259 
1260 #if !defined(VMA_DEDICATED_ALLOCATION)
1261  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
1262  #define VMA_DEDICATED_ALLOCATION 1
1263  #else
1264  #define VMA_DEDICATED_ALLOCATION 0
1265  #endif
1266 #endif
1267 
1277 VK_DEFINE_HANDLE(VmaAllocator)
1278 
1279 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
1281  VmaAllocator allocator,
1282  uint32_t memoryType,
1283  VkDeviceMemory memory,
1284  VkDeviceSize size);
1286 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
1287  VmaAllocator allocator,
1288  uint32_t memoryType,
1289  VkDeviceMemory memory,
1290  VkDeviceSize size);
1291 
1299 typedef struct VmaDeviceMemoryCallbacks {
1301  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
1303  PFN_vmaFreeDeviceMemoryFunction pfnFree;
1304 } VmaDeviceMemoryCallbacks;
1305 
1307 typedef enum VmaAllocatorCreateFlagBits {
1312  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1334  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1335 
1336  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1337 } VmaAllocatorCreateFlagBits;
1338 typedef VkFlags VmaAllocatorCreateFlags;
1339 
1344 typedef struct VmaVulkanFunctions {
1345  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1346  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1347  PFN_vkAllocateMemory vkAllocateMemory;
1348  PFN_vkFreeMemory vkFreeMemory;
1349  PFN_vkMapMemory vkMapMemory;
1350  PFN_vkUnmapMemory vkUnmapMemory;
1351  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
1352  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
1353  PFN_vkBindBufferMemory vkBindBufferMemory;
1354  PFN_vkBindImageMemory vkBindImageMemory;
1355  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1356  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1357  PFN_vkCreateBuffer vkCreateBuffer;
1358  PFN_vkDestroyBuffer vkDestroyBuffer;
1359  PFN_vkCreateImage vkCreateImage;
1360  PFN_vkDestroyImage vkDestroyImage;
1361 #if VMA_DEDICATED_ALLOCATION
1362  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1363  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1364 #endif
1365 } VmaVulkanFunctions;
1366 
1368 typedef enum VmaRecordFlagBits {
1374  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
1375 
1376  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1377 } VmaRecordFlagBits;
1378 typedef VkFlags VmaRecordFlags;
1379 
1380 /*
1381 Define this macro to 0/1 to disable/enable support for recording functionality,
1382 available through VmaAllocatorCreateInfo::pRecordSettings.
1383 */
1384 #ifndef VMA_RECORDING_ENABLED
1385  #ifdef _WIN32
1386  #define VMA_RECORDING_ENABLED 1
1387  #else
1388  #define VMA_RECORDING_ENABLED 0
1389  #endif
1390 #endif
1391 
1393 typedef struct VmaRecordSettings
1394 {
1396  VmaRecordFlags flags;
1404  const char* pFilePath;
1405 } VmaRecordSettings;
1406 
1408 typedef struct VmaAllocatorCreateInfo
1409 {
1411  VmaAllocatorCreateFlags flags;
1413 
1414  VkPhysicalDevice physicalDevice;
1416 
1417  VkDevice device;
1419 
1420  VkDeviceSize preferredLargeHeapBlockSize;
1422 
1423  const VkAllocationCallbacks* pAllocationCallbacks;
1425 
1426  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1440  uint32_t frameInUseCount;
1464  const VkDeviceSize* pHeapSizeLimit;
1476  const VmaVulkanFunctions* pVulkanFunctions;
1483  const VmaRecordSettings* pRecordSettings;
1484 } VmaAllocatorCreateInfo;
1485 
1487 VkResult vmaCreateAllocator(
1488  const VmaAllocatorCreateInfo* pCreateInfo,
1489  VmaAllocator* pAllocator);
1490 
1492 void vmaDestroyAllocator(
1493  VmaAllocator allocator);
1494 
1499 void vmaGetPhysicalDeviceProperties(
1500  VmaAllocator allocator,
1501  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1502 
1507 void vmaGetMemoryProperties(
1508  VmaAllocator allocator,
1509  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1510 
1517 void vmaGetMemoryTypeProperties(
1518  VmaAllocator allocator,
1519  uint32_t memoryTypeIndex,
1520  VkMemoryPropertyFlags* pFlags);
1521 
1530 void vmaSetCurrentFrameIndex(
1531  VmaAllocator allocator,
1532  uint32_t frameIndex);
1533 
1536 typedef struct VmaStatInfo
1537 {
1539  uint32_t blockCount;
1541  uint32_t allocationCount;
1543  uint32_t unusedRangeCount;
1545  VkDeviceSize usedBytes;
1547  VkDeviceSize unusedBytes;
1548  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1549  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1550 } VmaStatInfo;
1551 
1553 typedef struct VmaStats
1554 {
1555  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1556  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1557  VmaStatInfo total;
1558 } VmaStats;
1559 
1561 void vmaCalculateStats(
1562  VmaAllocator allocator,
1563  VmaStats* pStats);
1564 
1565 #define VMA_STATS_STRING_ENABLED 1
1566 
1567 #if VMA_STATS_STRING_ENABLED
1568 
1570 
1572 void vmaBuildStatsString(
1573  VmaAllocator allocator,
1574  char** ppStatsString,
1575  VkBool32 detailedMap);
1576 
1577 void vmaFreeStatsString(
1578  VmaAllocator allocator,
1579  char* pStatsString);
1580 
1581 #endif // #if VMA_STATS_STRING_ENABLED
1582 
1591 VK_DEFINE_HANDLE(VmaPool)
1592 
1593 typedef enum VmaMemoryUsage
1594 {
1598  VMA_MEMORY_USAGE_UNKNOWN = 0,
1615  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1625  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1632  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1641  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1642  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1643 } VmaMemoryUsage;
1644 
1646 typedef enum VmaAllocationCreateFlagBits {
1658  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1659 
1669  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1682  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1695  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1702  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1708  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1709 
1710  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1711 } VmaAllocationCreateFlagBits;
1712 typedef VkFlags VmaAllocationCreateFlags;
1713 
1714 typedef struct VmaAllocationCreateInfo
1715 {
1717  VmaAllocationCreateFlags flags;
1723  VmaMemoryUsage usage;
1728  VkMemoryPropertyFlags requiredFlags;
1733  VkMemoryPropertyFlags preferredFlags;
1741  uint32_t memoryTypeBits;
1747  VmaPool pool;
1754  void* pUserData;
1755 } VmaAllocationCreateInfo;
1756 
1773 VkResult vmaFindMemoryTypeIndex(
1774  VmaAllocator allocator,
1775  uint32_t memoryTypeBits,
1776  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1777  uint32_t* pMemoryTypeIndex);
1778 
1791 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1792  VmaAllocator allocator,
1793  const VkBufferCreateInfo* pBufferCreateInfo,
1794  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1795  uint32_t* pMemoryTypeIndex);
1796 
1809 VkResult vmaFindMemoryTypeIndexForImageInfo(
1810  VmaAllocator allocator,
1811  const VkImageCreateInfo* pImageCreateInfo,
1812  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1813  uint32_t* pMemoryTypeIndex);
1814 
1816 typedef enum VmaPoolCreateFlagBits {
1834  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1835 
1836  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1837 } VmaPoolCreateFlagBits;
1838 typedef VkFlags VmaPoolCreateFlags;
1839 
1842 typedef struct VmaPoolCreateInfo {
1845  uint32_t memoryTypeIndex;
1848  VmaPoolCreateFlags flags;
1853  VkDeviceSize blockSize;
1858  size_t minBlockCount;
1866  size_t maxBlockCount;
1880  uint32_t frameInUseCount;
1881 } VmaPoolCreateInfo;
1882 
1885 typedef struct VmaPoolStats {
1888  VkDeviceSize size;
1891  VkDeviceSize unusedSize;
1894  size_t allocationCount;
1897  size_t unusedRangeCount;
1904  VkDeviceSize unusedRangeSizeMax;
1905 } VmaPoolStats;
1906 
1913 VkResult vmaCreatePool(
1914  VmaAllocator allocator,
1915  const VmaPoolCreateInfo* pCreateInfo,
1916  VmaPool* pPool);
1917 
1920 void vmaDestroyPool(
1921  VmaAllocator allocator,
1922  VmaPool pool);
1923 
1930 void vmaGetPoolStats(
1931  VmaAllocator allocator,
1932  VmaPool pool,
1933  VmaPoolStats* pPoolStats);
1934 
1941 void vmaMakePoolAllocationsLost(
1942  VmaAllocator allocator,
1943  VmaPool pool,
1944  size_t* pLostAllocationCount);
1945 
1960 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
1961 
1986 VK_DEFINE_HANDLE(VmaAllocation)
1987 
1988 
1990 typedef struct VmaAllocationInfo {
1995  uint32_t memoryType;
2004  VkDeviceMemory deviceMemory;
2009  VkDeviceSize offset;
2014  VkDeviceSize size;
2023  void* pMappedData;
2028  void* pUserData;
2029 } VmaAllocationInfo;
2030 
2041 VkResult vmaAllocateMemory(
2042  VmaAllocator allocator,
2043  const VkMemoryRequirements* pVkMemoryRequirements,
2044  const VmaAllocationCreateInfo* pCreateInfo,
2045  VmaAllocation* pAllocation,
2046  VmaAllocationInfo* pAllocationInfo);
2047 
2054 VkResult vmaAllocateMemoryForBuffer(
2055  VmaAllocator allocator,
2056  VkBuffer buffer,
2057  const VmaAllocationCreateInfo* pCreateInfo,
2058  VmaAllocation* pAllocation,
2059  VmaAllocationInfo* pAllocationInfo);
2060 
2062 VkResult vmaAllocateMemoryForImage(
2063  VmaAllocator allocator,
2064  VkImage image,
2065  const VmaAllocationCreateInfo* pCreateInfo,
2066  VmaAllocation* pAllocation,
2067  VmaAllocationInfo* pAllocationInfo);
2068 
2070 void vmaFreeMemory(
2071  VmaAllocator allocator,
2072  VmaAllocation allocation);
2073 
2090 void vmaGetAllocationInfo(
2091  VmaAllocator allocator,
2092  VmaAllocation allocation,
2093  VmaAllocationInfo* pAllocationInfo);
2094 
2109 VkBool32 vmaTouchAllocation(
2110  VmaAllocator allocator,
2111  VmaAllocation allocation);
2112 
2126 void vmaSetAllocationUserData(
2127  VmaAllocator allocator,
2128  VmaAllocation allocation,
2129  void* pUserData);
2130 
2141 void vmaCreateLostAllocation(
2142  VmaAllocator allocator,
2143  VmaAllocation* pAllocation);
2144 
2179 VkResult vmaMapMemory(
2180  VmaAllocator allocator,
2181  VmaAllocation allocation,
2182  void** ppData);
2183 
2188 void vmaUnmapMemory(
2189  VmaAllocator allocator,
2190  VmaAllocation allocation);
2191 
2204 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2205 
2218 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2219 
2236 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
2237 
2239 typedef struct VmaDefragmentationInfo {
2244  VkDeviceSize maxBytesToMove;
2249  uint32_t maxAllocationsToMove;
2250 } VmaDefragmentationInfo;
2251 
2253 typedef struct VmaDefragmentationStats {
2255  VkDeviceSize bytesMoved;
2257  VkDeviceSize bytesFreed;
2259  uint32_t allocationsMoved;
2261  uint32_t deviceMemoryBlocksFreed;
2262 } VmaDefragmentationStats;
2263 
2346 VkResult vmaDefragment(
2347  VmaAllocator allocator,
2348  VmaAllocation* pAllocations,
2349  size_t allocationCount,
2350  VkBool32* pAllocationsChanged,
2351  const VmaDefragmentationInfo *pDefragmentationInfo,
2352  VmaDefragmentationStats* pDefragmentationStats);
2353 
2366 VkResult vmaBindBufferMemory(
2367  VmaAllocator allocator,
2368  VmaAllocation allocation,
2369  VkBuffer buffer);
2370 
2383 VkResult vmaBindImageMemory(
2384  VmaAllocator allocator,
2385  VmaAllocation allocation,
2386  VkImage image);
2387 
2414 VkResult vmaCreateBuffer(
2415  VmaAllocator allocator,
2416  const VkBufferCreateInfo* pBufferCreateInfo,
2417  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2418  VkBuffer* pBuffer,
2419  VmaAllocation* pAllocation,
2420  VmaAllocationInfo* pAllocationInfo);
2421 
2433 void vmaDestroyBuffer(
2434  VmaAllocator allocator,
2435  VkBuffer buffer,
2436  VmaAllocation allocation);
2437 
2439 VkResult vmaCreateImage(
2440  VmaAllocator allocator,
2441  const VkImageCreateInfo* pImageCreateInfo,
2442  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2443  VkImage* pImage,
2444  VmaAllocation* pAllocation,
2445  VmaAllocationInfo* pAllocationInfo);
2446 
2458 void vmaDestroyImage(
2459  VmaAllocator allocator,
2460  VkImage image,
2461  VmaAllocation allocation);
2462 
2463 #ifdef __cplusplus
2464 }
2465 #endif
2466 
2467 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
2468 
2469 // For Visual Studio IntelliSense.
2470 #if defined(__cplusplus) && defined(__INTELLISENSE__)
2471 #define VMA_IMPLEMENTATION
2472 #endif
2473 
2474 #ifdef VMA_IMPLEMENTATION
2475 #undef VMA_IMPLEMENTATION
2476 
2477 #include <cstdint>
2478 #include <cstdlib>
2479 #include <cstring>
2480 
2481 /*******************************************************************************
2482 CONFIGURATION SECTION
2483 
2484 Define some of these macros before each #include of this header or change them
2485 here if you need other then default behavior depending on your environment.
2486 */
2487 
2488 /*
2489 Define this macro to 1 to make the library fetch pointers to Vulkan functions
2490 internally, like:
2491 
2492  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
2493 
2494 Define to 0 if you are going to provide you own pointers to Vulkan functions via
2495 VmaAllocatorCreateInfo::pVulkanFunctions.
2496 */
2497 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
2498 #define VMA_STATIC_VULKAN_FUNCTIONS 1
2499 #endif
2500 
2501 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
2502 //#define VMA_USE_STL_CONTAINERS 1
2503 
2504 /* Set this macro to 1 to make the library including and using STL containers:
2505 std::pair, std::vector, std::list, std::unordered_map.
2506 
2507 Set it to 0 or undefined to make the library using its own implementation of
2508 the containers.
2509 */
2510 #if VMA_USE_STL_CONTAINERS
2511  #define VMA_USE_STL_VECTOR 1
2512  #define VMA_USE_STL_UNORDERED_MAP 1
2513  #define VMA_USE_STL_LIST 1
2514 #endif
2515 
2516 #if VMA_USE_STL_VECTOR
2517  #include <vector>
2518 #endif
2519 
2520 #if VMA_USE_STL_UNORDERED_MAP
2521  #include <unordered_map>
2522 #endif
2523 
2524 #if VMA_USE_STL_LIST
2525  #include <list>
2526 #endif
2527 
2528 /*
2529 Following headers are used in this CONFIGURATION section only, so feel free to
2530 remove them if not needed.
2531 */
2532 #include <cassert> // for assert
2533 #include <algorithm> // for min, max
2534 #include <mutex> // for std::mutex
2535 #include <atomic> // for std::atomic
2536 
2537 #ifndef VMA_NULL
2538  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2539  #define VMA_NULL nullptr
2540 #endif
2541 
2542 #if defined(__APPLE__) || defined(__ANDROID__)
2543 #include <cstdlib>
2544 void *aligned_alloc(size_t alignment, size_t size)
2545 {
2546  // alignment must be >= sizeof(void*)
2547  if(alignment < sizeof(void*))
2548  {
2549  alignment = sizeof(void*);
2550  }
2551 
2552  void *pointer;
2553  if(posix_memalign(&pointer, alignment, size) == 0)
2554  return pointer;
2555  return VMA_NULL;
2556 }
2557 #endif
2558 
2559 // If your compiler is not compatible with C++11 and definition of
2560 // aligned_alloc() function is missing, uncommeting following line may help:
2561 
2562 //#include <malloc.h>
2563 
2564 // Normal assert to check for programmer's errors, especially in Debug configuration.
2565 #ifndef VMA_ASSERT
2566  #ifdef _DEBUG
2567  #define VMA_ASSERT(expr) assert(expr)
2568  #else
2569  #define VMA_ASSERT(expr)
2570  #endif
2571 #endif
2572 
2573 // Assert that will be called very often, like inside data structures e.g. operator[].
2574 // Making it non-empty can make program slow.
2575 #ifndef VMA_HEAVY_ASSERT
2576  #ifdef _DEBUG
2577  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2578  #else
2579  #define VMA_HEAVY_ASSERT(expr)
2580  #endif
2581 #endif
2582 
2583 #ifndef VMA_ALIGN_OF
2584  #define VMA_ALIGN_OF(type) (__alignof(type))
2585 #endif
2586 
2587 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2588  #if defined(_WIN32)
2589  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2590  #else
2591  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2592  #endif
2593 #endif
2594 
2595 #ifndef VMA_SYSTEM_FREE
2596  #if defined(_WIN32)
2597  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2598  #else
2599  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2600  #endif
2601 #endif
2602 
2603 #ifndef VMA_MIN
2604  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2605 #endif
2606 
2607 #ifndef VMA_MAX
2608  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2609 #endif
2610 
2611 #ifndef VMA_SWAP
2612  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2613 #endif
2614 
2615 #ifndef VMA_SORT
2616  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2617 #endif
2618 
2619 #ifndef VMA_DEBUG_LOG
2620  #define VMA_DEBUG_LOG(format, ...)
2621  /*
2622  #define VMA_DEBUG_LOG(format, ...) do { \
2623  printf(format, __VA_ARGS__); \
2624  printf("\n"); \
2625  } while(false)
2626  */
2627 #endif
2628 
2629 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2630 #if VMA_STATS_STRING_ENABLED
2631  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2632  {
2633  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2634  }
2635  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2636  {
2637  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2638  }
2639  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2640  {
2641  snprintf(outStr, strLen, "%p", ptr);
2642  }
2643 #endif
2644 
2645 #ifndef VMA_MUTEX
2646  class VmaMutex
2647  {
2648  public:
2649  VmaMutex() { }
2650  ~VmaMutex() { }
2651  void Lock() { m_Mutex.lock(); }
2652  void Unlock() { m_Mutex.unlock(); }
2653  private:
2654  std::mutex m_Mutex;
2655  };
2656  #define VMA_MUTEX VmaMutex
2657 #endif
2658 
2659 /*
2660 If providing your own implementation, you need to implement a subset of std::atomic:
2661 
2662 - Constructor(uint32_t desired)
2663 - uint32_t load() const
2664 - void store(uint32_t desired)
2665 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2666 */
2667 #ifndef VMA_ATOMIC_UINT32
2668  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2669 #endif
2670 
2671 #ifndef VMA_BEST_FIT
2672 
2684  #define VMA_BEST_FIT (1)
2685 #endif
2686 
2687 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2688 
2692  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2693 #endif
2694 
2695 #ifndef VMA_DEBUG_ALIGNMENT
2696 
2700  #define VMA_DEBUG_ALIGNMENT (1)
2701 #endif
2702 
2703 #ifndef VMA_DEBUG_MARGIN
2704 
2708  #define VMA_DEBUG_MARGIN (0)
2709 #endif
2710 
2711 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
2712 
2716  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
2717 #endif
2718 
2719 #ifndef VMA_DEBUG_DETECT_CORRUPTION
2720 
2725  #define VMA_DEBUG_DETECT_CORRUPTION (0)
2726 #endif
2727 
2728 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2729 
2733  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2734 #endif
2735 
2736 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2737 
2741  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2742 #endif
2743 
2744 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2745  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2747 #endif
2748 
2749 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2750  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2752 #endif
2753 
2754 #ifndef VMA_CLASS_NO_COPY
2755  #define VMA_CLASS_NO_COPY(className) \
2756  private: \
2757  className(const className&) = delete; \
2758  className& operator=(const className&) = delete;
2759 #endif
2760 
2761 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2762 
2763 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
2764 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
2765 
2766 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
2767 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
2768 
2769 /*******************************************************************************
2770 END OF CONFIGURATION
2771 */
2772 
2773 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2774  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2775 
2776 // Returns number of bits set to 1 in (v).
2777 static inline uint32_t VmaCountBitsSet(uint32_t v)
2778 {
2779  uint32_t c = v - ((v >> 1) & 0x55555555);
2780  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2781  c = ((c >> 4) + c) & 0x0F0F0F0F;
2782  c = ((c >> 8) + c) & 0x00FF00FF;
2783  c = ((c >> 16) + c) & 0x0000FFFF;
2784  return c;
2785 }
2786 
2787 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2788 // Use types like uint32_t, uint64_t as T.
2789 template <typename T>
2790 static inline T VmaAlignUp(T val, T align)
2791 {
2792  return (val + align - 1) / align * align;
2793 }
2794 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
2795 // Use types like uint32_t, uint64_t as T.
2796 template <typename T>
2797 static inline T VmaAlignDown(T val, T align)
2798 {
2799  return val / align * align;
2800 }
2801 
2802 // Division with mathematical rounding to nearest number.
2803 template <typename T>
2804 inline T VmaRoundDiv(T x, T y)
2805 {
2806  return (x + (y / (T)2)) / y;
2807 }
2808 
2809 static inline bool VmaStrIsEmpty(const char* pStr)
2810 {
2811  return pStr == VMA_NULL || *pStr == '\0';
2812 }
2813 
2814 #ifndef VMA_SORT
2815 
2816 template<typename Iterator, typename Compare>
2817 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2818 {
2819  Iterator centerValue = end; --centerValue;
2820  Iterator insertIndex = beg;
2821  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2822  {
2823  if(cmp(*memTypeIndex, *centerValue))
2824  {
2825  if(insertIndex != memTypeIndex)
2826  {
2827  VMA_SWAP(*memTypeIndex, *insertIndex);
2828  }
2829  ++insertIndex;
2830  }
2831  }
2832  if(insertIndex != centerValue)
2833  {
2834  VMA_SWAP(*insertIndex, *centerValue);
2835  }
2836  return insertIndex;
2837 }
2838 
2839 template<typename Iterator, typename Compare>
2840 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2841 {
2842  if(beg < end)
2843  {
2844  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2845  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2846  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2847  }
2848 }
2849 
2850 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2851 
2852 #endif // #ifndef VMA_SORT
2853 
2854 /*
2855 Returns true if two memory blocks occupy overlapping pages.
2856 ResourceA must be in less memory offset than ResourceB.
2857 
2858 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2859 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2860 */
2861 static inline bool VmaBlocksOnSamePage(
2862  VkDeviceSize resourceAOffset,
2863  VkDeviceSize resourceASize,
2864  VkDeviceSize resourceBOffset,
2865  VkDeviceSize pageSize)
2866 {
2867  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2868  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2869  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2870  VkDeviceSize resourceBStart = resourceBOffset;
2871  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2872  return resourceAEndPage == resourceBStartPage;
2873 }
2874 
2875 enum VmaSuballocationType
2876 {
2877  VMA_SUBALLOCATION_TYPE_FREE = 0,
2878  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2879  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2880  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2881  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2882  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2883  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2884 };
2885 
2886 /*
2887 Returns true if given suballocation types could conflict and must respect
2888 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2889 or linear image and another one is optimal image. If type is unknown, behave
2890 conservatively.
2891 */
2892 static inline bool VmaIsBufferImageGranularityConflict(
2893  VmaSuballocationType suballocType1,
2894  VmaSuballocationType suballocType2)
2895 {
2896  if(suballocType1 > suballocType2)
2897  {
2898  VMA_SWAP(suballocType1, suballocType2);
2899  }
2900 
2901  switch(suballocType1)
2902  {
2903  case VMA_SUBALLOCATION_TYPE_FREE:
2904  return false;
2905  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2906  return true;
2907  case VMA_SUBALLOCATION_TYPE_BUFFER:
2908  return
2909  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2910  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2911  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2912  return
2913  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2914  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2915  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2916  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2917  return
2918  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2919  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2920  return false;
2921  default:
2922  VMA_ASSERT(0);
2923  return true;
2924  }
2925 }
2926 
2927 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
2928 {
2929  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
2930  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
2931  for(size_t i = 0; i < numberCount; ++i, ++pDst)
2932  {
2933  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
2934  }
2935 }
2936 
2937 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
2938 {
2939  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
2940  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
2941  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
2942  {
2943  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
2944  {
2945  return false;
2946  }
2947  }
2948  return true;
2949 }
2950 
2951 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2952 struct VmaMutexLock
2953 {
2954  VMA_CLASS_NO_COPY(VmaMutexLock)
2955 public:
2956  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2957  m_pMutex(useMutex ? &mutex : VMA_NULL)
2958  {
2959  if(m_pMutex)
2960  {
2961  m_pMutex->Lock();
2962  }
2963  }
2964 
2965  ~VmaMutexLock()
2966  {
2967  if(m_pMutex)
2968  {
2969  m_pMutex->Unlock();
2970  }
2971  }
2972 
2973 private:
2974  VMA_MUTEX* m_pMutex;
2975 };
2976 
2977 #if VMA_DEBUG_GLOBAL_MUTEX
2978  static VMA_MUTEX gDebugGlobalMutex;
2979  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2980 #else
2981  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2982 #endif
2983 
2984 // Minimum size of a free suballocation to register it in the free suballocation collection.
2985 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2986 
2987 /*
2988 Performs binary search and returns iterator to first element that is greater or
2989 equal to (key), according to comparison (cmp).
2990 
2991 Cmp should return true if first argument is less than second argument.
2992 
2993 Returned value is the found element, if present in the collection or place where
2994 new element with value (key) should be inserted.
2995 */
2996 template <typename IterT, typename KeyT, typename CmpT>
2997 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2998 {
2999  size_t down = 0, up = (end - beg);
3000  while(down < up)
3001  {
3002  const size_t mid = (down + up) / 2;
3003  if(cmp(*(beg+mid), key))
3004  {
3005  down = mid + 1;
3006  }
3007  else
3008  {
3009  up = mid;
3010  }
3011  }
3012  return beg + down;
3013 }
3014 
3016 // Memory allocation
3017 
3018 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
3019 {
3020  if((pAllocationCallbacks != VMA_NULL) &&
3021  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
3022  {
3023  return (*pAllocationCallbacks->pfnAllocation)(
3024  pAllocationCallbacks->pUserData,
3025  size,
3026  alignment,
3027  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3028  }
3029  else
3030  {
3031  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
3032  }
3033 }
3034 
3035 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
3036 {
3037  if((pAllocationCallbacks != VMA_NULL) &&
3038  (pAllocationCallbacks->pfnFree != VMA_NULL))
3039  {
3040  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
3041  }
3042  else
3043  {
3044  VMA_SYSTEM_FREE(ptr);
3045  }
3046 }
3047 
3048 template<typename T>
3049 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
3050 {
3051  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
3052 }
3053 
3054 template<typename T>
3055 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
3056 {
3057  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
3058 }
3059 
3060 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
3061 
3062 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
3063 
3064 template<typename T>
3065 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
3066 {
3067  ptr->~T();
3068  VmaFree(pAllocationCallbacks, ptr);
3069 }
3070 
3071 template<typename T>
3072 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
3073 {
3074  if(ptr != VMA_NULL)
3075  {
3076  for(size_t i = count; i--; )
3077  {
3078  ptr[i].~T();
3079  }
3080  VmaFree(pAllocationCallbacks, ptr);
3081  }
3082 }
3083 
3084 // STL-compatible allocator.
3085 template<typename T>
3086 class VmaStlAllocator
3087 {
3088 public:
3089  const VkAllocationCallbacks* const m_pCallbacks;
3090  typedef T value_type;
3091 
3092  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
3093  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
3094 
3095  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
3096  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
3097 
3098  template<typename U>
3099  bool operator==(const VmaStlAllocator<U>& rhs) const
3100  {
3101  return m_pCallbacks == rhs.m_pCallbacks;
3102  }
3103  template<typename U>
3104  bool operator!=(const VmaStlAllocator<U>& rhs) const
3105  {
3106  return m_pCallbacks != rhs.m_pCallbacks;
3107  }
3108 
3109  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
3110 };
3111 
3112 #if VMA_USE_STL_VECTOR
3113 
3114 #define VmaVector std::vector
3115 
3116 template<typename T, typename allocatorT>
3117 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
3118 {
3119  vec.insert(vec.begin() + index, item);
3120 }
3121 
3122 template<typename T, typename allocatorT>
3123 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
3124 {
3125  vec.erase(vec.begin() + index);
3126 }
3127 
3128 #else // #if VMA_USE_STL_VECTOR
3129 
3130 /* Class with interface compatible with subset of std::vector.
3131 T must be POD because constructors and destructors are not called and memcpy is
3132 used for these objects. */
3133 template<typename T, typename AllocatorT>
3134 class VmaVector
3135 {
3136 public:
3137  typedef T value_type;
3138 
3139  VmaVector(const AllocatorT& allocator) :
3140  m_Allocator(allocator),
3141  m_pArray(VMA_NULL),
3142  m_Count(0),
3143  m_Capacity(0)
3144  {
3145  }
3146 
3147  VmaVector(size_t count, const AllocatorT& allocator) :
3148  m_Allocator(allocator),
3149  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
3150  m_Count(count),
3151  m_Capacity(count)
3152  {
3153  }
3154 
3155  VmaVector(const VmaVector<T, AllocatorT>& src) :
3156  m_Allocator(src.m_Allocator),
3157  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
3158  m_Count(src.m_Count),
3159  m_Capacity(src.m_Count)
3160  {
3161  if(m_Count != 0)
3162  {
3163  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
3164  }
3165  }
3166 
3167  ~VmaVector()
3168  {
3169  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3170  }
3171 
3172  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
3173  {
3174  if(&rhs != this)
3175  {
3176  resize(rhs.m_Count);
3177  if(m_Count != 0)
3178  {
3179  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
3180  }
3181  }
3182  return *this;
3183  }
3184 
3185  bool empty() const { return m_Count == 0; }
3186  size_t size() const { return m_Count; }
3187  T* data() { return m_pArray; }
3188  const T* data() const { return m_pArray; }
3189 
3190  T& operator[](size_t index)
3191  {
3192  VMA_HEAVY_ASSERT(index < m_Count);
3193  return m_pArray[index];
3194  }
3195  const T& operator[](size_t index) const
3196  {
3197  VMA_HEAVY_ASSERT(index < m_Count);
3198  return m_pArray[index];
3199  }
3200 
3201  T& front()
3202  {
3203  VMA_HEAVY_ASSERT(m_Count > 0);
3204  return m_pArray[0];
3205  }
3206  const T& front() const
3207  {
3208  VMA_HEAVY_ASSERT(m_Count > 0);
3209  return m_pArray[0];
3210  }
3211  T& back()
3212  {
3213  VMA_HEAVY_ASSERT(m_Count > 0);
3214  return m_pArray[m_Count - 1];
3215  }
3216  const T& back() const
3217  {
3218  VMA_HEAVY_ASSERT(m_Count > 0);
3219  return m_pArray[m_Count - 1];
3220  }
3221 
3222  void reserve(size_t newCapacity, bool freeMemory = false)
3223  {
3224  newCapacity = VMA_MAX(newCapacity, m_Count);
3225 
3226  if((newCapacity < m_Capacity) && !freeMemory)
3227  {
3228  newCapacity = m_Capacity;
3229  }
3230 
3231  if(newCapacity != m_Capacity)
3232  {
3233  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
3234  if(m_Count != 0)
3235  {
3236  memcpy(newArray, m_pArray, m_Count * sizeof(T));
3237  }
3238  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3239  m_Capacity = newCapacity;
3240  m_pArray = newArray;
3241  }
3242  }
3243 
3244  void resize(size_t newCount, bool freeMemory = false)
3245  {
3246  size_t newCapacity = m_Capacity;
3247  if(newCount > m_Capacity)
3248  {
3249  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
3250  }
3251  else if(freeMemory)
3252  {
3253  newCapacity = newCount;
3254  }
3255 
3256  if(newCapacity != m_Capacity)
3257  {
3258  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
3259  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
3260  if(elementsToCopy != 0)
3261  {
3262  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
3263  }
3264  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3265  m_Capacity = newCapacity;
3266  m_pArray = newArray;
3267  }
3268 
3269  m_Count = newCount;
3270  }
3271 
3272  void clear(bool freeMemory = false)
3273  {
3274  resize(0, freeMemory);
3275  }
3276 
3277  void insert(size_t index, const T& src)
3278  {
3279  VMA_HEAVY_ASSERT(index <= m_Count);
3280  const size_t oldCount = size();
3281  resize(oldCount + 1);
3282  if(index < oldCount)
3283  {
3284  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
3285  }
3286  m_pArray[index] = src;
3287  }
3288 
3289  void remove(size_t index)
3290  {
3291  VMA_HEAVY_ASSERT(index < m_Count);
3292  const size_t oldCount = size();
3293  if(index < oldCount - 1)
3294  {
3295  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
3296  }
3297  resize(oldCount - 1);
3298  }
3299 
3300  void push_back(const T& src)
3301  {
3302  const size_t newIndex = size();
3303  resize(newIndex + 1);
3304  m_pArray[newIndex] = src;
3305  }
3306 
3307  void pop_back()
3308  {
3309  VMA_HEAVY_ASSERT(m_Count > 0);
3310  resize(size() - 1);
3311  }
3312 
3313  void push_front(const T& src)
3314  {
3315  insert(0, src);
3316  }
3317 
3318  void pop_front()
3319  {
3320  VMA_HEAVY_ASSERT(m_Count > 0);
3321  remove(0);
3322  }
3323 
3324  typedef T* iterator;
3325 
3326  iterator begin() { return m_pArray; }
3327  iterator end() { return m_pArray + m_Count; }
3328 
3329 private:
3330  AllocatorT m_Allocator;
3331  T* m_pArray;
3332  size_t m_Count;
3333  size_t m_Capacity;
3334 };
3335 
3336 template<typename T, typename allocatorT>
3337 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
3338 {
3339  vec.insert(index, item);
3340 }
3341 
3342 template<typename T, typename allocatorT>
3343 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
3344 {
3345  vec.remove(index);
3346 }
3347 
3348 #endif // #if VMA_USE_STL_VECTOR
3349 
3350 template<typename CmpLess, typename VectorT>
3351 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
3352 {
3353  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3354  vector.data(),
3355  vector.data() + vector.size(),
3356  value,
3357  CmpLess()) - vector.data();
3358  VmaVectorInsert(vector, indexToInsert, value);
3359  return indexToInsert;
3360 }
3361 
3362 template<typename CmpLess, typename VectorT>
3363 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
3364 {
3365  CmpLess comparator;
3366  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3367  vector.begin(),
3368  vector.end(),
3369  value,
3370  comparator);
3371  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
3372  {
3373  size_t indexToRemove = it - vector.begin();
3374  VmaVectorRemove(vector, indexToRemove);
3375  return true;
3376  }
3377  return false;
3378 }
3379 
3380 template<typename CmpLess, typename VectorT>
3381 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
3382 {
3383  CmpLess comparator;
3384  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3385  vector.data(),
3386  vector.data() + vector.size(),
3387  value,
3388  comparator);
3389  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
3390  {
3391  return it - vector.begin();
3392  }
3393  else
3394  {
3395  return vector.size();
3396  }
3397 }
3398 
3400 // class VmaPoolAllocator
3401 
3402 /*
3403 Allocator for objects of type T using a list of arrays (pools) to speed up
3404 allocation. Number of elements that can be allocated is not bounded because
3405 allocator can create multiple blocks.
3406 */
3407 template<typename T>
3408 class VmaPoolAllocator
3409 {
3410  VMA_CLASS_NO_COPY(VmaPoolAllocator)
3411 public:
3412  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
3413  ~VmaPoolAllocator();
3414  void Clear();
3415  T* Alloc();
3416  void Free(T* ptr);
3417 
3418 private:
3419  union Item
3420  {
3421  uint32_t NextFreeIndex;
3422  T Value;
3423  };
3424 
3425  struct ItemBlock
3426  {
3427  Item* pItems;
3428  uint32_t FirstFreeIndex;
3429  };
3430 
3431  const VkAllocationCallbacks* m_pAllocationCallbacks;
3432  size_t m_ItemsPerBlock;
3433  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
3434 
3435  ItemBlock& CreateNewBlock();
3436 };
3437 
3438 template<typename T>
3439 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
3440  m_pAllocationCallbacks(pAllocationCallbacks),
3441  m_ItemsPerBlock(itemsPerBlock),
3442  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
3443 {
3444  VMA_ASSERT(itemsPerBlock > 0);
3445 }
3446 
3447 template<typename T>
3448 VmaPoolAllocator<T>::~VmaPoolAllocator()
3449 {
3450  Clear();
3451 }
3452 
3453 template<typename T>
3454 void VmaPoolAllocator<T>::Clear()
3455 {
3456  for(size_t i = m_ItemBlocks.size(); i--; )
3457  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
3458  m_ItemBlocks.clear();
3459 }
3460 
3461 template<typename T>
3462 T* VmaPoolAllocator<T>::Alloc()
3463 {
3464  for(size_t i = m_ItemBlocks.size(); i--; )
3465  {
3466  ItemBlock& block = m_ItemBlocks[i];
3467  // This block has some free items: Use first one.
3468  if(block.FirstFreeIndex != UINT32_MAX)
3469  {
3470  Item* const pItem = &block.pItems[block.FirstFreeIndex];
3471  block.FirstFreeIndex = pItem->NextFreeIndex;
3472  return &pItem->Value;
3473  }
3474  }
3475 
3476  // No block has free item: Create new one and use it.
3477  ItemBlock& newBlock = CreateNewBlock();
3478  Item* const pItem = &newBlock.pItems[0];
3479  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
3480  return &pItem->Value;
3481 }
3482 
3483 template<typename T>
3484 void VmaPoolAllocator<T>::Free(T* ptr)
3485 {
3486  // Search all memory blocks to find ptr.
3487  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
3488  {
3489  ItemBlock& block = m_ItemBlocks[i];
3490 
3491  // Casting to union.
3492  Item* pItemPtr;
3493  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
3494 
3495  // Check if pItemPtr is in address range of this block.
3496  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
3497  {
3498  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
3499  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
3500  block.FirstFreeIndex = index;
3501  return;
3502  }
3503  }
3504  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
3505 }
3506 
3507 template<typename T>
3508 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
3509 {
3510  ItemBlock newBlock = {
3511  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
3512 
3513  m_ItemBlocks.push_back(newBlock);
3514 
3515  // Setup singly-linked list of all free items in this block.
3516  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
3517  newBlock.pItems[i].NextFreeIndex = i + 1;
3518  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
3519  return m_ItemBlocks.back();
3520 }
3521 
3523 // class VmaRawList, VmaList
3524 
3525 #if VMA_USE_STL_LIST
3526 
3527 #define VmaList std::list
3528 
3529 #else // #if VMA_USE_STL_LIST
3530 
3531 template<typename T>
3532 struct VmaListItem
3533 {
3534  VmaListItem* pPrev;
3535  VmaListItem* pNext;
3536  T Value;
3537 };
3538 
3539 // Doubly linked list.
3540 template<typename T>
3541 class VmaRawList
3542 {
3543  VMA_CLASS_NO_COPY(VmaRawList)
3544 public:
3545  typedef VmaListItem<T> ItemType;
3546 
3547  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
3548  ~VmaRawList();
3549  void Clear();
3550 
3551  size_t GetCount() const { return m_Count; }
3552  bool IsEmpty() const { return m_Count == 0; }
3553 
3554  ItemType* Front() { return m_pFront; }
3555  const ItemType* Front() const { return m_pFront; }
3556  ItemType* Back() { return m_pBack; }
3557  const ItemType* Back() const { return m_pBack; }
3558 
3559  ItemType* PushBack();
3560  ItemType* PushFront();
3561  ItemType* PushBack(const T& value);
3562  ItemType* PushFront(const T& value);
3563  void PopBack();
3564  void PopFront();
3565 
3566  // Item can be null - it means PushBack.
3567  ItemType* InsertBefore(ItemType* pItem);
3568  // Item can be null - it means PushFront.
3569  ItemType* InsertAfter(ItemType* pItem);
3570 
3571  ItemType* InsertBefore(ItemType* pItem, const T& value);
3572  ItemType* InsertAfter(ItemType* pItem, const T& value);
3573 
3574  void Remove(ItemType* pItem);
3575 
3576 private:
3577  const VkAllocationCallbacks* const m_pAllocationCallbacks;
3578  VmaPoolAllocator<ItemType> m_ItemAllocator;
3579  ItemType* m_pFront;
3580  ItemType* m_pBack;
3581  size_t m_Count;
3582 };
3583 
3584 template<typename T>
3585 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3586  m_pAllocationCallbacks(pAllocationCallbacks),
3587  m_ItemAllocator(pAllocationCallbacks, 128),
3588  m_pFront(VMA_NULL),
3589  m_pBack(VMA_NULL),
3590  m_Count(0)
3591 {
3592 }
3593 
3594 template<typename T>
3595 VmaRawList<T>::~VmaRawList()
3596 {
3597  // Intentionally not calling Clear, because that would be unnecessary
3598  // computations to return all items to m_ItemAllocator as free.
3599 }
3600 
3601 template<typename T>
3602 void VmaRawList<T>::Clear()
3603 {
3604  if(IsEmpty() == false)
3605  {
3606  ItemType* pItem = m_pBack;
3607  while(pItem != VMA_NULL)
3608  {
3609  ItemType* const pPrevItem = pItem->pPrev;
3610  m_ItemAllocator.Free(pItem);
3611  pItem = pPrevItem;
3612  }
3613  m_pFront = VMA_NULL;
3614  m_pBack = VMA_NULL;
3615  m_Count = 0;
3616  }
3617 }
3618 
3619 template<typename T>
3620 VmaListItem<T>* VmaRawList<T>::PushBack()
3621 {
3622  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3623  pNewItem->pNext = VMA_NULL;
3624  if(IsEmpty())
3625  {
3626  pNewItem->pPrev = VMA_NULL;
3627  m_pFront = pNewItem;
3628  m_pBack = pNewItem;
3629  m_Count = 1;
3630  }
3631  else
3632  {
3633  pNewItem->pPrev = m_pBack;
3634  m_pBack->pNext = pNewItem;
3635  m_pBack = pNewItem;
3636  ++m_Count;
3637  }
3638  return pNewItem;
3639 }
3640 
3641 template<typename T>
3642 VmaListItem<T>* VmaRawList<T>::PushFront()
3643 {
3644  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3645  pNewItem->pPrev = VMA_NULL;
3646  if(IsEmpty())
3647  {
3648  pNewItem->pNext = VMA_NULL;
3649  m_pFront = pNewItem;
3650  m_pBack = pNewItem;
3651  m_Count = 1;
3652  }
3653  else
3654  {
3655  pNewItem->pNext = m_pFront;
3656  m_pFront->pPrev = pNewItem;
3657  m_pFront = pNewItem;
3658  ++m_Count;
3659  }
3660  return pNewItem;
3661 }
3662 
3663 template<typename T>
3664 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3665 {
3666  ItemType* const pNewItem = PushBack();
3667  pNewItem->Value = value;
3668  return pNewItem;
3669 }
3670 
3671 template<typename T>
3672 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3673 {
3674  ItemType* const pNewItem = PushFront();
3675  pNewItem->Value = value;
3676  return pNewItem;
3677 }
3678 
3679 template<typename T>
3680 void VmaRawList<T>::PopBack()
3681 {
3682  VMA_HEAVY_ASSERT(m_Count > 0);
3683  ItemType* const pBackItem = m_pBack;
3684  ItemType* const pPrevItem = pBackItem->pPrev;
3685  if(pPrevItem != VMA_NULL)
3686  {
3687  pPrevItem->pNext = VMA_NULL;
3688  }
3689  m_pBack = pPrevItem;
3690  m_ItemAllocator.Free(pBackItem);
3691  --m_Count;
3692 }
3693 
3694 template<typename T>
3695 void VmaRawList<T>::PopFront()
3696 {
3697  VMA_HEAVY_ASSERT(m_Count > 0);
3698  ItemType* const pFrontItem = m_pFront;
3699  ItemType* const pNextItem = pFrontItem->pNext;
3700  if(pNextItem != VMA_NULL)
3701  {
3702  pNextItem->pPrev = VMA_NULL;
3703  }
3704  m_pFront = pNextItem;
3705  m_ItemAllocator.Free(pFrontItem);
3706  --m_Count;
3707 }
3708 
3709 template<typename T>
3710 void VmaRawList<T>::Remove(ItemType* pItem)
3711 {
3712  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3713  VMA_HEAVY_ASSERT(m_Count > 0);
3714 
3715  if(pItem->pPrev != VMA_NULL)
3716  {
3717  pItem->pPrev->pNext = pItem->pNext;
3718  }
3719  else
3720  {
3721  VMA_HEAVY_ASSERT(m_pFront == pItem);
3722  m_pFront = pItem->pNext;
3723  }
3724 
3725  if(pItem->pNext != VMA_NULL)
3726  {
3727  pItem->pNext->pPrev = pItem->pPrev;
3728  }
3729  else
3730  {
3731  VMA_HEAVY_ASSERT(m_pBack == pItem);
3732  m_pBack = pItem->pPrev;
3733  }
3734 
3735  m_ItemAllocator.Free(pItem);
3736  --m_Count;
3737 }
3738 
3739 template<typename T>
3740 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3741 {
3742  if(pItem != VMA_NULL)
3743  {
3744  ItemType* const prevItem = pItem->pPrev;
3745  ItemType* const newItem = m_ItemAllocator.Alloc();
3746  newItem->pPrev = prevItem;
3747  newItem->pNext = pItem;
3748  pItem->pPrev = newItem;
3749  if(prevItem != VMA_NULL)
3750  {
3751  prevItem->pNext = newItem;
3752  }
3753  else
3754  {
3755  VMA_HEAVY_ASSERT(m_pFront == pItem);
3756  m_pFront = newItem;
3757  }
3758  ++m_Count;
3759  return newItem;
3760  }
3761  else
3762  return PushBack();
3763 }
3764 
3765 template<typename T>
3766 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3767 {
3768  if(pItem != VMA_NULL)
3769  {
3770  ItemType* const nextItem = pItem->pNext;
3771  ItemType* const newItem = m_ItemAllocator.Alloc();
3772  newItem->pNext = nextItem;
3773  newItem->pPrev = pItem;
3774  pItem->pNext = newItem;
3775  if(nextItem != VMA_NULL)
3776  {
3777  nextItem->pPrev = newItem;
3778  }
3779  else
3780  {
3781  VMA_HEAVY_ASSERT(m_pBack == pItem);
3782  m_pBack = newItem;
3783  }
3784  ++m_Count;
3785  return newItem;
3786  }
3787  else
3788  return PushFront();
3789 }
3790 
3791 template<typename T>
3792 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3793 {
3794  ItemType* const newItem = InsertBefore(pItem);
3795  newItem->Value = value;
3796  return newItem;
3797 }
3798 
3799 template<typename T>
3800 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3801 {
3802  ItemType* const newItem = InsertAfter(pItem);
3803  newItem->Value = value;
3804  return newItem;
3805 }
3806 
3807 template<typename T, typename AllocatorT>
3808 class VmaList
3809 {
3810  VMA_CLASS_NO_COPY(VmaList)
3811 public:
3812  class iterator
3813  {
3814  public:
3815  iterator() :
3816  m_pList(VMA_NULL),
3817  m_pItem(VMA_NULL)
3818  {
3819  }
3820 
3821  T& operator*() const
3822  {
3823  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3824  return m_pItem->Value;
3825  }
3826  T* operator->() const
3827  {
3828  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3829  return &m_pItem->Value;
3830  }
3831 
3832  iterator& operator++()
3833  {
3834  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3835  m_pItem = m_pItem->pNext;
3836  return *this;
3837  }
3838  iterator& operator--()
3839  {
3840  if(m_pItem != VMA_NULL)
3841  {
3842  m_pItem = m_pItem->pPrev;
3843  }
3844  else
3845  {
3846  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3847  m_pItem = m_pList->Back();
3848  }
3849  return *this;
3850  }
3851 
3852  iterator operator++(int)
3853  {
3854  iterator result = *this;
3855  ++*this;
3856  return result;
3857  }
3858  iterator operator--(int)
3859  {
3860  iterator result = *this;
3861  --*this;
3862  return result;
3863  }
3864 
3865  bool operator==(const iterator& rhs) const
3866  {
3867  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3868  return m_pItem == rhs.m_pItem;
3869  }
3870  bool operator!=(const iterator& rhs) const
3871  {
3872  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3873  return m_pItem != rhs.m_pItem;
3874  }
3875 
3876  private:
3877  VmaRawList<T>* m_pList;
3878  VmaListItem<T>* m_pItem;
3879 
3880  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3881  m_pList(pList),
3882  m_pItem(pItem)
3883  {
3884  }
3885 
3886  friend class VmaList<T, AllocatorT>;
3887  };
3888 
3889  class const_iterator
3890  {
3891  public:
3892  const_iterator() :
3893  m_pList(VMA_NULL),
3894  m_pItem(VMA_NULL)
3895  {
3896  }
3897 
3898  const_iterator(const iterator& src) :
3899  m_pList(src.m_pList),
3900  m_pItem(src.m_pItem)
3901  {
3902  }
3903 
3904  const T& operator*() const
3905  {
3906  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3907  return m_pItem->Value;
3908  }
3909  const T* operator->() const
3910  {
3911  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3912  return &m_pItem->Value;
3913  }
3914 
3915  const_iterator& operator++()
3916  {
3917  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3918  m_pItem = m_pItem->pNext;
3919  return *this;
3920  }
3921  const_iterator& operator--()
3922  {
3923  if(m_pItem != VMA_NULL)
3924  {
3925  m_pItem = m_pItem->pPrev;
3926  }
3927  else
3928  {
3929  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3930  m_pItem = m_pList->Back();
3931  }
3932  return *this;
3933  }
3934 
3935  const_iterator operator++(int)
3936  {
3937  const_iterator result = *this;
3938  ++*this;
3939  return result;
3940  }
3941  const_iterator operator--(int)
3942  {
3943  const_iterator result = *this;
3944  --*this;
3945  return result;
3946  }
3947 
3948  bool operator==(const const_iterator& rhs) const
3949  {
3950  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3951  return m_pItem == rhs.m_pItem;
3952  }
3953  bool operator!=(const const_iterator& rhs) const
3954  {
3955  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3956  return m_pItem != rhs.m_pItem;
3957  }
3958 
3959  private:
3960  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3961  m_pList(pList),
3962  m_pItem(pItem)
3963  {
3964  }
3965 
3966  const VmaRawList<T>* m_pList;
3967  const VmaListItem<T>* m_pItem;
3968 
3969  friend class VmaList<T, AllocatorT>;
3970  };
3971 
3972  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3973 
3974  bool empty() const { return m_RawList.IsEmpty(); }
3975  size_t size() const { return m_RawList.GetCount(); }
3976 
3977  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3978  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3979 
3980  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3981  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3982 
3983  void clear() { m_RawList.Clear(); }
3984  void push_back(const T& value) { m_RawList.PushBack(value); }
3985  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3986  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3987 
3988 private:
3989  VmaRawList<T> m_RawList;
3990 };
3991 
3992 #endif // #if VMA_USE_STL_LIST
3993 
3995 // class VmaMap
3996 
3997 // Unused in this version.
3998 #if 0
3999 
4000 #if VMA_USE_STL_UNORDERED_MAP
4001 
4002 #define VmaPair std::pair
4003 
4004 #define VMA_MAP_TYPE(KeyT, ValueT) \
4005  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
4006 
4007 #else // #if VMA_USE_STL_UNORDERED_MAP
4008 
4009 template<typename T1, typename T2>
4010 struct VmaPair
4011 {
4012  T1 first;
4013  T2 second;
4014 
4015  VmaPair() : first(), second() { }
4016  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
4017 };
4018 
4019 /* Class compatible with subset of interface of std::unordered_map.
4020 KeyT, ValueT must be POD because they will be stored in VmaVector.
4021 */
4022 template<typename KeyT, typename ValueT>
4023 class VmaMap
4024 {
4025 public:
4026  typedef VmaPair<KeyT, ValueT> PairType;
4027  typedef PairType* iterator;
4028 
4029  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
4030 
4031  iterator begin() { return m_Vector.begin(); }
4032  iterator end() { return m_Vector.end(); }
4033 
4034  void insert(const PairType& pair);
4035  iterator find(const KeyT& key);
4036  void erase(iterator it);
4037 
4038 private:
4039  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
4040 };
4041 
4042 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
4043 
4044 template<typename FirstT, typename SecondT>
4045 struct VmaPairFirstLess
4046 {
4047  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
4048  {
4049  return lhs.first < rhs.first;
4050  }
4051  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
4052  {
4053  return lhs.first < rhsFirst;
4054  }
4055 };
4056 
4057 template<typename KeyT, typename ValueT>
4058 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
4059 {
4060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4061  m_Vector.data(),
4062  m_Vector.data() + m_Vector.size(),
4063  pair,
4064  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
4065  VmaVectorInsert(m_Vector, indexToInsert, pair);
4066 }
4067 
4068 template<typename KeyT, typename ValueT>
4069 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
4070 {
4071  PairType* it = VmaBinaryFindFirstNotLess(
4072  m_Vector.data(),
4073  m_Vector.data() + m_Vector.size(),
4074  key,
4075  VmaPairFirstLess<KeyT, ValueT>());
4076  if((it != m_Vector.end()) && (it->first == key))
4077  {
4078  return it;
4079  }
4080  else
4081  {
4082  return m_Vector.end();
4083  }
4084 }
4085 
4086 template<typename KeyT, typename ValueT>
4087 void VmaMap<KeyT, ValueT>::erase(iterator it)
4088 {
4089  VmaVectorRemove(m_Vector, it - m_Vector.begin());
4090 }
4091 
4092 #endif // #if VMA_USE_STL_UNORDERED_MAP
4093 
4094 #endif // #if 0
4095 
4097 
4098 class VmaDeviceMemoryBlock;
4099 
4100 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
4101 
4102 struct VmaAllocation_T
4103 {
4104  VMA_CLASS_NO_COPY(VmaAllocation_T)
4105 private:
4106  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
4107 
4108  enum FLAGS
4109  {
4110  FLAG_USER_DATA_STRING = 0x01,
4111  };
4112 
4113 public:
4114  enum ALLOCATION_TYPE
4115  {
4116  ALLOCATION_TYPE_NONE,
4117  ALLOCATION_TYPE_BLOCK,
4118  ALLOCATION_TYPE_DEDICATED,
4119  };
4120 
4121  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
4122  m_Alignment(1),
4123  m_Size(0),
4124  m_pUserData(VMA_NULL),
4125  m_LastUseFrameIndex(currentFrameIndex),
4126  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
4127  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
4128  m_MapCount(0),
4129  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
4130  {
4131 #if VMA_STATS_STRING_ENABLED
4132  m_CreationFrameIndex = currentFrameIndex;
4133  m_BufferImageUsage = 0;
4134 #endif
4135  }
4136 
4137  ~VmaAllocation_T()
4138  {
4139  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
4140 
4141  // Check if owned string was freed.
4142  VMA_ASSERT(m_pUserData == VMA_NULL);
4143  }
4144 
4145  void InitBlockAllocation(
4146  VmaPool hPool,
4147  VmaDeviceMemoryBlock* block,
4148  VkDeviceSize offset,
4149  VkDeviceSize alignment,
4150  VkDeviceSize size,
4151  VmaSuballocationType suballocationType,
4152  bool mapped,
4153  bool canBecomeLost)
4154  {
4155  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4156  VMA_ASSERT(block != VMA_NULL);
4157  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4158  m_Alignment = alignment;
4159  m_Size = size;
4160  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4161  m_SuballocationType = (uint8_t)suballocationType;
4162  m_BlockAllocation.m_hPool = hPool;
4163  m_BlockAllocation.m_Block = block;
4164  m_BlockAllocation.m_Offset = offset;
4165  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
4166  }
4167 
4168  void InitLost()
4169  {
4170  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4171  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
4172  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4173  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
4174  m_BlockAllocation.m_Block = VMA_NULL;
4175  m_BlockAllocation.m_Offset = 0;
4176  m_BlockAllocation.m_CanBecomeLost = true;
4177  }
4178 
4179  void ChangeBlockAllocation(
4180  VmaAllocator hAllocator,
4181  VmaDeviceMemoryBlock* block,
4182  VkDeviceSize offset);
4183 
4184  // pMappedData not null means allocation is created with MAPPED flag.
4185  void InitDedicatedAllocation(
4186  uint32_t memoryTypeIndex,
4187  VkDeviceMemory hMemory,
4188  VmaSuballocationType suballocationType,
4189  void* pMappedData,
4190  VkDeviceSize size)
4191  {
4192  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4193  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
4194  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
4195  m_Alignment = 0;
4196  m_Size = size;
4197  m_SuballocationType = (uint8_t)suballocationType;
4198  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4199  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
4200  m_DedicatedAllocation.m_hMemory = hMemory;
4201  m_DedicatedAllocation.m_pMappedData = pMappedData;
4202  }
4203 
4204  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
4205  VkDeviceSize GetAlignment() const { return m_Alignment; }
4206  VkDeviceSize GetSize() const { return m_Size; }
4207  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
4208  void* GetUserData() const { return m_pUserData; }
4209  void SetUserData(VmaAllocator hAllocator, void* pUserData);
4210  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
4211 
4212  VmaDeviceMemoryBlock* GetBlock() const
4213  {
4214  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4215  return m_BlockAllocation.m_Block;
4216  }
4217  VkDeviceSize GetOffset() const;
4218  VkDeviceMemory GetMemory() const;
4219  uint32_t GetMemoryTypeIndex() const;
4220  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
4221  void* GetMappedData() const;
4222  bool CanBecomeLost() const;
4223  VmaPool GetPool() const;
4224 
4225  uint32_t GetLastUseFrameIndex() const
4226  {
4227  return m_LastUseFrameIndex.load();
4228  }
4229  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
4230  {
4231  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
4232  }
4233  /*
4234  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
4235  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
4236  - Else, returns false.
4237 
4238  If hAllocation is already lost, assert - you should not call it then.
4239  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
4240  */
4241  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4242 
4243  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
4244  {
4245  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
4246  outInfo.blockCount = 1;
4247  outInfo.allocationCount = 1;
4248  outInfo.unusedRangeCount = 0;
4249  outInfo.usedBytes = m_Size;
4250  outInfo.unusedBytes = 0;
4251  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
4252  outInfo.unusedRangeSizeMin = UINT64_MAX;
4253  outInfo.unusedRangeSizeMax = 0;
4254  }
4255 
4256  void BlockAllocMap();
4257  void BlockAllocUnmap();
4258  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
4259  void DedicatedAllocUnmap(VmaAllocator hAllocator);
4260 
4261 #if VMA_STATS_STRING_ENABLED
4262  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
4263  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
4264 
4265  void InitBufferImageUsage(uint32_t bufferImageUsage)
4266  {
4267  VMA_ASSERT(m_BufferImageUsage == 0);
4268  m_BufferImageUsage = bufferImageUsage;
4269  }
4270 
4271  void PrintParameters(class VmaJsonWriter& json) const;
4272 #endif
4273 
4274 private:
4275  VkDeviceSize m_Alignment;
4276  VkDeviceSize m_Size;
4277  void* m_pUserData;
4278  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
4279  uint8_t m_Type; // ALLOCATION_TYPE
4280  uint8_t m_SuballocationType; // VmaSuballocationType
4281  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
4282  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
4283  uint8_t m_MapCount;
4284  uint8_t m_Flags; // enum FLAGS
4285 
4286  // Allocation out of VmaDeviceMemoryBlock.
4287  struct BlockAllocation
4288  {
4289  VmaPool m_hPool; // Null if belongs to general memory.
4290  VmaDeviceMemoryBlock* m_Block;
4291  VkDeviceSize m_Offset;
4292  bool m_CanBecomeLost;
4293  };
4294 
4295  // Allocation for an object that has its own private VkDeviceMemory.
4296  struct DedicatedAllocation
4297  {
4298  uint32_t m_MemoryTypeIndex;
4299  VkDeviceMemory m_hMemory;
4300  void* m_pMappedData; // Not null means memory is mapped.
4301  };
4302 
4303  union
4304  {
4305  // Allocation out of VmaDeviceMemoryBlock.
4306  BlockAllocation m_BlockAllocation;
4307  // Allocation for an object that has its own private VkDeviceMemory.
4308  DedicatedAllocation m_DedicatedAllocation;
4309  };
4310 
4311 #if VMA_STATS_STRING_ENABLED
4312  uint32_t m_CreationFrameIndex;
4313  uint32_t m_BufferImageUsage; // 0 if unknown.
4314 #endif
4315 
4316  void FreeUserDataString(VmaAllocator hAllocator);
4317 };
4318 
4319 /*
4320 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
4321 allocated memory block or free.
4322 */
4323 struct VmaSuballocation
4324 {
4325  VkDeviceSize offset;
4326  VkDeviceSize size;
4327  VmaAllocation hAllocation;
4328  VmaSuballocationType type;
4329 };
4330 
4331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
4332 
4333 // Cost of one additional allocation lost, as equivalent in bytes.
4334 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
4335 
4336 /*
4337 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
4338 
4339 If canMakeOtherLost was false:
4340 - item points to a FREE suballocation.
4341 - itemsToMakeLostCount is 0.
4342 
4343 If canMakeOtherLost was true:
4344 - item points to first of sequence of suballocations, which are either FREE,
4345  or point to VmaAllocations that can become lost.
4346 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
4347  the requested allocation to succeed.
4348 */
4349 struct VmaAllocationRequest
4350 {
4351  VkDeviceSize offset;
4352  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
4353  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
4354  VmaSuballocationList::iterator item;
4355  size_t itemsToMakeLostCount;
4356 
4357  VkDeviceSize CalcCost() const
4358  {
4359  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
4360  }
4361 };
4362 
4363 /*
4364 Data structure used for bookkeeping of allocations and unused ranges of memory
4365 in a single VkDeviceMemory block.
4366 */
4367 class VmaBlockMetadata
4368 {
4369  VMA_CLASS_NO_COPY(VmaBlockMetadata)
4370 public:
4371  VmaBlockMetadata(VmaAllocator hAllocator);
4372  ~VmaBlockMetadata();
4373  void Init(VkDeviceSize size);
4374 
4375  // Validates all data structures inside this object. If not valid, returns false.
4376  bool Validate() const;
4377  VkDeviceSize GetSize() const { return m_Size; }
4378  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
4379  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4380  VkDeviceSize GetUnusedRangeSizeMax() const;
4381  // Returns true if this block is empty - contains only single free suballocation.
4382  bool IsEmpty() const;
4383 
4384  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4385  void AddPoolStats(VmaPoolStats& inoutStats) const;
4386 
4387 #if VMA_STATS_STRING_ENABLED
4388  void PrintDetailedMap(class VmaJsonWriter& json) const;
4389 #endif
4390 
4391  // Tries to find a place for suballocation with given parameters inside this block.
4392  // If succeeded, fills pAllocationRequest and returns true.
4393  // If failed, returns false.
4394  bool CreateAllocationRequest(
4395  uint32_t currentFrameIndex,
4396  uint32_t frameInUseCount,
4397  VkDeviceSize bufferImageGranularity,
4398  VkDeviceSize allocSize,
4399  VkDeviceSize allocAlignment,
4400  VmaSuballocationType allocType,
4401  bool canMakeOtherLost,
4402  VmaAllocationRequest* pAllocationRequest);
4403 
4404  bool MakeRequestedAllocationsLost(
4405  uint32_t currentFrameIndex,
4406  uint32_t frameInUseCount,
4407  VmaAllocationRequest* pAllocationRequest);
4408 
4409  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4410 
4411  VkResult CheckCorruption(const void* pBlockData);
4412 
4413  // Makes actual allocation based on request. Request must already be checked and valid.
4414  void Alloc(
4415  const VmaAllocationRequest& request,
4416  VmaSuballocationType type,
4417  VkDeviceSize allocSize,
4418  VmaAllocation hAllocation);
4419 
4420  // Frees suballocation assigned to given memory region.
4421  void Free(const VmaAllocation allocation);
4422  void FreeAtOffset(VkDeviceSize offset);
4423 
4424 private:
4425  VkDeviceSize m_Size;
4426  uint32_t m_FreeCount;
4427  VkDeviceSize m_SumFreeSize;
4428  VmaSuballocationList m_Suballocations;
4429  // Suballocations that are free and have size greater than certain threshold.
4430  // Sorted by size, ascending.
4431  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
4432 
4433  bool ValidateFreeSuballocationList() const;
4434 
4435  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
4436  // If yes, fills pOffset and returns true. If no, returns false.
4437  bool CheckAllocation(
4438  uint32_t currentFrameIndex,
4439  uint32_t frameInUseCount,
4440  VkDeviceSize bufferImageGranularity,
4441  VkDeviceSize allocSize,
4442  VkDeviceSize allocAlignment,
4443  VmaSuballocationType allocType,
4444  VmaSuballocationList::const_iterator suballocItem,
4445  bool canMakeOtherLost,
4446  VkDeviceSize* pOffset,
4447  size_t* itemsToMakeLostCount,
4448  VkDeviceSize* pSumFreeSize,
4449  VkDeviceSize* pSumItemSize) const;
4450  // Given free suballocation, it merges it with following one, which must also be free.
4451  void MergeFreeWithNext(VmaSuballocationList::iterator item);
4452  // Releases given suballocation, making it free.
4453  // Merges it with adjacent free suballocations if applicable.
4454  // Returns iterator to new free suballocation at this place.
4455  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
4456  // Given free suballocation, it inserts it into sorted list of
4457  // m_FreeSuballocationsBySize if it's suitable.
4458  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
4459  // Given free suballocation, it removes it from sorted list of
4460  // m_FreeSuballocationsBySize if it's suitable.
4461  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
4462 };
4463 
4464 /*
4465 Represents a single block of device memory (`VkDeviceMemory`) with all the
4466 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
4467 
4468 Thread-safety: This class must be externally synchronized.
4469 */
4470 class VmaDeviceMemoryBlock
4471 {
4472  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
4473 public:
4474  VmaBlockMetadata m_Metadata;
4475 
4476  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
4477 
4478  ~VmaDeviceMemoryBlock()
4479  {
4480  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
4481  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
4482  }
4483 
4484  // Always call after construction.
4485  void Init(
4486  uint32_t newMemoryTypeIndex,
4487  VkDeviceMemory newMemory,
4488  VkDeviceSize newSize,
4489  uint32_t id);
4490  // Always call before destruction.
4491  void Destroy(VmaAllocator allocator);
4492 
4493  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
4494  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4495  uint32_t GetId() const { return m_Id; }
4496  void* GetMappedData() const { return m_pMappedData; }
4497 
4498  // Validates all data structures inside this object. If not valid, returns false.
4499  bool Validate() const;
4500 
4501  VkResult CheckCorruption(VmaAllocator hAllocator);
4502 
4503  // ppData can be null.
4504  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
4505  void Unmap(VmaAllocator hAllocator, uint32_t count);
4506 
4507  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4508  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4509 
4510  VkResult BindBufferMemory(
4511  const VmaAllocator hAllocator,
4512  const VmaAllocation hAllocation,
4513  VkBuffer hBuffer);
4514  VkResult BindImageMemory(
4515  const VmaAllocator hAllocator,
4516  const VmaAllocation hAllocation,
4517  VkImage hImage);
4518 
4519 private:
4520  uint32_t m_MemoryTypeIndex;
4521  uint32_t m_Id;
4522  VkDeviceMemory m_hMemory;
4523 
4524  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
4525  // Also protects m_MapCount, m_pMappedData.
4526  VMA_MUTEX m_Mutex;
4527  uint32_t m_MapCount;
4528  void* m_pMappedData;
4529 };
4530 
4531 struct VmaPointerLess
4532 {
4533  bool operator()(const void* lhs, const void* rhs) const
4534  {
4535  return lhs < rhs;
4536  }
4537 };
4538 
4539 class VmaDefragmentator;
4540 
4541 /*
4542 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
4543 Vulkan memory type.
4544 
4545 Synchronized internally with a mutex.
4546 */
4547 struct VmaBlockVector
4548 {
4549  VMA_CLASS_NO_COPY(VmaBlockVector)
4550 public:
4551  VmaBlockVector(
4552  VmaAllocator hAllocator,
4553  uint32_t memoryTypeIndex,
4554  VkDeviceSize preferredBlockSize,
4555  size_t minBlockCount,
4556  size_t maxBlockCount,
4557  VkDeviceSize bufferImageGranularity,
4558  uint32_t frameInUseCount,
4559  bool isCustomPool);
4560  ~VmaBlockVector();
4561 
4562  VkResult CreateMinBlocks();
4563 
4564  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4565  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
4566  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
4567  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
4568 
4569  void GetPoolStats(VmaPoolStats* pStats);
4570 
4571  bool IsEmpty() const { return m_Blocks.empty(); }
4572  bool IsCorruptionDetectionEnabled() const;
4573 
4574  VkResult Allocate(
4575  VmaPool hCurrentPool,
4576  uint32_t currentFrameIndex,
4577  VkDeviceSize size,
4578  VkDeviceSize alignment,
4579  const VmaAllocationCreateInfo& createInfo,
4580  VmaSuballocationType suballocType,
4581  VmaAllocation* pAllocation);
4582 
4583  void Free(
4584  VmaAllocation hAllocation);
4585 
4586  // Adds statistics of this BlockVector to pStats.
4587  void AddStats(VmaStats* pStats);
4588 
4589 #if VMA_STATS_STRING_ENABLED
4590  void PrintDetailedMap(class VmaJsonWriter& json);
4591 #endif
4592 
4593  void MakePoolAllocationsLost(
4594  uint32_t currentFrameIndex,
4595  size_t* pLostAllocationCount);
4596  VkResult CheckCorruption();
4597 
4598  VmaDefragmentator* EnsureDefragmentator(
4599  VmaAllocator hAllocator,
4600  uint32_t currentFrameIndex);
4601 
4602  VkResult Defragment(
4603  VmaDefragmentationStats* pDefragmentationStats,
4604  VkDeviceSize& maxBytesToMove,
4605  uint32_t& maxAllocationsToMove);
4606 
4607  void DestroyDefragmentator();
4608 
4609 private:
4610  friend class VmaDefragmentator;
4611 
4612  const VmaAllocator m_hAllocator;
4613  const uint32_t m_MemoryTypeIndex;
4614  const VkDeviceSize m_PreferredBlockSize;
4615  const size_t m_MinBlockCount;
4616  const size_t m_MaxBlockCount;
4617  const VkDeviceSize m_BufferImageGranularity;
4618  const uint32_t m_FrameInUseCount;
4619  const bool m_IsCustomPool;
4620  VMA_MUTEX m_Mutex;
4621  // Incrementally sorted by sumFreeSize, ascending.
4622  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4623  /* There can be at most one allocation that is completely empty - a
4624  hysteresis to avoid pessimistic case of alternating creation and destruction
4625  of a VkDeviceMemory. */
4626  bool m_HasEmptyBlock;
4627  VmaDefragmentator* m_pDefragmentator;
4628  uint32_t m_NextBlockId;
4629 
4630  VkDeviceSize CalcMaxBlockSize() const;
4631 
4632  // Finds and removes given block from vector.
4633  void Remove(VmaDeviceMemoryBlock* pBlock);
4634 
4635  // Performs single step in sorting m_Blocks. They may not be fully sorted
4636  // after this call.
4637  void IncrementallySortBlocks();
4638 
4639  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
4640 };
4641 
4642 struct VmaPool_T
4643 {
4644  VMA_CLASS_NO_COPY(VmaPool_T)
4645 public:
4646  VmaBlockVector m_BlockVector;
4647 
4648  VmaPool_T(
4649  VmaAllocator hAllocator,
4650  const VmaPoolCreateInfo& createInfo);
4651  ~VmaPool_T();
4652 
4653  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
4654  uint32_t GetId() const { return m_Id; }
4655  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
4656 
4657 #if VMA_STATS_STRING_ENABLED
4658  //void PrintDetailedMap(class VmaStringBuilder& sb);
4659 #endif
4660 
4661 private:
4662  uint32_t m_Id;
4663 };
4664 
4665 class VmaDefragmentator
4666 {
4667  VMA_CLASS_NO_COPY(VmaDefragmentator)
4668 private:
4669  const VmaAllocator m_hAllocator;
4670  VmaBlockVector* const m_pBlockVector;
4671  uint32_t m_CurrentFrameIndex;
4672  VkDeviceSize m_BytesMoved;
4673  uint32_t m_AllocationsMoved;
4674 
4675  struct AllocationInfo
4676  {
4677  VmaAllocation m_hAllocation;
4678  VkBool32* m_pChanged;
4679 
4680  AllocationInfo() :
4681  m_hAllocation(VK_NULL_HANDLE),
4682  m_pChanged(VMA_NULL)
4683  {
4684  }
4685  };
4686 
4687  struct AllocationInfoSizeGreater
4688  {
4689  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
4690  {
4691  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
4692  }
4693  };
4694 
4695  // Used between AddAllocation and Defragment.
4696  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4697 
4698  struct BlockInfo
4699  {
4700  VmaDeviceMemoryBlock* m_pBlock;
4701  bool m_HasNonMovableAllocations;
4702  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4703 
4704  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
4705  m_pBlock(VMA_NULL),
4706  m_HasNonMovableAllocations(true),
4707  m_Allocations(pAllocationCallbacks),
4708  m_pMappedDataForDefragmentation(VMA_NULL)
4709  {
4710  }
4711 
4712  void CalcHasNonMovableAllocations()
4713  {
4714  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
4715  const size_t defragmentAllocCount = m_Allocations.size();
4716  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
4717  }
4718 
4719  void SortAllocationsBySizeDescecnding()
4720  {
4721  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
4722  }
4723 
4724  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
4725  void Unmap(VmaAllocator hAllocator);
4726 
4727  private:
4728  // Not null if mapped for defragmentation only, not originally mapped.
4729  void* m_pMappedDataForDefragmentation;
4730  };
4731 
4732  struct BlockPointerLess
4733  {
4734  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
4735  {
4736  return pLhsBlockInfo->m_pBlock < pRhsBlock;
4737  }
4738  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4739  {
4740  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
4741  }
4742  };
4743 
4744  // 1. Blocks with some non-movable allocations go first.
4745  // 2. Blocks with smaller sumFreeSize go first.
4746  struct BlockInfoCompareMoveDestination
4747  {
4748  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4749  {
4750  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
4751  {
4752  return true;
4753  }
4754  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
4755  {
4756  return false;
4757  }
4758  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
4759  {
4760  return true;
4761  }
4762  return false;
4763  }
4764  };
4765 
4766  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
4767  BlockInfoVector m_Blocks;
4768 
4769  VkResult DefragmentRound(
4770  VkDeviceSize maxBytesToMove,
4771  uint32_t maxAllocationsToMove);
4772 
4773  static bool MoveMakesSense(
4774  size_t dstBlockIndex, VkDeviceSize dstOffset,
4775  size_t srcBlockIndex, VkDeviceSize srcOffset);
4776 
4777 public:
4778  VmaDefragmentator(
4779  VmaAllocator hAllocator,
4780  VmaBlockVector* pBlockVector,
4781  uint32_t currentFrameIndex);
4782 
4783  ~VmaDefragmentator();
4784 
4785  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
4786  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
4787 
4788  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
4789 
4790  VkResult Defragment(
4791  VkDeviceSize maxBytesToMove,
4792  uint32_t maxAllocationsToMove);
4793 };
4794 
4795 #if VMA_RECORDING_ENABLED
4796 
4797 class VmaRecorder
4798 {
4799 public:
4800  VmaRecorder();
4801  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
4802  ~VmaRecorder();
4803 
4804  void RecordCreateAllocator(uint32_t frameIndex);
4805  void RecordDestroyAllocator(uint32_t frameIndex);
4806  void RecordCreatePool(uint32_t frameIndex,
4807  const VmaPoolCreateInfo& createInfo,
4808  VmaPool pool);
4809  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
4810  void RecordAllocateMemory(uint32_t frameIndex,
4811  const VkMemoryRequirements& vkMemReq,
4812  const VmaAllocationCreateInfo& createInfo,
4813  VmaAllocation allocation);
4814  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
4815  const VkMemoryRequirements& vkMemReq,
4816  bool requiresDedicatedAllocation,
4817  bool prefersDedicatedAllocation,
4818  const VmaAllocationCreateInfo& createInfo,
4819  VmaAllocation allocation);
4820  void RecordAllocateMemoryForImage(uint32_t frameIndex,
4821  const VkMemoryRequirements& vkMemReq,
4822  bool requiresDedicatedAllocation,
4823  bool prefersDedicatedAllocation,
4824  const VmaAllocationCreateInfo& createInfo,
4825  VmaAllocation allocation);
4826  void RecordFreeMemory(uint32_t frameIndex,
4827  VmaAllocation allocation);
4828  void RecordSetAllocationUserData(uint32_t frameIndex,
4829  VmaAllocation allocation,
4830  const void* pUserData);
4831  void RecordCreateLostAllocation(uint32_t frameIndex,
4832  VmaAllocation allocation);
4833  void RecordMapMemory(uint32_t frameIndex,
4834  VmaAllocation allocation);
4835  void RecordUnmapMemory(uint32_t frameIndex,
4836  VmaAllocation allocation);
4837  void RecordFlushAllocation(uint32_t frameIndex,
4838  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
4839  void RecordInvalidateAllocation(uint32_t frameIndex,
4840  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
4841  void RecordCreateBuffer(uint32_t frameIndex,
4842  const VkBufferCreateInfo& bufCreateInfo,
4843  const VmaAllocationCreateInfo& allocCreateInfo,
4844  VmaAllocation allocation);
4845  void RecordCreateImage(uint32_t frameIndex,
4846  const VkImageCreateInfo& imageCreateInfo,
4847  const VmaAllocationCreateInfo& allocCreateInfo,
4848  VmaAllocation allocation);
4849  void RecordDestroyBuffer(uint32_t frameIndex,
4850  VmaAllocation allocation);
4851  void RecordDestroyImage(uint32_t frameIndex,
4852  VmaAllocation allocation);
4853  void RecordTouchAllocation(uint32_t frameIndex,
4854  VmaAllocation allocation);
4855  void RecordGetAllocationInfo(uint32_t frameIndex,
4856  VmaAllocation allocation);
4857 
4858 private:
4859  struct CallParams
4860  {
4861  uint32_t threadId;
4862  double time;
4863  };
4864 
4865  class UserDataString
4866  {
4867  public:
4868  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
4869  const char* GetString() const { return m_Str; }
4870 
4871  private:
4872  char m_PtrStr[17];
4873  const char* m_Str;
4874  };
4875 
4876  bool m_UseMutex;
4877  VmaRecordFlags m_Flags;
4878  FILE* m_File;
4879  VMA_MUTEX m_FileMutex;
4880  int64_t m_Freq;
4881  int64_t m_StartCounter;
4882 
4883  void GetBasicParams(CallParams& outParams);
4884  void Flush();
4885 };
4886 
4887 #endif // #if VMA_RECORDING_ENABLED
4888 
4889 // Main allocator object.
4890 struct VmaAllocator_T
4891 {
4892  VMA_CLASS_NO_COPY(VmaAllocator_T)
4893 public:
4894  bool m_UseMutex;
4895  bool m_UseKhrDedicatedAllocation;
4896  VkDevice m_hDevice;
4897  bool m_AllocationCallbacksSpecified;
4898  VkAllocationCallbacks m_AllocationCallbacks;
4899  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
4900 
4901  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
4902  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
4903  VMA_MUTEX m_HeapSizeLimitMutex;
4904 
4905  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
4906  VkPhysicalDeviceMemoryProperties m_MemProps;
4907 
4908  // Default pools.
4909  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
4910 
4911  // Each vector is sorted by memory (handle value).
4912  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4913  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4914  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4915 
4916  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4917  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
4918  ~VmaAllocator_T();
4919 
4920  const VkAllocationCallbacks* GetAllocationCallbacks() const
4921  {
4922  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4923  }
4924  const VmaVulkanFunctions& GetVulkanFunctions() const
4925  {
4926  return m_VulkanFunctions;
4927  }
4928 
4929  VkDeviceSize GetBufferImageGranularity() const
4930  {
4931  return VMA_MAX(
4932  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4933  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4934  }
4935 
4936  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4937  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4938 
4939  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4940  {
4941  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4942  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4943  }
4944  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
4945  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
4946  {
4947  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
4948  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
4949  }
4950  // Minimum alignment for all allocations in specific memory type.
4951  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
4952  {
4953  return IsMemoryTypeNonCoherent(memTypeIndex) ?
4954  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
4955  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
4956  }
4957 
4958  bool IsIntegratedGpu() const
4959  {
4960  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
4961  }
4962 
4963 #if VMA_RECORDING_ENABLED
4964  VmaRecorder* GetRecorder() const { return m_pRecorder; }
4965 #endif
4966 
4967  void GetBufferMemoryRequirements(
4968  VkBuffer hBuffer,
4969  VkMemoryRequirements& memReq,
4970  bool& requiresDedicatedAllocation,
4971  bool& prefersDedicatedAllocation) const;
4972  void GetImageMemoryRequirements(
4973  VkImage hImage,
4974  VkMemoryRequirements& memReq,
4975  bool& requiresDedicatedAllocation,
4976  bool& prefersDedicatedAllocation) const;
4977 
4978  // Main allocation function.
4979  VkResult AllocateMemory(
4980  const VkMemoryRequirements& vkMemReq,
4981  bool requiresDedicatedAllocation,
4982  bool prefersDedicatedAllocation,
4983  VkBuffer dedicatedBuffer,
4984  VkImage dedicatedImage,
4985  const VmaAllocationCreateInfo& createInfo,
4986  VmaSuballocationType suballocType,
4987  VmaAllocation* pAllocation);
4988 
4989  // Main deallocation function.
4990  void FreeMemory(const VmaAllocation allocation);
4991 
4992  void CalculateStats(VmaStats* pStats);
4993 
4994 #if VMA_STATS_STRING_ENABLED
4995  void PrintDetailedMap(class VmaJsonWriter& json);
4996 #endif
4997 
4998  VkResult Defragment(
4999  VmaAllocation* pAllocations,
5000  size_t allocationCount,
5001  VkBool32* pAllocationsChanged,
5002  const VmaDefragmentationInfo* pDefragmentationInfo,
5003  VmaDefragmentationStats* pDefragmentationStats);
5004 
5005  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
5006  bool TouchAllocation(VmaAllocation hAllocation);
5007 
5008  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
5009  void DestroyPool(VmaPool pool);
5010  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
5011 
5012  void SetCurrentFrameIndex(uint32_t frameIndex);
5013  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
5014 
5015  void MakePoolAllocationsLost(
5016  VmaPool hPool,
5017  size_t* pLostAllocationCount);
5018  VkResult CheckPoolCorruption(VmaPool hPool);
5019  VkResult CheckCorruption(uint32_t memoryTypeBits);
5020 
5021  void CreateLostAllocation(VmaAllocation* pAllocation);
5022 
5023  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
5024  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
5025 
5026  VkResult Map(VmaAllocation hAllocation, void** ppData);
5027  void Unmap(VmaAllocation hAllocation);
5028 
5029  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
5030  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
5031 
5032  void FlushOrInvalidateAllocation(
5033  VmaAllocation hAllocation,
5034  VkDeviceSize offset, VkDeviceSize size,
5035  VMA_CACHE_OPERATION op);
5036 
5037  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
5038 
5039 private:
5040  VkDeviceSize m_PreferredLargeHeapBlockSize;
5041 
5042  VkPhysicalDevice m_PhysicalDevice;
5043  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
5044 
5045  VMA_MUTEX m_PoolsMutex;
5046  // Protected by m_PoolsMutex. Sorted by pointer value.
5047  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
5048  uint32_t m_NextPoolId;
5049 
5050  VmaVulkanFunctions m_VulkanFunctions;
5051 
5052 #if VMA_RECORDING_ENABLED
5053  VmaRecorder* m_pRecorder;
5054 #endif
5055 
5056  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
5057 
5058  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
5059 
5060  VkResult AllocateMemoryOfType(
5061  VkDeviceSize size,
5062  VkDeviceSize alignment,
5063  bool dedicatedAllocation,
5064  VkBuffer dedicatedBuffer,
5065  VkImage dedicatedImage,
5066  const VmaAllocationCreateInfo& createInfo,
5067  uint32_t memTypeIndex,
5068  VmaSuballocationType suballocType,
5069  VmaAllocation* pAllocation);
5070 
5071  // Allocates and registers new VkDeviceMemory specifically for single allocation.
5072  VkResult AllocateDedicatedMemory(
5073  VkDeviceSize size,
5074  VmaSuballocationType suballocType,
5075  uint32_t memTypeIndex,
5076  bool map,
5077  bool isUserDataString,
5078  void* pUserData,
5079  VkBuffer dedicatedBuffer,
5080  VkImage dedicatedImage,
5081  VmaAllocation* pAllocation);
5082 
5083  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
5084  void FreeDedicatedMemory(VmaAllocation allocation);
5085 };
5086 
5088 // Memory allocation #2 after VmaAllocator_T definition
5089 
5090 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
5091 {
5092  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
5093 }
5094 
5095 static void VmaFree(VmaAllocator hAllocator, void* ptr)
5096 {
5097  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
5098 }
5099 
5100 template<typename T>
5101 static T* VmaAllocate(VmaAllocator hAllocator)
5102 {
5103  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
5104 }
5105 
5106 template<typename T>
5107 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
5108 {
5109  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
5110 }
5111 
5112 template<typename T>
5113 static void vma_delete(VmaAllocator hAllocator, T* ptr)
5114 {
5115  if(ptr != VMA_NULL)
5116  {
5117  ptr->~T();
5118  VmaFree(hAllocator, ptr);
5119  }
5120 }
5121 
5122 template<typename T>
5123 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
5124 {
5125  if(ptr != VMA_NULL)
5126  {
5127  for(size_t i = count; i--; )
5128  ptr[i].~T();
5129  VmaFree(hAllocator, ptr);
5130  }
5131 }
5132 
5134 // VmaStringBuilder
5135 
5136 #if VMA_STATS_STRING_ENABLED
5137 
5138 class VmaStringBuilder
5139 {
5140 public:
5141  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
5142  size_t GetLength() const { return m_Data.size(); }
5143  const char* GetData() const { return m_Data.data(); }
5144 
5145  void Add(char ch) { m_Data.push_back(ch); }
5146  void Add(const char* pStr);
5147  void AddNewLine() { Add('\n'); }
5148  void AddNumber(uint32_t num);
5149  void AddNumber(uint64_t num);
5150  void AddPointer(const void* ptr);
5151 
5152 private:
5153  VmaVector< char, VmaStlAllocator<char> > m_Data;
5154 };
5155 
5156 void VmaStringBuilder::Add(const char* pStr)
5157 {
5158  const size_t strLen = strlen(pStr);
5159  if(strLen > 0)
5160  {
5161  const size_t oldCount = m_Data.size();
5162  m_Data.resize(oldCount + strLen);
5163  memcpy(m_Data.data() + oldCount, pStr, strLen);
5164  }
5165 }
5166 
5167 void VmaStringBuilder::AddNumber(uint32_t num)
5168 {
5169  char buf[11];
5170  VmaUint32ToStr(buf, sizeof(buf), num);
5171  Add(buf);
5172 }
5173 
5174 void VmaStringBuilder::AddNumber(uint64_t num)
5175 {
5176  char buf[21];
5177  VmaUint64ToStr(buf, sizeof(buf), num);
5178  Add(buf);
5179 }
5180 
5181 void VmaStringBuilder::AddPointer(const void* ptr)
5182 {
5183  char buf[21];
5184  VmaPtrToStr(buf, sizeof(buf), ptr);
5185  Add(buf);
5186 }
5187 
5188 #endif // #if VMA_STATS_STRING_ENABLED
5189 
5191 // VmaJsonWriter
5192 
5193 #if VMA_STATS_STRING_ENABLED
5194 
5195 class VmaJsonWriter
5196 {
5197  VMA_CLASS_NO_COPY(VmaJsonWriter)
5198 public:
5199  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
5200  ~VmaJsonWriter();
5201 
5202  void BeginObject(bool singleLine = false);
5203  void EndObject();
5204 
5205  void BeginArray(bool singleLine = false);
5206  void EndArray();
5207 
5208  void WriteString(const char* pStr);
5209  void BeginString(const char* pStr = VMA_NULL);
5210  void ContinueString(const char* pStr);
5211  void ContinueString(uint32_t n);
5212  void ContinueString(uint64_t n);
5213  void ContinueString_Pointer(const void* ptr);
5214  void EndString(const char* pStr = VMA_NULL);
5215 
5216  void WriteNumber(uint32_t n);
5217  void WriteNumber(uint64_t n);
5218  void WriteBool(bool b);
5219  void WriteNull();
5220 
5221 private:
5222  static const char* const INDENT;
5223 
5224  enum COLLECTION_TYPE
5225  {
5226  COLLECTION_TYPE_OBJECT,
5227  COLLECTION_TYPE_ARRAY,
5228  };
5229  struct StackItem
5230  {
5231  COLLECTION_TYPE type;
5232  uint32_t valueCount;
5233  bool singleLineMode;
5234  };
5235 
5236  VmaStringBuilder& m_SB;
5237  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
5238  bool m_InsideString;
5239 
5240  void BeginValue(bool isString);
5241  void WriteIndent(bool oneLess = false);
5242 };
5243 
5244 const char* const VmaJsonWriter::INDENT = " ";
5245 
5246 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
5247  m_SB(sb),
5248  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
5249  m_InsideString(false)
5250 {
5251 }
5252 
5253 VmaJsonWriter::~VmaJsonWriter()
5254 {
5255  VMA_ASSERT(!m_InsideString);
5256  VMA_ASSERT(m_Stack.empty());
5257 }
5258 
5259 void VmaJsonWriter::BeginObject(bool singleLine)
5260 {
5261  VMA_ASSERT(!m_InsideString);
5262 
5263  BeginValue(false);
5264  m_SB.Add('{');
5265 
5266  StackItem item;
5267  item.type = COLLECTION_TYPE_OBJECT;
5268  item.valueCount = 0;
5269  item.singleLineMode = singleLine;
5270  m_Stack.push_back(item);
5271 }
5272 
5273 void VmaJsonWriter::EndObject()
5274 {
5275  VMA_ASSERT(!m_InsideString);
5276 
5277  WriteIndent(true);
5278  m_SB.Add('}');
5279 
5280  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
5281  m_Stack.pop_back();
5282 }
5283 
5284 void VmaJsonWriter::BeginArray(bool singleLine)
5285 {
5286  VMA_ASSERT(!m_InsideString);
5287 
5288  BeginValue(false);
5289  m_SB.Add('[');
5290 
5291  StackItem item;
5292  item.type = COLLECTION_TYPE_ARRAY;
5293  item.valueCount = 0;
5294  item.singleLineMode = singleLine;
5295  m_Stack.push_back(item);
5296 }
5297 
5298 void VmaJsonWriter::EndArray()
5299 {
5300  VMA_ASSERT(!m_InsideString);
5301 
5302  WriteIndent(true);
5303  m_SB.Add(']');
5304 
5305  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
5306  m_Stack.pop_back();
5307 }
5308 
5309 void VmaJsonWriter::WriteString(const char* pStr)
5310 {
5311  BeginString(pStr);
5312  EndString();
5313 }
5314 
5315 void VmaJsonWriter::BeginString(const char* pStr)
5316 {
5317  VMA_ASSERT(!m_InsideString);
5318 
5319  BeginValue(true);
5320  m_SB.Add('"');
5321  m_InsideString = true;
5322  if(pStr != VMA_NULL && pStr[0] != '\0')
5323  {
5324  ContinueString(pStr);
5325  }
5326 }
5327 
5328 void VmaJsonWriter::ContinueString(const char* pStr)
5329 {
5330  VMA_ASSERT(m_InsideString);
5331 
5332  const size_t strLen = strlen(pStr);
5333  for(size_t i = 0; i < strLen; ++i)
5334  {
5335  char ch = pStr[i];
5336  if(ch == '\\')
5337  {
5338  m_SB.Add("\\\\");
5339  }
5340  else if(ch == '"')
5341  {
5342  m_SB.Add("\\\"");
5343  }
5344  else if(ch >= 32)
5345  {
5346  m_SB.Add(ch);
5347  }
5348  else switch(ch)
5349  {
5350  case '\b':
5351  m_SB.Add("\\b");
5352  break;
5353  case '\f':
5354  m_SB.Add("\\f");
5355  break;
5356  case '\n':
5357  m_SB.Add("\\n");
5358  break;
5359  case '\r':
5360  m_SB.Add("\\r");
5361  break;
5362  case '\t':
5363  m_SB.Add("\\t");
5364  break;
5365  default:
5366  VMA_ASSERT(0 && "Character not currently supported.");
5367  break;
5368  }
5369  }
5370 }
5371 
5372 void VmaJsonWriter::ContinueString(uint32_t n)
5373 {
5374  VMA_ASSERT(m_InsideString);
5375  m_SB.AddNumber(n);
5376 }
5377 
5378 void VmaJsonWriter::ContinueString(uint64_t n)
5379 {
5380  VMA_ASSERT(m_InsideString);
5381  m_SB.AddNumber(n);
5382 }
5383 
5384 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
5385 {
5386  VMA_ASSERT(m_InsideString);
5387  m_SB.AddPointer(ptr);
5388 }
5389 
5390 void VmaJsonWriter::EndString(const char* pStr)
5391 {
5392  VMA_ASSERT(m_InsideString);
5393  if(pStr != VMA_NULL && pStr[0] != '\0')
5394  {
5395  ContinueString(pStr);
5396  }
5397  m_SB.Add('"');
5398  m_InsideString = false;
5399 }
5400 
5401 void VmaJsonWriter::WriteNumber(uint32_t n)
5402 {
5403  VMA_ASSERT(!m_InsideString);
5404  BeginValue(false);
5405  m_SB.AddNumber(n);
5406 }
5407 
5408 void VmaJsonWriter::WriteNumber(uint64_t n)
5409 {
5410  VMA_ASSERT(!m_InsideString);
5411  BeginValue(false);
5412  m_SB.AddNumber(n);
5413 }
5414 
5415 void VmaJsonWriter::WriteBool(bool b)
5416 {
5417  VMA_ASSERT(!m_InsideString);
5418  BeginValue(false);
5419  m_SB.Add(b ? "true" : "false");
5420 }
5421 
5422 void VmaJsonWriter::WriteNull()
5423 {
5424  VMA_ASSERT(!m_InsideString);
5425  BeginValue(false);
5426  m_SB.Add("null");
5427 }
5428 
5429 void VmaJsonWriter::BeginValue(bool isString)
5430 {
5431  if(!m_Stack.empty())
5432  {
5433  StackItem& currItem = m_Stack.back();
5434  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5435  currItem.valueCount % 2 == 0)
5436  {
5437  VMA_ASSERT(isString);
5438  }
5439 
5440  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5441  currItem.valueCount % 2 != 0)
5442  {
5443  m_SB.Add(": ");
5444  }
5445  else if(currItem.valueCount > 0)
5446  {
5447  m_SB.Add(", ");
5448  WriteIndent();
5449  }
5450  else
5451  {
5452  WriteIndent();
5453  }
5454  ++currItem.valueCount;
5455  }
5456 }
5457 
5458 void VmaJsonWriter::WriteIndent(bool oneLess)
5459 {
5460  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
5461  {
5462  m_SB.AddNewLine();
5463 
5464  size_t count = m_Stack.size();
5465  if(count > 0 && oneLess)
5466  {
5467  --count;
5468  }
5469  for(size_t i = 0; i < count; ++i)
5470  {
5471  m_SB.Add(INDENT);
5472  }
5473  }
5474 }
5475 
5476 #endif // #if VMA_STATS_STRING_ENABLED
5477 
5479 
5480 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
5481 {
5482  if(IsUserDataString())
5483  {
5484  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
5485 
5486  FreeUserDataString(hAllocator);
5487 
5488  if(pUserData != VMA_NULL)
5489  {
5490  const char* const newStrSrc = (char*)pUserData;
5491  const size_t newStrLen = strlen(newStrSrc);
5492  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
5493  memcpy(newStrDst, newStrSrc, newStrLen + 1);
5494  m_pUserData = newStrDst;
5495  }
5496  }
5497  else
5498  {
5499  m_pUserData = pUserData;
5500  }
5501 }
5502 
5503 void VmaAllocation_T::ChangeBlockAllocation(
5504  VmaAllocator hAllocator,
5505  VmaDeviceMemoryBlock* block,
5506  VkDeviceSize offset)
5507 {
5508  VMA_ASSERT(block != VMA_NULL);
5509  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5510 
5511  // Move mapping reference counter from old block to new block.
5512  if(block != m_BlockAllocation.m_Block)
5513  {
5514  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
5515  if(IsPersistentMap())
5516  ++mapRefCount;
5517  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
5518  block->Map(hAllocator, mapRefCount, VMA_NULL);
5519  }
5520 
5521  m_BlockAllocation.m_Block = block;
5522  m_BlockAllocation.m_Offset = offset;
5523 }
5524 
5525 VkDeviceSize VmaAllocation_T::GetOffset() const
5526 {
5527  switch(m_Type)
5528  {
5529  case ALLOCATION_TYPE_BLOCK:
5530  return m_BlockAllocation.m_Offset;
5531  case ALLOCATION_TYPE_DEDICATED:
5532  return 0;
5533  default:
5534  VMA_ASSERT(0);
5535  return 0;
5536  }
5537 }
5538 
5539 VkDeviceMemory VmaAllocation_T::GetMemory() const
5540 {
5541  switch(m_Type)
5542  {
5543  case ALLOCATION_TYPE_BLOCK:
5544  return m_BlockAllocation.m_Block->GetDeviceMemory();
5545  case ALLOCATION_TYPE_DEDICATED:
5546  return m_DedicatedAllocation.m_hMemory;
5547  default:
5548  VMA_ASSERT(0);
5549  return VK_NULL_HANDLE;
5550  }
5551 }
5552 
5553 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
5554 {
5555  switch(m_Type)
5556  {
5557  case ALLOCATION_TYPE_BLOCK:
5558  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
5559  case ALLOCATION_TYPE_DEDICATED:
5560  return m_DedicatedAllocation.m_MemoryTypeIndex;
5561  default:
5562  VMA_ASSERT(0);
5563  return UINT32_MAX;
5564  }
5565 }
5566 
5567 void* VmaAllocation_T::GetMappedData() const
5568 {
5569  switch(m_Type)
5570  {
5571  case ALLOCATION_TYPE_BLOCK:
5572  if(m_MapCount != 0)
5573  {
5574  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
5575  VMA_ASSERT(pBlockData != VMA_NULL);
5576  return (char*)pBlockData + m_BlockAllocation.m_Offset;
5577  }
5578  else
5579  {
5580  return VMA_NULL;
5581  }
5582  break;
5583  case ALLOCATION_TYPE_DEDICATED:
5584  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
5585  return m_DedicatedAllocation.m_pMappedData;
5586  default:
5587  VMA_ASSERT(0);
5588  return VMA_NULL;
5589  }
5590 }
5591 
5592 bool VmaAllocation_T::CanBecomeLost() const
5593 {
5594  switch(m_Type)
5595  {
5596  case ALLOCATION_TYPE_BLOCK:
5597  return m_BlockAllocation.m_CanBecomeLost;
5598  case ALLOCATION_TYPE_DEDICATED:
5599  return false;
5600  default:
5601  VMA_ASSERT(0);
5602  return false;
5603  }
5604 }
5605 
5606 VmaPool VmaAllocation_T::GetPool() const
5607 {
5608  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5609  return m_BlockAllocation.m_hPool;
5610 }
5611 
5612 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5613 {
5614  VMA_ASSERT(CanBecomeLost());
5615 
5616  /*
5617  Warning: This is a carefully designed algorithm.
5618  Do not modify unless you really know what you're doing :)
5619  */
5620  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
5621  for(;;)
5622  {
5623  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
5624  {
5625  VMA_ASSERT(0);
5626  return false;
5627  }
5628  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
5629  {
5630  return false;
5631  }
5632  else // Last use time earlier than current time.
5633  {
5634  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
5635  {
5636  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
5637  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
5638  return true;
5639  }
5640  }
5641  }
5642 }
5643 
5644 #if VMA_STATS_STRING_ENABLED
5645 
5646 // Correspond to values of enum VmaSuballocationType.
5647 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
5648  "FREE",
5649  "UNKNOWN",
5650  "BUFFER",
5651  "IMAGE_UNKNOWN",
5652  "IMAGE_LINEAR",
5653  "IMAGE_OPTIMAL",
5654 };
5655 
5656 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
5657 {
5658  json.WriteString("Type");
5659  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
5660 
5661  json.WriteString("Size");
5662  json.WriteNumber(m_Size);
5663 
5664  if(m_pUserData != VMA_NULL)
5665  {
5666  json.WriteString("UserData");
5667  if(IsUserDataString())
5668  {
5669  json.WriteString((const char*)m_pUserData);
5670  }
5671  else
5672  {
5673  json.BeginString();
5674  json.ContinueString_Pointer(m_pUserData);
5675  json.EndString();
5676  }
5677  }
5678 
5679  json.WriteString("CreationFrameIndex");
5680  json.WriteNumber(m_CreationFrameIndex);
5681 
5682  json.WriteString("LastUseFrameIndex");
5683  json.WriteNumber(GetLastUseFrameIndex());
5684 
5685  if(m_BufferImageUsage != 0)
5686  {
5687  json.WriteString("Usage");
5688  json.WriteNumber(m_BufferImageUsage);
5689  }
5690 }
5691 
5692 #endif
5693 
5694 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
5695 {
5696  VMA_ASSERT(IsUserDataString());
5697  if(m_pUserData != VMA_NULL)
5698  {
5699  char* const oldStr = (char*)m_pUserData;
5700  const size_t oldStrLen = strlen(oldStr);
5701  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
5702  m_pUserData = VMA_NULL;
5703  }
5704 }
5705 
5706 void VmaAllocation_T::BlockAllocMap()
5707 {
5708  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5709 
5710  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5711  {
5712  ++m_MapCount;
5713  }
5714  else
5715  {
5716  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
5717  }
5718 }
5719 
5720 void VmaAllocation_T::BlockAllocUnmap()
5721 {
5722  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5723 
5724  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5725  {
5726  --m_MapCount;
5727  }
5728  else
5729  {
5730  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
5731  }
5732 }
5733 
5734 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
5735 {
5736  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5737 
5738  if(m_MapCount != 0)
5739  {
5740  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5741  {
5742  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
5743  *ppData = m_DedicatedAllocation.m_pMappedData;
5744  ++m_MapCount;
5745  return VK_SUCCESS;
5746  }
5747  else
5748  {
5749  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
5750  return VK_ERROR_MEMORY_MAP_FAILED;
5751  }
5752  }
5753  else
5754  {
5755  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5756  hAllocator->m_hDevice,
5757  m_DedicatedAllocation.m_hMemory,
5758  0, // offset
5759  VK_WHOLE_SIZE,
5760  0, // flags
5761  ppData);
5762  if(result == VK_SUCCESS)
5763  {
5764  m_DedicatedAllocation.m_pMappedData = *ppData;
5765  m_MapCount = 1;
5766  }
5767  return result;
5768  }
5769 }
5770 
5771 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
5772 {
5773  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5774 
5775  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5776  {
5777  --m_MapCount;
5778  if(m_MapCount == 0)
5779  {
5780  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
5781  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
5782  hAllocator->m_hDevice,
5783  m_DedicatedAllocation.m_hMemory);
5784  }
5785  }
5786  else
5787  {
5788  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
5789  }
5790 }
5791 
5792 #if VMA_STATS_STRING_ENABLED
5793 
5794 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
5795 {
5796  json.BeginObject();
5797 
5798  json.WriteString("Blocks");
5799  json.WriteNumber(stat.blockCount);
5800 
5801  json.WriteString("Allocations");
5802  json.WriteNumber(stat.allocationCount);
5803 
5804  json.WriteString("UnusedRanges");
5805  json.WriteNumber(stat.unusedRangeCount);
5806 
5807  json.WriteString("UsedBytes");
5808  json.WriteNumber(stat.usedBytes);
5809 
5810  json.WriteString("UnusedBytes");
5811  json.WriteNumber(stat.unusedBytes);
5812 
5813  if(stat.allocationCount > 1)
5814  {
5815  json.WriteString("AllocationSize");
5816  json.BeginObject(true);
5817  json.WriteString("Min");
5818  json.WriteNumber(stat.allocationSizeMin);
5819  json.WriteString("Avg");
5820  json.WriteNumber(stat.allocationSizeAvg);
5821  json.WriteString("Max");
5822  json.WriteNumber(stat.allocationSizeMax);
5823  json.EndObject();
5824  }
5825 
5826  if(stat.unusedRangeCount > 1)
5827  {
5828  json.WriteString("UnusedRangeSize");
5829  json.BeginObject(true);
5830  json.WriteString("Min");
5831  json.WriteNumber(stat.unusedRangeSizeMin);
5832  json.WriteString("Avg");
5833  json.WriteNumber(stat.unusedRangeSizeAvg);
5834  json.WriteString("Max");
5835  json.WriteNumber(stat.unusedRangeSizeMax);
5836  json.EndObject();
5837  }
5838 
5839  json.EndObject();
5840 }
5841 
5842 #endif // #if VMA_STATS_STRING_ENABLED
5843 
5844 struct VmaSuballocationItemSizeLess
5845 {
5846  bool operator()(
5847  const VmaSuballocationList::iterator lhs,
5848  const VmaSuballocationList::iterator rhs) const
5849  {
5850  return lhs->size < rhs->size;
5851  }
5852  bool operator()(
5853  const VmaSuballocationList::iterator lhs,
5854  VkDeviceSize rhsSize) const
5855  {
5856  return lhs->size < rhsSize;
5857  }
5858 };
5859 
5861 // class VmaBlockMetadata
5862 
5863 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
5864  m_Size(0),
5865  m_FreeCount(0),
5866  m_SumFreeSize(0),
5867  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
5868  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
5869 {
5870 }
5871 
5872 VmaBlockMetadata::~VmaBlockMetadata()
5873 {
5874 }
5875 
5876 void VmaBlockMetadata::Init(VkDeviceSize size)
5877 {
5878  m_Size = size;
5879  m_FreeCount = 1;
5880  m_SumFreeSize = size;
5881 
5882  VmaSuballocation suballoc = {};
5883  suballoc.offset = 0;
5884  suballoc.size = size;
5885  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5886  suballoc.hAllocation = VK_NULL_HANDLE;
5887 
5888  m_Suballocations.push_back(suballoc);
5889  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
5890  --suballocItem;
5891  m_FreeSuballocationsBySize.push_back(suballocItem);
5892 }
5893 
5894 bool VmaBlockMetadata::Validate() const
5895 {
5896  if(m_Suballocations.empty())
5897  {
5898  return false;
5899  }
5900 
5901  // Expected offset of new suballocation as calculates from previous ones.
5902  VkDeviceSize calculatedOffset = 0;
5903  // Expected number of free suballocations as calculated from traversing their list.
5904  uint32_t calculatedFreeCount = 0;
5905  // Expected sum size of free suballocations as calculated from traversing their list.
5906  VkDeviceSize calculatedSumFreeSize = 0;
5907  // Expected number of free suballocations that should be registered in
5908  // m_FreeSuballocationsBySize calculated from traversing their list.
5909  size_t freeSuballocationsToRegister = 0;
5910  // True if previous visited suballocation was free.
5911  bool prevFree = false;
5912 
5913  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5914  suballocItem != m_Suballocations.cend();
5915  ++suballocItem)
5916  {
5917  const VmaSuballocation& subAlloc = *suballocItem;
5918 
5919  // Actual offset of this suballocation doesn't match expected one.
5920  if(subAlloc.offset != calculatedOffset)
5921  {
5922  return false;
5923  }
5924 
5925  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
5926  // Two adjacent free suballocations are invalid. They should be merged.
5927  if(prevFree && currFree)
5928  {
5929  return false;
5930  }
5931 
5932  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
5933  {
5934  return false;
5935  }
5936 
5937  if(currFree)
5938  {
5939  calculatedSumFreeSize += subAlloc.size;
5940  ++calculatedFreeCount;
5941  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5942  {
5943  ++freeSuballocationsToRegister;
5944  }
5945 
5946  // Margin required between allocations - every free space must be at least that large.
5947  if(subAlloc.size < VMA_DEBUG_MARGIN)
5948  {
5949  return false;
5950  }
5951  }
5952  else
5953  {
5954  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
5955  {
5956  return false;
5957  }
5958  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
5959  {
5960  return false;
5961  }
5962 
5963  // Margin required between allocations - previous allocation must be free.
5964  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
5965  {
5966  return false;
5967  }
5968  }
5969 
5970  calculatedOffset += subAlloc.size;
5971  prevFree = currFree;
5972  }
5973 
5974  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
5975  // match expected one.
5976  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
5977  {
5978  return false;
5979  }
5980 
5981  VkDeviceSize lastSize = 0;
5982  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
5983  {
5984  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
5985 
5986  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
5987  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5988  {
5989  return false;
5990  }
5991  // They must be sorted by size ascending.
5992  if(suballocItem->size < lastSize)
5993  {
5994  return false;
5995  }
5996 
5997  lastSize = suballocItem->size;
5998  }
5999 
6000  // Check if totals match calculacted values.
6001  if(!ValidateFreeSuballocationList() ||
6002  (calculatedOffset != m_Size) ||
6003  (calculatedSumFreeSize != m_SumFreeSize) ||
6004  (calculatedFreeCount != m_FreeCount))
6005  {
6006  return false;
6007  }
6008 
6009  return true;
6010 }
6011 
6012 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
6013 {
6014  if(!m_FreeSuballocationsBySize.empty())
6015  {
6016  return m_FreeSuballocationsBySize.back()->size;
6017  }
6018  else
6019  {
6020  return 0;
6021  }
6022 }
6023 
6024 bool VmaBlockMetadata::IsEmpty() const
6025 {
6026  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
6027 }
6028 
6029 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
6030 {
6031  outInfo.blockCount = 1;
6032 
6033  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6034  outInfo.allocationCount = rangeCount - m_FreeCount;
6035  outInfo.unusedRangeCount = m_FreeCount;
6036 
6037  outInfo.unusedBytes = m_SumFreeSize;
6038  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
6039 
6040  outInfo.allocationSizeMin = UINT64_MAX;
6041  outInfo.allocationSizeMax = 0;
6042  outInfo.unusedRangeSizeMin = UINT64_MAX;
6043  outInfo.unusedRangeSizeMax = 0;
6044 
6045  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6046  suballocItem != m_Suballocations.cend();
6047  ++suballocItem)
6048  {
6049  const VmaSuballocation& suballoc = *suballocItem;
6050  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
6051  {
6052  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
6053  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
6054  }
6055  else
6056  {
6057  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
6058  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
6059  }
6060  }
6061 }
6062 
6063 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
6064 {
6065  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6066 
6067  inoutStats.size += m_Size;
6068  inoutStats.unusedSize += m_SumFreeSize;
6069  inoutStats.allocationCount += rangeCount - m_FreeCount;
6070  inoutStats.unusedRangeCount += m_FreeCount;
6071  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
6072 }
6073 
6074 #if VMA_STATS_STRING_ENABLED
6075 
6076 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
6077 {
6078  json.BeginObject();
6079 
6080  json.WriteString("TotalBytes");
6081  json.WriteNumber(m_Size);
6082 
6083  json.WriteString("UnusedBytes");
6084  json.WriteNumber(m_SumFreeSize);
6085 
6086  json.WriteString("Allocations");
6087  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
6088 
6089  json.WriteString("UnusedRanges");
6090  json.WriteNumber(m_FreeCount);
6091 
6092  json.WriteString("Suballocations");
6093  json.BeginArray();
6094  size_t i = 0;
6095  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6096  suballocItem != m_Suballocations.cend();
6097  ++suballocItem, ++i)
6098  {
6099  json.BeginObject(true);
6100 
6101  json.WriteString("Offset");
6102  json.WriteNumber(suballocItem->offset);
6103 
6104  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6105  {
6106  json.WriteString("Type");
6107  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
6108 
6109  json.WriteString("Size");
6110  json.WriteNumber(suballocItem->size);
6111  }
6112  else
6113  {
6114  suballocItem->hAllocation->PrintParameters(json);
6115  }
6116 
6117  json.EndObject();
6118  }
6119  json.EndArray();
6120 
6121  json.EndObject();
6122 }
6123 
6124 #endif // #if VMA_STATS_STRING_ENABLED
6125 
6126 /*
6127 How many suitable free suballocations to analyze before choosing best one.
6128 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
6129  be chosen.
6130 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
6131  suballocations will be analized and best one will be chosen.
6132 - Any other value is also acceptable.
6133 */
6134 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
6135 
6136 bool VmaBlockMetadata::CreateAllocationRequest(
6137  uint32_t currentFrameIndex,
6138  uint32_t frameInUseCount,
6139  VkDeviceSize bufferImageGranularity,
6140  VkDeviceSize allocSize,
6141  VkDeviceSize allocAlignment,
6142  VmaSuballocationType allocType,
6143  bool canMakeOtherLost,
6144  VmaAllocationRequest* pAllocationRequest)
6145 {
6146  VMA_ASSERT(allocSize > 0);
6147  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6148  VMA_ASSERT(pAllocationRequest != VMA_NULL);
6149  VMA_HEAVY_ASSERT(Validate());
6150 
6151  // There is not enough total free space in this block to fullfill the request: Early return.
6152  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
6153  {
6154  return false;
6155  }
6156 
6157  // New algorithm, efficiently searching freeSuballocationsBySize.
6158  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
6159  if(freeSuballocCount > 0)
6160  {
6161  if(VMA_BEST_FIT)
6162  {
6163  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
6164  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6165  m_FreeSuballocationsBySize.data(),
6166  m_FreeSuballocationsBySize.data() + freeSuballocCount,
6167  allocSize + 2 * VMA_DEBUG_MARGIN,
6168  VmaSuballocationItemSizeLess());
6169  size_t index = it - m_FreeSuballocationsBySize.data();
6170  for(; index < freeSuballocCount; ++index)
6171  {
6172  if(CheckAllocation(
6173  currentFrameIndex,
6174  frameInUseCount,
6175  bufferImageGranularity,
6176  allocSize,
6177  allocAlignment,
6178  allocType,
6179  m_FreeSuballocationsBySize[index],
6180  false, // canMakeOtherLost
6181  &pAllocationRequest->offset,
6182  &pAllocationRequest->itemsToMakeLostCount,
6183  &pAllocationRequest->sumFreeSize,
6184  &pAllocationRequest->sumItemSize))
6185  {
6186  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6187  return true;
6188  }
6189  }
6190  }
6191  else
6192  {
6193  // Search staring from biggest suballocations.
6194  for(size_t index = freeSuballocCount; index--; )
6195  {
6196  if(CheckAllocation(
6197  currentFrameIndex,
6198  frameInUseCount,
6199  bufferImageGranularity,
6200  allocSize,
6201  allocAlignment,
6202  allocType,
6203  m_FreeSuballocationsBySize[index],
6204  false, // canMakeOtherLost
6205  &pAllocationRequest->offset,
6206  &pAllocationRequest->itemsToMakeLostCount,
6207  &pAllocationRequest->sumFreeSize,
6208  &pAllocationRequest->sumItemSize))
6209  {
6210  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6211  return true;
6212  }
6213  }
6214  }
6215  }
6216 
6217  if(canMakeOtherLost)
6218  {
6219  // Brute-force algorithm. TODO: Come up with something better.
6220 
6221  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
6222  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
6223 
6224  VmaAllocationRequest tmpAllocRequest = {};
6225  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
6226  suballocIt != m_Suballocations.end();
6227  ++suballocIt)
6228  {
6229  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
6230  suballocIt->hAllocation->CanBecomeLost())
6231  {
6232  if(CheckAllocation(
6233  currentFrameIndex,
6234  frameInUseCount,
6235  bufferImageGranularity,
6236  allocSize,
6237  allocAlignment,
6238  allocType,
6239  suballocIt,
6240  canMakeOtherLost,
6241  &tmpAllocRequest.offset,
6242  &tmpAllocRequest.itemsToMakeLostCount,
6243  &tmpAllocRequest.sumFreeSize,
6244  &tmpAllocRequest.sumItemSize))
6245  {
6246  tmpAllocRequest.item = suballocIt;
6247 
6248  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
6249  {
6250  *pAllocationRequest = tmpAllocRequest;
6251  }
6252  }
6253  }
6254  }
6255 
6256  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
6257  {
6258  return true;
6259  }
6260  }
6261 
6262  return false;
6263 }
6264 
6265 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
6266  uint32_t currentFrameIndex,
6267  uint32_t frameInUseCount,
6268  VmaAllocationRequest* pAllocationRequest)
6269 {
6270  while(pAllocationRequest->itemsToMakeLostCount > 0)
6271  {
6272  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
6273  {
6274  ++pAllocationRequest->item;
6275  }
6276  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6277  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
6278  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
6279  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6280  {
6281  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
6282  --pAllocationRequest->itemsToMakeLostCount;
6283  }
6284  else
6285  {
6286  return false;
6287  }
6288  }
6289 
6290  VMA_HEAVY_ASSERT(Validate());
6291  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6292  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
6293 
6294  return true;
6295 }
6296 
6297 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
6298 {
6299  uint32_t lostAllocationCount = 0;
6300  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6301  it != m_Suballocations.end();
6302  ++it)
6303  {
6304  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
6305  it->hAllocation->CanBecomeLost() &&
6306  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6307  {
6308  it = FreeSuballocation(it);
6309  ++lostAllocationCount;
6310  }
6311  }
6312  return lostAllocationCount;
6313 }
6314 
6315 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
6316 {
6317  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6318  it != m_Suballocations.end();
6319  ++it)
6320  {
6321  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6322  {
6323  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
6324  {
6325  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
6326  return VK_ERROR_VALIDATION_FAILED_EXT;
6327  }
6328  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
6329  {
6330  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
6331  return VK_ERROR_VALIDATION_FAILED_EXT;
6332  }
6333  }
6334  }
6335 
6336  return VK_SUCCESS;
6337 }
6338 
6339 void VmaBlockMetadata::Alloc(
6340  const VmaAllocationRequest& request,
6341  VmaSuballocationType type,
6342  VkDeviceSize allocSize,
6343  VmaAllocation hAllocation)
6344 {
6345  VMA_ASSERT(request.item != m_Suballocations.end());
6346  VmaSuballocation& suballoc = *request.item;
6347  // Given suballocation is a free block.
6348  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6349  // Given offset is inside this suballocation.
6350  VMA_ASSERT(request.offset >= suballoc.offset);
6351  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
6352  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
6353  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
6354 
6355  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
6356  // it to become used.
6357  UnregisterFreeSuballocation(request.item);
6358 
6359  suballoc.offset = request.offset;
6360  suballoc.size = allocSize;
6361  suballoc.type = type;
6362  suballoc.hAllocation = hAllocation;
6363 
6364  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
6365  if(paddingEnd)
6366  {
6367  VmaSuballocation paddingSuballoc = {};
6368  paddingSuballoc.offset = request.offset + allocSize;
6369  paddingSuballoc.size = paddingEnd;
6370  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6371  VmaSuballocationList::iterator next = request.item;
6372  ++next;
6373  const VmaSuballocationList::iterator paddingEndItem =
6374  m_Suballocations.insert(next, paddingSuballoc);
6375  RegisterFreeSuballocation(paddingEndItem);
6376  }
6377 
6378  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
6379  if(paddingBegin)
6380  {
6381  VmaSuballocation paddingSuballoc = {};
6382  paddingSuballoc.offset = request.offset - paddingBegin;
6383  paddingSuballoc.size = paddingBegin;
6384  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6385  const VmaSuballocationList::iterator paddingBeginItem =
6386  m_Suballocations.insert(request.item, paddingSuballoc);
6387  RegisterFreeSuballocation(paddingBeginItem);
6388  }
6389 
6390  // Update totals.
6391  m_FreeCount = m_FreeCount - 1;
6392  if(paddingBegin > 0)
6393  {
6394  ++m_FreeCount;
6395  }
6396  if(paddingEnd > 0)
6397  {
6398  ++m_FreeCount;
6399  }
6400  m_SumFreeSize -= allocSize;
6401 }
6402 
6403 void VmaBlockMetadata::Free(const VmaAllocation allocation)
6404 {
6405  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6406  suballocItem != m_Suballocations.end();
6407  ++suballocItem)
6408  {
6409  VmaSuballocation& suballoc = *suballocItem;
6410  if(suballoc.hAllocation == allocation)
6411  {
6412  FreeSuballocation(suballocItem);
6413  VMA_HEAVY_ASSERT(Validate());
6414  return;
6415  }
6416  }
6417  VMA_ASSERT(0 && "Not found!");
6418 }
6419 
6420 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
6421 {
6422  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6423  suballocItem != m_Suballocations.end();
6424  ++suballocItem)
6425  {
6426  VmaSuballocation& suballoc = *suballocItem;
6427  if(suballoc.offset == offset)
6428  {
6429  FreeSuballocation(suballocItem);
6430  return;
6431  }
6432  }
6433  VMA_ASSERT(0 && "Not found!");
6434 }
6435 
6436 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
6437 {
6438  VkDeviceSize lastSize = 0;
6439  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
6440  {
6441  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
6442 
6443  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6444  {
6445  VMA_ASSERT(0);
6446  return false;
6447  }
6448  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6449  {
6450  VMA_ASSERT(0);
6451  return false;
6452  }
6453  if(it->size < lastSize)
6454  {
6455  VMA_ASSERT(0);
6456  return false;
6457  }
6458 
6459  lastSize = it->size;
6460  }
6461  return true;
6462 }
6463 
6464 bool VmaBlockMetadata::CheckAllocation(
6465  uint32_t currentFrameIndex,
6466  uint32_t frameInUseCount,
6467  VkDeviceSize bufferImageGranularity,
6468  VkDeviceSize allocSize,
6469  VkDeviceSize allocAlignment,
6470  VmaSuballocationType allocType,
6471  VmaSuballocationList::const_iterator suballocItem,
6472  bool canMakeOtherLost,
6473  VkDeviceSize* pOffset,
6474  size_t* itemsToMakeLostCount,
6475  VkDeviceSize* pSumFreeSize,
6476  VkDeviceSize* pSumItemSize) const
6477 {
6478  VMA_ASSERT(allocSize > 0);
6479  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6480  VMA_ASSERT(suballocItem != m_Suballocations.cend());
6481  VMA_ASSERT(pOffset != VMA_NULL);
6482 
6483  *itemsToMakeLostCount = 0;
6484  *pSumFreeSize = 0;
6485  *pSumItemSize = 0;
6486 
6487  if(canMakeOtherLost)
6488  {
6489  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6490  {
6491  *pSumFreeSize = suballocItem->size;
6492  }
6493  else
6494  {
6495  if(suballocItem->hAllocation->CanBecomeLost() &&
6496  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6497  {
6498  ++*itemsToMakeLostCount;
6499  *pSumItemSize = suballocItem->size;
6500  }
6501  else
6502  {
6503  return false;
6504  }
6505  }
6506 
6507  // Remaining size is too small for this request: Early return.
6508  if(m_Size - suballocItem->offset < allocSize)
6509  {
6510  return false;
6511  }
6512 
6513  // Start from offset equal to beginning of this suballocation.
6514  *pOffset = suballocItem->offset;
6515 
6516  // Apply VMA_DEBUG_MARGIN at the beginning.
6517  if(VMA_DEBUG_MARGIN > 0)
6518  {
6519  *pOffset += VMA_DEBUG_MARGIN;
6520  }
6521 
6522  // Apply alignment.
6523  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6524 
6525  // Check previous suballocations for BufferImageGranularity conflicts.
6526  // Make bigger alignment if necessary.
6527  if(bufferImageGranularity > 1)
6528  {
6529  bool bufferImageGranularityConflict = false;
6530  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6531  while(prevSuballocItem != m_Suballocations.cbegin())
6532  {
6533  --prevSuballocItem;
6534  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6535  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6536  {
6537  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6538  {
6539  bufferImageGranularityConflict = true;
6540  break;
6541  }
6542  }
6543  else
6544  // Already on previous page.
6545  break;
6546  }
6547  if(bufferImageGranularityConflict)
6548  {
6549  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6550  }
6551  }
6552 
6553  // Now that we have final *pOffset, check if we are past suballocItem.
6554  // If yes, return false - this function should be called for another suballocItem as starting point.
6555  if(*pOffset >= suballocItem->offset + suballocItem->size)
6556  {
6557  return false;
6558  }
6559 
6560  // Calculate padding at the beginning based on current offset.
6561  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
6562 
6563  // Calculate required margin at the end.
6564  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6565 
6566  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
6567  // Another early return check.
6568  if(suballocItem->offset + totalSize > m_Size)
6569  {
6570  return false;
6571  }
6572 
6573  // Advance lastSuballocItem until desired size is reached.
6574  // Update itemsToMakeLostCount.
6575  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
6576  if(totalSize > suballocItem->size)
6577  {
6578  VkDeviceSize remainingSize = totalSize - suballocItem->size;
6579  while(remainingSize > 0)
6580  {
6581  ++lastSuballocItem;
6582  if(lastSuballocItem == m_Suballocations.cend())
6583  {
6584  return false;
6585  }
6586  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6587  {
6588  *pSumFreeSize += lastSuballocItem->size;
6589  }
6590  else
6591  {
6592  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
6593  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
6594  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6595  {
6596  ++*itemsToMakeLostCount;
6597  *pSumItemSize += lastSuballocItem->size;
6598  }
6599  else
6600  {
6601  return false;
6602  }
6603  }
6604  remainingSize = (lastSuballocItem->size < remainingSize) ?
6605  remainingSize - lastSuballocItem->size : 0;
6606  }
6607  }
6608 
6609  // Check next suballocations for BufferImageGranularity conflicts.
6610  // If conflict exists, we must mark more allocations lost or fail.
6611  if(bufferImageGranularity > 1)
6612  {
6613  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
6614  ++nextSuballocItem;
6615  while(nextSuballocItem != m_Suballocations.cend())
6616  {
6617  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6618  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6619  {
6620  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6621  {
6622  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
6623  if(nextSuballoc.hAllocation->CanBecomeLost() &&
6624  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6625  {
6626  ++*itemsToMakeLostCount;
6627  }
6628  else
6629  {
6630  return false;
6631  }
6632  }
6633  }
6634  else
6635  {
6636  // Already on next page.
6637  break;
6638  }
6639  ++nextSuballocItem;
6640  }
6641  }
6642  }
6643  else
6644  {
6645  const VmaSuballocation& suballoc = *suballocItem;
6646  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6647 
6648  *pSumFreeSize = suballoc.size;
6649 
6650  // Size of this suballocation is too small for this request: Early return.
6651  if(suballoc.size < allocSize)
6652  {
6653  return false;
6654  }
6655 
6656  // Start from offset equal to beginning of this suballocation.
6657  *pOffset = suballoc.offset;
6658 
6659  // Apply VMA_DEBUG_MARGIN at the beginning.
6660  if(VMA_DEBUG_MARGIN > 0)
6661  {
6662  *pOffset += VMA_DEBUG_MARGIN;
6663  }
6664 
6665  // Apply alignment.
6666  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6667 
6668  // Check previous suballocations for BufferImageGranularity conflicts.
6669  // Make bigger alignment if necessary.
6670  if(bufferImageGranularity > 1)
6671  {
6672  bool bufferImageGranularityConflict = false;
6673  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6674  while(prevSuballocItem != m_Suballocations.cbegin())
6675  {
6676  --prevSuballocItem;
6677  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6678  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6679  {
6680  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6681  {
6682  bufferImageGranularityConflict = true;
6683  break;
6684  }
6685  }
6686  else
6687  // Already on previous page.
6688  break;
6689  }
6690  if(bufferImageGranularityConflict)
6691  {
6692  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6693  }
6694  }
6695 
6696  // Calculate padding at the beginning based on current offset.
6697  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
6698 
6699  // Calculate required margin at the end.
6700  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6701 
6702  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
6703  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
6704  {
6705  return false;
6706  }
6707 
6708  // Check next suballocations for BufferImageGranularity conflicts.
6709  // If conflict exists, allocation cannot be made here.
6710  if(bufferImageGranularity > 1)
6711  {
6712  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
6713  ++nextSuballocItem;
6714  while(nextSuballocItem != m_Suballocations.cend())
6715  {
6716  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6717  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6718  {
6719  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6720  {
6721  return false;
6722  }
6723  }
6724  else
6725  {
6726  // Already on next page.
6727  break;
6728  }
6729  ++nextSuballocItem;
6730  }
6731  }
6732  }
6733 
6734  // All tests passed: Success. pOffset is already filled.
6735  return true;
6736 }
6737 
6738 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
6739 {
6740  VMA_ASSERT(item != m_Suballocations.end());
6741  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6742 
6743  VmaSuballocationList::iterator nextItem = item;
6744  ++nextItem;
6745  VMA_ASSERT(nextItem != m_Suballocations.end());
6746  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
6747 
6748  item->size += nextItem->size;
6749  --m_FreeCount;
6750  m_Suballocations.erase(nextItem);
6751 }
6752 
6753 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
6754 {
6755  // Change this suballocation to be marked as free.
6756  VmaSuballocation& suballoc = *suballocItem;
6757  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6758  suballoc.hAllocation = VK_NULL_HANDLE;
6759 
6760  // Update totals.
6761  ++m_FreeCount;
6762  m_SumFreeSize += suballoc.size;
6763 
6764  // Merge with previous and/or next suballocation if it's also free.
6765  bool mergeWithNext = false;
6766  bool mergeWithPrev = false;
6767 
6768  VmaSuballocationList::iterator nextItem = suballocItem;
6769  ++nextItem;
6770  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
6771  {
6772  mergeWithNext = true;
6773  }
6774 
6775  VmaSuballocationList::iterator prevItem = suballocItem;
6776  if(suballocItem != m_Suballocations.begin())
6777  {
6778  --prevItem;
6779  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6780  {
6781  mergeWithPrev = true;
6782  }
6783  }
6784 
6785  if(mergeWithNext)
6786  {
6787  UnregisterFreeSuballocation(nextItem);
6788  MergeFreeWithNext(suballocItem);
6789  }
6790 
6791  if(mergeWithPrev)
6792  {
6793  UnregisterFreeSuballocation(prevItem);
6794  MergeFreeWithNext(prevItem);
6795  RegisterFreeSuballocation(prevItem);
6796  return prevItem;
6797  }
6798  else
6799  {
6800  RegisterFreeSuballocation(suballocItem);
6801  return suballocItem;
6802  }
6803 }
6804 
6805 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
6806 {
6807  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6808  VMA_ASSERT(item->size > 0);
6809 
6810  // You may want to enable this validation at the beginning or at the end of
6811  // this function, depending on what do you want to check.
6812  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6813 
6814  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6815  {
6816  if(m_FreeSuballocationsBySize.empty())
6817  {
6818  m_FreeSuballocationsBySize.push_back(item);
6819  }
6820  else
6821  {
6822  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
6823  }
6824  }
6825 
6826  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6827 }
6828 
6829 
6830 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
6831 {
6832  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6833  VMA_ASSERT(item->size > 0);
6834 
6835  // You may want to enable this validation at the beginning or at the end of
6836  // this function, depending on what do you want to check.
6837  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6838 
6839  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6840  {
6841  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6842  m_FreeSuballocationsBySize.data(),
6843  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
6844  item,
6845  VmaSuballocationItemSizeLess());
6846  for(size_t index = it - m_FreeSuballocationsBySize.data();
6847  index < m_FreeSuballocationsBySize.size();
6848  ++index)
6849  {
6850  if(m_FreeSuballocationsBySize[index] == item)
6851  {
6852  VmaVectorRemove(m_FreeSuballocationsBySize, index);
6853  return;
6854  }
6855  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
6856  }
6857  VMA_ASSERT(0 && "Not found.");
6858  }
6859 
6860  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6861 }
6862 
6864 // class VmaDeviceMemoryBlock
6865 
6866 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
6867  m_Metadata(hAllocator),
6868  m_MemoryTypeIndex(UINT32_MAX),
6869  m_Id(0),
6870  m_hMemory(VK_NULL_HANDLE),
6871  m_MapCount(0),
6872  m_pMappedData(VMA_NULL)
6873 {
6874 }
6875 
6876 void VmaDeviceMemoryBlock::Init(
6877  uint32_t newMemoryTypeIndex,
6878  VkDeviceMemory newMemory,
6879  VkDeviceSize newSize,
6880  uint32_t id)
6881 {
6882  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6883 
6884  m_MemoryTypeIndex = newMemoryTypeIndex;
6885  m_Id = id;
6886  m_hMemory = newMemory;
6887 
6888  m_Metadata.Init(newSize);
6889 }
6890 
6891 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
6892 {
6893  // This is the most important assert in the entire library.
6894  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
6895  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
6896 
6897  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
6898  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
6899  m_hMemory = VK_NULL_HANDLE;
6900 }
6901 
6902 bool VmaDeviceMemoryBlock::Validate() const
6903 {
6904  if((m_hMemory == VK_NULL_HANDLE) ||
6905  (m_Metadata.GetSize() == 0))
6906  {
6907  return false;
6908  }
6909 
6910  return m_Metadata.Validate();
6911 }
6912 
6913 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
6914 {
6915  void* pData = nullptr;
6916  VkResult res = Map(hAllocator, 1, &pData);
6917  if(res != VK_SUCCESS)
6918  {
6919  return res;
6920  }
6921 
6922  res = m_Metadata.CheckCorruption(pData);
6923 
6924  Unmap(hAllocator, 1);
6925 
6926  return res;
6927 }
6928 
6929 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
6930 {
6931  if(count == 0)
6932  {
6933  return VK_SUCCESS;
6934  }
6935 
6936  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6937  if(m_MapCount != 0)
6938  {
6939  m_MapCount += count;
6940  VMA_ASSERT(m_pMappedData != VMA_NULL);
6941  if(ppData != VMA_NULL)
6942  {
6943  *ppData = m_pMappedData;
6944  }
6945  return VK_SUCCESS;
6946  }
6947  else
6948  {
6949  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6950  hAllocator->m_hDevice,
6951  m_hMemory,
6952  0, // offset
6953  VK_WHOLE_SIZE,
6954  0, // flags
6955  &m_pMappedData);
6956  if(result == VK_SUCCESS)
6957  {
6958  if(ppData != VMA_NULL)
6959  {
6960  *ppData = m_pMappedData;
6961  }
6962  m_MapCount = count;
6963  }
6964  return result;
6965  }
6966 }
6967 
6968 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
6969 {
6970  if(count == 0)
6971  {
6972  return;
6973  }
6974 
6975  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6976  if(m_MapCount >= count)
6977  {
6978  m_MapCount -= count;
6979  if(m_MapCount == 0)
6980  {
6981  m_pMappedData = VMA_NULL;
6982  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
6983  }
6984  }
6985  else
6986  {
6987  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
6988  }
6989 }
6990 
6991 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
6992 {
6993  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
6994  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
6995 
6996  void* pData;
6997  VkResult res = Map(hAllocator, 1, &pData);
6998  if(res != VK_SUCCESS)
6999  {
7000  return res;
7001  }
7002 
7003  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
7004  VmaWriteMagicValue(pData, allocOffset + allocSize);
7005 
7006  Unmap(hAllocator, 1);
7007 
7008  return VK_SUCCESS;
7009 }
7010 
7011 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
7012 {
7013  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
7014  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
7015 
7016  void* pData;
7017  VkResult res = Map(hAllocator, 1, &pData);
7018  if(res != VK_SUCCESS)
7019  {
7020  return res;
7021  }
7022 
7023  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
7024  {
7025  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
7026  }
7027  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
7028  {
7029  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
7030  }
7031 
7032  Unmap(hAllocator, 1);
7033 
7034  return VK_SUCCESS;
7035 }
7036 
7037 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
7038  const VmaAllocator hAllocator,
7039  const VmaAllocation hAllocation,
7040  VkBuffer hBuffer)
7041 {
7042  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
7043  hAllocation->GetBlock() == this);
7044  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
7045  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
7046  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
7047  hAllocator->m_hDevice,
7048  hBuffer,
7049  m_hMemory,
7050  hAllocation->GetOffset());
7051 }
7052 
7053 VkResult VmaDeviceMemoryBlock::BindImageMemory(
7054  const VmaAllocator hAllocator,
7055  const VmaAllocation hAllocation,
7056  VkImage hImage)
7057 {
7058  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
7059  hAllocation->GetBlock() == this);
7060  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
7061  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
7062  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
7063  hAllocator->m_hDevice,
7064  hImage,
7065  m_hMemory,
7066  hAllocation->GetOffset());
7067 }
7068 
7069 static void InitStatInfo(VmaStatInfo& outInfo)
7070 {
7071  memset(&outInfo, 0, sizeof(outInfo));
7072  outInfo.allocationSizeMin = UINT64_MAX;
7073  outInfo.unusedRangeSizeMin = UINT64_MAX;
7074 }
7075 
7076 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
7077 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
7078 {
7079  inoutInfo.blockCount += srcInfo.blockCount;
7080  inoutInfo.allocationCount += srcInfo.allocationCount;
7081  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
7082  inoutInfo.usedBytes += srcInfo.usedBytes;
7083  inoutInfo.unusedBytes += srcInfo.unusedBytes;
7084  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
7085  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
7086  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
7087  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
7088 }
7089 
7090 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
7091 {
7092  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
7093  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
7094  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
7095  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
7096 }
7097 
7098 VmaPool_T::VmaPool_T(
7099  VmaAllocator hAllocator,
7100  const VmaPoolCreateInfo& createInfo) :
7101  m_BlockVector(
7102  hAllocator,
7103  createInfo.memoryTypeIndex,
7104  createInfo.blockSize,
7105  createInfo.minBlockCount,
7106  createInfo.maxBlockCount,
7107  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
7108  createInfo.frameInUseCount,
7109  true), // isCustomPool
7110  m_Id(0)
7111 {
7112 }
7113 
7114 VmaPool_T::~VmaPool_T()
7115 {
7116 }
7117 
7118 #if VMA_STATS_STRING_ENABLED
7119 
7120 #endif // #if VMA_STATS_STRING_ENABLED
7121 
7122 VmaBlockVector::VmaBlockVector(
7123  VmaAllocator hAllocator,
7124  uint32_t memoryTypeIndex,
7125  VkDeviceSize preferredBlockSize,
7126  size_t minBlockCount,
7127  size_t maxBlockCount,
7128  VkDeviceSize bufferImageGranularity,
7129  uint32_t frameInUseCount,
7130  bool isCustomPool) :
7131  m_hAllocator(hAllocator),
7132  m_MemoryTypeIndex(memoryTypeIndex),
7133  m_PreferredBlockSize(preferredBlockSize),
7134  m_MinBlockCount(minBlockCount),
7135  m_MaxBlockCount(maxBlockCount),
7136  m_BufferImageGranularity(bufferImageGranularity),
7137  m_FrameInUseCount(frameInUseCount),
7138  m_IsCustomPool(isCustomPool),
7139  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
7140  m_HasEmptyBlock(false),
7141  m_pDefragmentator(VMA_NULL),
7142  m_NextBlockId(0)
7143 {
7144 }
7145 
7146 VmaBlockVector::~VmaBlockVector()
7147 {
7148  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
7149 
7150  for(size_t i = m_Blocks.size(); i--; )
7151  {
7152  m_Blocks[i]->Destroy(m_hAllocator);
7153  vma_delete(m_hAllocator, m_Blocks[i]);
7154  }
7155 }
7156 
7157 VkResult VmaBlockVector::CreateMinBlocks()
7158 {
7159  for(size_t i = 0; i < m_MinBlockCount; ++i)
7160  {
7161  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
7162  if(res != VK_SUCCESS)
7163  {
7164  return res;
7165  }
7166  }
7167  return VK_SUCCESS;
7168 }
7169 
7170 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
7171 {
7172  pStats->size = 0;
7173  pStats->unusedSize = 0;
7174  pStats->allocationCount = 0;
7175  pStats->unusedRangeCount = 0;
7176  pStats->unusedRangeSizeMax = 0;
7177 
7178  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7179 
7180  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7181  {
7182  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7183  VMA_ASSERT(pBlock);
7184  VMA_HEAVY_ASSERT(pBlock->Validate());
7185  pBlock->m_Metadata.AddPoolStats(*pStats);
7186  }
7187 }
7188 
7189 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
7190 {
7191  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7192  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
7193  (VMA_DEBUG_MARGIN > 0) &&
7194  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
7195 }
7196 
7197 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
7198 
7199 VkResult VmaBlockVector::Allocate(
7200  VmaPool hCurrentPool,
7201  uint32_t currentFrameIndex,
7202  VkDeviceSize size,
7203  VkDeviceSize alignment,
7204  const VmaAllocationCreateInfo& createInfo,
7205  VmaSuballocationType suballocType,
7206  VmaAllocation* pAllocation)
7207 {
7208  // Early reject: requested allocation size is larger that maximum block size for this block vector.
7209  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
7210  {
7211  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7212  }
7213 
7214  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
7215  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
7216 
7217  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7218 
7219  // 1. Search existing allocations. Try to allocate without making other allocations lost.
7220  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
7221  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
7222  {
7223  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
7224  VMA_ASSERT(pCurrBlock);
7225  VmaAllocationRequest currRequest = {};
7226  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
7227  currentFrameIndex,
7228  m_FrameInUseCount,
7229  m_BufferImageGranularity,
7230  size,
7231  alignment,
7232  suballocType,
7233  false, // canMakeOtherLost
7234  &currRequest))
7235  {
7236  // Allocate from pCurrBlock.
7237  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
7238 
7239  if(mapped)
7240  {
7241  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
7242  if(res != VK_SUCCESS)
7243  {
7244  return res;
7245  }
7246  }
7247 
7248  // We no longer have an empty Allocation.
7249  if(pCurrBlock->m_Metadata.IsEmpty())
7250  {
7251  m_HasEmptyBlock = false;
7252  }
7253 
7254  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7255  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
7256  (*pAllocation)->InitBlockAllocation(
7257  hCurrentPool,
7258  pCurrBlock,
7259  currRequest.offset,
7260  alignment,
7261  size,
7262  suballocType,
7263  mapped,
7264  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7265  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
7266  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
7267  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7268  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7269  {
7270  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7271  }
7272  if(IsCorruptionDetectionEnabled())
7273  {
7274  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
7275  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7276  }
7277  return VK_SUCCESS;
7278  }
7279  }
7280 
7281  const bool canCreateNewBlock =
7282  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
7283  (m_Blocks.size() < m_MaxBlockCount);
7284 
7285  // 2. Try to create new block.
7286  if(canCreateNewBlock)
7287  {
7288  // Calculate optimal size for new block.
7289  VkDeviceSize newBlockSize = m_PreferredBlockSize;
7290  uint32_t newBlockSizeShift = 0;
7291  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
7292 
7293  // Allocating blocks of other sizes is allowed only in default pools.
7294  // In custom pools block size is fixed.
7295  if(m_IsCustomPool == false)
7296  {
7297  // Allocate 1/8, 1/4, 1/2 as first blocks.
7298  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
7299  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
7300  {
7301  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
7302  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
7303  {
7304  newBlockSize = smallerNewBlockSize;
7305  ++newBlockSizeShift;
7306  }
7307  else
7308  {
7309  break;
7310  }
7311  }
7312  }
7313 
7314  size_t newBlockIndex = 0;
7315  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
7316  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
7317  if(m_IsCustomPool == false)
7318  {
7319  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
7320  {
7321  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
7322  if(smallerNewBlockSize >= size)
7323  {
7324  newBlockSize = smallerNewBlockSize;
7325  ++newBlockSizeShift;
7326  res = CreateBlock(newBlockSize, &newBlockIndex);
7327  }
7328  else
7329  {
7330  break;
7331  }
7332  }
7333  }
7334 
7335  if(res == VK_SUCCESS)
7336  {
7337  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
7338  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
7339 
7340  if(mapped)
7341  {
7342  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
7343  if(res != VK_SUCCESS)
7344  {
7345  return res;
7346  }
7347  }
7348 
7349  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
7350  VmaAllocationRequest allocRequest;
7351  if(pBlock->m_Metadata.CreateAllocationRequest(
7352  currentFrameIndex,
7353  m_FrameInUseCount,
7354  m_BufferImageGranularity,
7355  size,
7356  alignment,
7357  suballocType,
7358  false, // canMakeOtherLost
7359  &allocRequest))
7360  {
7361  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7362  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
7363  (*pAllocation)->InitBlockAllocation(
7364  hCurrentPool,
7365  pBlock,
7366  allocRequest.offset,
7367  alignment,
7368  size,
7369  suballocType,
7370  mapped,
7371  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7372  VMA_HEAVY_ASSERT(pBlock->Validate());
7373  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
7374  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7375  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7376  {
7377  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7378  }
7379  if(IsCorruptionDetectionEnabled())
7380  {
7381  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
7382  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7383  }
7384  return VK_SUCCESS;
7385  }
7386  else
7387  {
7388  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
7389  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7390  }
7391  }
7392  }
7393 
7394  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
7395 
7396  // 3. Try to allocate from existing blocks with making other allocations lost.
7397  if(canMakeOtherLost)
7398  {
7399  uint32_t tryIndex = 0;
7400  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
7401  {
7402  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
7403  VmaAllocationRequest bestRequest = {};
7404  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
7405 
7406  // 1. Search existing allocations.
7407  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
7408  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
7409  {
7410  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
7411  VMA_ASSERT(pCurrBlock);
7412  VmaAllocationRequest currRequest = {};
7413  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
7414  currentFrameIndex,
7415  m_FrameInUseCount,
7416  m_BufferImageGranularity,
7417  size,
7418  alignment,
7419  suballocType,
7420  canMakeOtherLost,
7421  &currRequest))
7422  {
7423  const VkDeviceSize currRequestCost = currRequest.CalcCost();
7424  if(pBestRequestBlock == VMA_NULL ||
7425  currRequestCost < bestRequestCost)
7426  {
7427  pBestRequestBlock = pCurrBlock;
7428  bestRequest = currRequest;
7429  bestRequestCost = currRequestCost;
7430 
7431  if(bestRequestCost == 0)
7432  {
7433  break;
7434  }
7435  }
7436  }
7437  }
7438 
7439  if(pBestRequestBlock != VMA_NULL)
7440  {
7441  if(mapped)
7442  {
7443  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
7444  if(res != VK_SUCCESS)
7445  {
7446  return res;
7447  }
7448  }
7449 
7450  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
7451  currentFrameIndex,
7452  m_FrameInUseCount,
7453  &bestRequest))
7454  {
7455  // We no longer have an empty Allocation.
7456  if(pBestRequestBlock->m_Metadata.IsEmpty())
7457  {
7458  m_HasEmptyBlock = false;
7459  }
7460  // Allocate from this pBlock.
7461  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7462  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
7463  (*pAllocation)->InitBlockAllocation(
7464  hCurrentPool,
7465  pBestRequestBlock,
7466  bestRequest.offset,
7467  alignment,
7468  size,
7469  suballocType,
7470  mapped,
7471  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7472  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
7473  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
7474  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7475  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7476  {
7477  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7478  }
7479  if(IsCorruptionDetectionEnabled())
7480  {
7481  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
7482  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7483  }
7484  return VK_SUCCESS;
7485  }
7486  // else: Some allocations must have been touched while we are here. Next try.
7487  }
7488  else
7489  {
7490  // Could not find place in any of the blocks - break outer loop.
7491  break;
7492  }
7493  }
7494  /* Maximum number of tries exceeded - a very unlike event when many other
7495  threads are simultaneously touching allocations making it impossible to make
7496  lost at the same time as we try to allocate. */
7497  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
7498  {
7499  return VK_ERROR_TOO_MANY_OBJECTS;
7500  }
7501  }
7502 
7503  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7504 }
7505 
7506 void VmaBlockVector::Free(
7507  VmaAllocation hAllocation)
7508 {
7509  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
7510 
7511  // Scope for lock.
7512  {
7513  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7514 
7515  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
7516 
7517  if(IsCorruptionDetectionEnabled())
7518  {
7519  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
7520  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
7521  }
7522 
7523  if(hAllocation->IsPersistentMap())
7524  {
7525  pBlock->Unmap(m_hAllocator, 1);
7526  }
7527 
7528  pBlock->m_Metadata.Free(hAllocation);
7529  VMA_HEAVY_ASSERT(pBlock->Validate());
7530 
7531  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
7532 
7533  // pBlock became empty after this deallocation.
7534  if(pBlock->m_Metadata.IsEmpty())
7535  {
7536  // Already has empty Allocation. We don't want to have two, so delete this one.
7537  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
7538  {
7539  pBlockToDelete = pBlock;
7540  Remove(pBlock);
7541  }
7542  // We now have first empty block.
7543  else
7544  {
7545  m_HasEmptyBlock = true;
7546  }
7547  }
7548  // pBlock didn't become empty, but we have another empty block - find and free that one.
7549  // (This is optional, heuristics.)
7550  else if(m_HasEmptyBlock)
7551  {
7552  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
7553  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
7554  {
7555  pBlockToDelete = pLastBlock;
7556  m_Blocks.pop_back();
7557  m_HasEmptyBlock = false;
7558  }
7559  }
7560 
7561  IncrementallySortBlocks();
7562  }
7563 
7564  // Destruction of a free Allocation. Deferred until this point, outside of mutex
7565  // lock, for performance reason.
7566  if(pBlockToDelete != VMA_NULL)
7567  {
7568  VMA_DEBUG_LOG(" Deleted empty allocation");
7569  pBlockToDelete->Destroy(m_hAllocator);
7570  vma_delete(m_hAllocator, pBlockToDelete);
7571  }
7572 }
7573 
7574 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
7575 {
7576  VkDeviceSize result = 0;
7577  for(size_t i = m_Blocks.size(); i--; )
7578  {
7579  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
7580  if(result >= m_PreferredBlockSize)
7581  {
7582  break;
7583  }
7584  }
7585  return result;
7586 }
7587 
7588 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
7589 {
7590  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7591  {
7592  if(m_Blocks[blockIndex] == pBlock)
7593  {
7594  VmaVectorRemove(m_Blocks, blockIndex);
7595  return;
7596  }
7597  }
7598  VMA_ASSERT(0);
7599 }
7600 
7601 void VmaBlockVector::IncrementallySortBlocks()
7602 {
7603  // Bubble sort only until first swap.
7604  for(size_t i = 1; i < m_Blocks.size(); ++i)
7605  {
7606  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
7607  {
7608  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
7609  return;
7610  }
7611  }
7612 }
7613 
7614 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
7615 {
7616  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7617  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
7618  allocInfo.allocationSize = blockSize;
7619  VkDeviceMemory mem = VK_NULL_HANDLE;
7620  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
7621  if(res < 0)
7622  {
7623  return res;
7624  }
7625 
7626  // New VkDeviceMemory successfully created.
7627 
7628  // Create new Allocation for it.
7629  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
7630  pBlock->Init(
7631  m_MemoryTypeIndex,
7632  mem,
7633  allocInfo.allocationSize,
7634  m_NextBlockId++);
7635 
7636  m_Blocks.push_back(pBlock);
7637  if(pNewBlockIndex != VMA_NULL)
7638  {
7639  *pNewBlockIndex = m_Blocks.size() - 1;
7640  }
7641 
7642  return VK_SUCCESS;
7643 }
7644 
7645 #if VMA_STATS_STRING_ENABLED
7646 
7647 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
7648 {
7649  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7650 
7651  json.BeginObject();
7652 
7653  if(m_IsCustomPool)
7654  {
7655  json.WriteString("MemoryTypeIndex");
7656  json.WriteNumber(m_MemoryTypeIndex);
7657 
7658  json.WriteString("BlockSize");
7659  json.WriteNumber(m_PreferredBlockSize);
7660 
7661  json.WriteString("BlockCount");
7662  json.BeginObject(true);
7663  if(m_MinBlockCount > 0)
7664  {
7665  json.WriteString("Min");
7666  json.WriteNumber((uint64_t)m_MinBlockCount);
7667  }
7668  if(m_MaxBlockCount < SIZE_MAX)
7669  {
7670  json.WriteString("Max");
7671  json.WriteNumber((uint64_t)m_MaxBlockCount);
7672  }
7673  json.WriteString("Cur");
7674  json.WriteNumber((uint64_t)m_Blocks.size());
7675  json.EndObject();
7676 
7677  if(m_FrameInUseCount > 0)
7678  {
7679  json.WriteString("FrameInUseCount");
7680  json.WriteNumber(m_FrameInUseCount);
7681  }
7682  }
7683  else
7684  {
7685  json.WriteString("PreferredBlockSize");
7686  json.WriteNumber(m_PreferredBlockSize);
7687  }
7688 
7689  json.WriteString("Blocks");
7690  json.BeginObject();
7691  for(size_t i = 0; i < m_Blocks.size(); ++i)
7692  {
7693  json.BeginString();
7694  json.ContinueString(m_Blocks[i]->GetId());
7695  json.EndString();
7696 
7697  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
7698  }
7699  json.EndObject();
7700 
7701  json.EndObject();
7702 }
7703 
7704 #endif // #if VMA_STATS_STRING_ENABLED
7705 
7706 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
7707  VmaAllocator hAllocator,
7708  uint32_t currentFrameIndex)
7709 {
7710  if(m_pDefragmentator == VMA_NULL)
7711  {
7712  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
7713  hAllocator,
7714  this,
7715  currentFrameIndex);
7716  }
7717 
7718  return m_pDefragmentator;
7719 }
7720 
7721 VkResult VmaBlockVector::Defragment(
7722  VmaDefragmentationStats* pDefragmentationStats,
7723  VkDeviceSize& maxBytesToMove,
7724  uint32_t& maxAllocationsToMove)
7725 {
7726  if(m_pDefragmentator == VMA_NULL)
7727  {
7728  return VK_SUCCESS;
7729  }
7730 
7731  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7732 
7733  // Defragment.
7734  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
7735 
7736  // Accumulate statistics.
7737  if(pDefragmentationStats != VMA_NULL)
7738  {
7739  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
7740  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
7741  pDefragmentationStats->bytesMoved += bytesMoved;
7742  pDefragmentationStats->allocationsMoved += allocationsMoved;
7743  VMA_ASSERT(bytesMoved <= maxBytesToMove);
7744  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
7745  maxBytesToMove -= bytesMoved;
7746  maxAllocationsToMove -= allocationsMoved;
7747  }
7748 
7749  // Free empty blocks.
7750  m_HasEmptyBlock = false;
7751  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
7752  {
7753  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
7754  if(pBlock->m_Metadata.IsEmpty())
7755  {
7756  if(m_Blocks.size() > m_MinBlockCount)
7757  {
7758  if(pDefragmentationStats != VMA_NULL)
7759  {
7760  ++pDefragmentationStats->deviceMemoryBlocksFreed;
7761  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
7762  }
7763 
7764  VmaVectorRemove(m_Blocks, blockIndex);
7765  pBlock->Destroy(m_hAllocator);
7766  vma_delete(m_hAllocator, pBlock);
7767  }
7768  else
7769  {
7770  m_HasEmptyBlock = true;
7771  }
7772  }
7773  }
7774 
7775  return result;
7776 }
7777 
7778 void VmaBlockVector::DestroyDefragmentator()
7779 {
7780  if(m_pDefragmentator != VMA_NULL)
7781  {
7782  vma_delete(m_hAllocator, m_pDefragmentator);
7783  m_pDefragmentator = VMA_NULL;
7784  }
7785 }
7786 
7787 void VmaBlockVector::MakePoolAllocationsLost(
7788  uint32_t currentFrameIndex,
7789  size_t* pLostAllocationCount)
7790 {
7791  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7792  size_t lostAllocationCount = 0;
7793  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7794  {
7795  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7796  VMA_ASSERT(pBlock);
7797  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
7798  }
7799  if(pLostAllocationCount != VMA_NULL)
7800  {
7801  *pLostAllocationCount = lostAllocationCount;
7802  }
7803 }
7804 
7805 VkResult VmaBlockVector::CheckCorruption()
7806 {
7807  if(!IsCorruptionDetectionEnabled())
7808  {
7809  return VK_ERROR_FEATURE_NOT_PRESENT;
7810  }
7811 
7812  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7813  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7814  {
7815  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7816  VMA_ASSERT(pBlock);
7817  VkResult res = pBlock->CheckCorruption(m_hAllocator);
7818  if(res != VK_SUCCESS)
7819  {
7820  return res;
7821  }
7822  }
7823  return VK_SUCCESS;
7824 }
7825 
7826 void VmaBlockVector::AddStats(VmaStats* pStats)
7827 {
7828  const uint32_t memTypeIndex = m_MemoryTypeIndex;
7829  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
7830 
7831  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7832 
7833  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7834  {
7835  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7836  VMA_ASSERT(pBlock);
7837  VMA_HEAVY_ASSERT(pBlock->Validate());
7838  VmaStatInfo allocationStatInfo;
7839  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
7840  VmaAddStatInfo(pStats->total, allocationStatInfo);
7841  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7842  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7843  }
7844 }
7845 
7847 // VmaDefragmentator members definition
7848 
7849 VmaDefragmentator::VmaDefragmentator(
7850  VmaAllocator hAllocator,
7851  VmaBlockVector* pBlockVector,
7852  uint32_t currentFrameIndex) :
7853  m_hAllocator(hAllocator),
7854  m_pBlockVector(pBlockVector),
7855  m_CurrentFrameIndex(currentFrameIndex),
7856  m_BytesMoved(0),
7857  m_AllocationsMoved(0),
7858  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
7859  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
7860 {
7861 }
7862 
7863 VmaDefragmentator::~VmaDefragmentator()
7864 {
7865  for(size_t i = m_Blocks.size(); i--; )
7866  {
7867  vma_delete(m_hAllocator, m_Blocks[i]);
7868  }
7869 }
7870 
7871 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
7872 {
7873  AllocationInfo allocInfo;
7874  allocInfo.m_hAllocation = hAlloc;
7875  allocInfo.m_pChanged = pChanged;
7876  m_Allocations.push_back(allocInfo);
7877 }
7878 
7879 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
7880 {
7881  // It has already been mapped for defragmentation.
7882  if(m_pMappedDataForDefragmentation)
7883  {
7884  *ppMappedData = m_pMappedDataForDefragmentation;
7885  return VK_SUCCESS;
7886  }
7887 
7888  // It is originally mapped.
7889  if(m_pBlock->GetMappedData())
7890  {
7891  *ppMappedData = m_pBlock->GetMappedData();
7892  return VK_SUCCESS;
7893  }
7894 
7895  // Map on first usage.
7896  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
7897  *ppMappedData = m_pMappedDataForDefragmentation;
7898  return res;
7899 }
7900 
7901 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
7902 {
7903  if(m_pMappedDataForDefragmentation != VMA_NULL)
7904  {
7905  m_pBlock->Unmap(hAllocator, 1);
7906  }
7907 }
7908 
7909 VkResult VmaDefragmentator::DefragmentRound(
7910  VkDeviceSize maxBytesToMove,
7911  uint32_t maxAllocationsToMove)
7912 {
7913  if(m_Blocks.empty())
7914  {
7915  return VK_SUCCESS;
7916  }
7917 
7918  size_t srcBlockIndex = m_Blocks.size() - 1;
7919  size_t srcAllocIndex = SIZE_MAX;
7920  for(;;)
7921  {
7922  // 1. Find next allocation to move.
7923  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
7924  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
7925  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
7926  {
7927  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
7928  {
7929  // Finished: no more allocations to process.
7930  if(srcBlockIndex == 0)
7931  {
7932  return VK_SUCCESS;
7933  }
7934  else
7935  {
7936  --srcBlockIndex;
7937  srcAllocIndex = SIZE_MAX;
7938  }
7939  }
7940  else
7941  {
7942  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
7943  }
7944  }
7945 
7946  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
7947  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
7948 
7949  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
7950  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
7951  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
7952  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
7953 
7954  // 2. Try to find new place for this allocation in preceding or current block.
7955  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
7956  {
7957  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
7958  VmaAllocationRequest dstAllocRequest;
7959  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
7960  m_CurrentFrameIndex,
7961  m_pBlockVector->GetFrameInUseCount(),
7962  m_pBlockVector->GetBufferImageGranularity(),
7963  size,
7964  alignment,
7965  suballocType,
7966  false, // canMakeOtherLost
7967  &dstAllocRequest) &&
7968  MoveMakesSense(
7969  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
7970  {
7971  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
7972 
7973  // Reached limit on number of allocations or bytes to move.
7974  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
7975  (m_BytesMoved + size > maxBytesToMove))
7976  {
7977  return VK_INCOMPLETE;
7978  }
7979 
7980  void* pDstMappedData = VMA_NULL;
7981  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
7982  if(res != VK_SUCCESS)
7983  {
7984  return res;
7985  }
7986 
7987  void* pSrcMappedData = VMA_NULL;
7988  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
7989  if(res != VK_SUCCESS)
7990  {
7991  return res;
7992  }
7993 
7994  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
7995  memcpy(
7996  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
7997  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
7998  static_cast<size_t>(size));
7999 
8000  if(VMA_DEBUG_MARGIN > 0)
8001  {
8002  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
8003  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
8004  }
8005 
8006  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
8007  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
8008 
8009  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
8010 
8011  if(allocInfo.m_pChanged != VMA_NULL)
8012  {
8013  *allocInfo.m_pChanged = VK_TRUE;
8014  }
8015 
8016  ++m_AllocationsMoved;
8017  m_BytesMoved += size;
8018 
8019  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
8020 
8021  break;
8022  }
8023  }
8024 
8025  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
8026 
8027  if(srcAllocIndex > 0)
8028  {
8029  --srcAllocIndex;
8030  }
8031  else
8032  {
8033  if(srcBlockIndex > 0)
8034  {
8035  --srcBlockIndex;
8036  srcAllocIndex = SIZE_MAX;
8037  }
8038  else
8039  {
8040  return VK_SUCCESS;
8041  }
8042  }
8043  }
8044 }
8045 
8046 VkResult VmaDefragmentator::Defragment(
8047  VkDeviceSize maxBytesToMove,
8048  uint32_t maxAllocationsToMove)
8049 {
8050  if(m_Allocations.empty())
8051  {
8052  return VK_SUCCESS;
8053  }
8054 
8055  // Create block info for each block.
8056  const size_t blockCount = m_pBlockVector->m_Blocks.size();
8057  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8058  {
8059  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
8060  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
8061  m_Blocks.push_back(pBlockInfo);
8062  }
8063 
8064  // Sort them by m_pBlock pointer value.
8065  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
8066 
8067  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
8068  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
8069  {
8070  AllocationInfo& allocInfo = m_Allocations[blockIndex];
8071  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
8072  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
8073  {
8074  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
8075  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
8076  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
8077  {
8078  (*it)->m_Allocations.push_back(allocInfo);
8079  }
8080  else
8081  {
8082  VMA_ASSERT(0);
8083  }
8084  }
8085  }
8086  m_Allocations.clear();
8087 
8088  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8089  {
8090  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
8091  pBlockInfo->CalcHasNonMovableAllocations();
8092  pBlockInfo->SortAllocationsBySizeDescecnding();
8093  }
8094 
8095  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
8096  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
8097 
8098  // Execute defragmentation rounds (the main part).
8099  VkResult result = VK_SUCCESS;
8100  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
8101  {
8102  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
8103  }
8104 
8105  // Unmap blocks that were mapped for defragmentation.
8106  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8107  {
8108  m_Blocks[blockIndex]->Unmap(m_hAllocator);
8109  }
8110 
8111  return result;
8112 }
8113 
8114 bool VmaDefragmentator::MoveMakesSense(
8115  size_t dstBlockIndex, VkDeviceSize dstOffset,
8116  size_t srcBlockIndex, VkDeviceSize srcOffset)
8117 {
8118  if(dstBlockIndex < srcBlockIndex)
8119  {
8120  return true;
8121  }
8122  if(dstBlockIndex > srcBlockIndex)
8123  {
8124  return false;
8125  }
8126  if(dstOffset < srcOffset)
8127  {
8128  return true;
8129  }
8130  return false;
8131 }
8132 
8134 // VmaRecorder
8135 
8136 #if VMA_RECORDING_ENABLED
8137 
8138 VmaRecorder::VmaRecorder() :
8139  m_UseMutex(true),
8140  m_Flags(0),
8141  m_File(VMA_NULL),
8142  m_Freq(INT64_MAX),
8143  m_StartCounter(INT64_MAX)
8144 {
8145 }
8146 
8147 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
8148 {
8149  m_UseMutex = useMutex;
8150  m_Flags = settings.flags;
8151 
8152  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
8153  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
8154 
8155  // Open file for writing.
8156  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
8157  if(err != 0)
8158  {
8159  return VK_ERROR_INITIALIZATION_FAILED;
8160  }
8161 
8162  // Write header.
8163  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
8164  fprintf(m_File, "%s\n", "1,2");
8165 
8166  return VK_SUCCESS;
8167 }
8168 
8169 VmaRecorder::~VmaRecorder()
8170 {
8171  if(m_File != VMA_NULL)
8172  {
8173  fclose(m_File);
8174  }
8175 }
8176 
8177 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
8178 {
8179  CallParams callParams;
8180  GetBasicParams(callParams);
8181 
8182  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8183  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
8184  Flush();
8185 }
8186 
8187 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
8188 {
8189  CallParams callParams;
8190  GetBasicParams(callParams);
8191 
8192  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8193  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
8194  Flush();
8195 }
8196 
8197 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
8198 {
8199  CallParams callParams;
8200  GetBasicParams(callParams);
8201 
8202  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8203  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
8204  createInfo.memoryTypeIndex,
8205  createInfo.flags,
8206  createInfo.blockSize,
8207  createInfo.minBlockCount,
8208  createInfo.maxBlockCount,
8209  createInfo.frameInUseCount,
8210  pool);
8211  Flush();
8212 }
8213 
8214 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
8215 {
8216  CallParams callParams;
8217  GetBasicParams(callParams);
8218 
8219  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8220  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
8221  pool);
8222  Flush();
8223 }
8224 
8225 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
8226  const VkMemoryRequirements& vkMemReq,
8227  const VmaAllocationCreateInfo& createInfo,
8228  VmaAllocation allocation)
8229 {
8230  CallParams callParams;
8231  GetBasicParams(callParams);
8232 
8233  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8234  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8235  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8236  vkMemReq.size,
8237  vkMemReq.alignment,
8238  vkMemReq.memoryTypeBits,
8239  createInfo.flags,
8240  createInfo.usage,
8241  createInfo.requiredFlags,
8242  createInfo.preferredFlags,
8243  createInfo.memoryTypeBits,
8244  createInfo.pool,
8245  allocation,
8246  userDataStr.GetString());
8247  Flush();
8248 }
8249 
8250 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
8251  const VkMemoryRequirements& vkMemReq,
8252  bool requiresDedicatedAllocation,
8253  bool prefersDedicatedAllocation,
8254  const VmaAllocationCreateInfo& createInfo,
8255  VmaAllocation allocation)
8256 {
8257  CallParams callParams;
8258  GetBasicParams(callParams);
8259 
8260  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8261  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8262  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8263  vkMemReq.size,
8264  vkMemReq.alignment,
8265  vkMemReq.memoryTypeBits,
8266  requiresDedicatedAllocation ? 1 : 0,
8267  prefersDedicatedAllocation ? 1 : 0,
8268  createInfo.flags,
8269  createInfo.usage,
8270  createInfo.requiredFlags,
8271  createInfo.preferredFlags,
8272  createInfo.memoryTypeBits,
8273  createInfo.pool,
8274  allocation,
8275  userDataStr.GetString());
8276  Flush();
8277 }
8278 
8279 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
8280  const VkMemoryRequirements& vkMemReq,
8281  bool requiresDedicatedAllocation,
8282  bool prefersDedicatedAllocation,
8283  const VmaAllocationCreateInfo& createInfo,
8284  VmaAllocation allocation)
8285 {
8286  CallParams callParams;
8287  GetBasicParams(callParams);
8288 
8289  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8290  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8291  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8292  vkMemReq.size,
8293  vkMemReq.alignment,
8294  vkMemReq.memoryTypeBits,
8295  requiresDedicatedAllocation ? 1 : 0,
8296  prefersDedicatedAllocation ? 1 : 0,
8297  createInfo.flags,
8298  createInfo.usage,
8299  createInfo.requiredFlags,
8300  createInfo.preferredFlags,
8301  createInfo.memoryTypeBits,
8302  createInfo.pool,
8303  allocation,
8304  userDataStr.GetString());
8305  Flush();
8306 }
8307 
8308 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
8309  VmaAllocation allocation)
8310 {
8311  CallParams callParams;
8312  GetBasicParams(callParams);
8313 
8314  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8315  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8316  allocation);
8317  Flush();
8318 }
8319 
8320 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
8321  VmaAllocation allocation,
8322  const void* pUserData)
8323 {
8324  CallParams callParams;
8325  GetBasicParams(callParams);
8326 
8327  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8328  UserDataString userDataStr(
8329  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
8330  pUserData);
8331  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8332  allocation,
8333  userDataStr.GetString());
8334  Flush();
8335 }
8336 
8337 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
8338  VmaAllocation allocation)
8339 {
8340  CallParams callParams;
8341  GetBasicParams(callParams);
8342 
8343  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8344  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
8345  allocation);
8346  Flush();
8347 }
8348 
8349 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
8350  VmaAllocation allocation)
8351 {
8352  CallParams callParams;
8353  GetBasicParams(callParams);
8354 
8355  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8356  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8357  allocation);
8358  Flush();
8359 }
8360 
8361 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
8362  VmaAllocation allocation)
8363 {
8364  CallParams callParams;
8365  GetBasicParams(callParams);
8366 
8367  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8368  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8369  allocation);
8370  Flush();
8371 }
8372 
8373 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
8374  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
8375 {
8376  CallParams callParams;
8377  GetBasicParams(callParams);
8378 
8379  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8380  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
8381  allocation,
8382  offset,
8383  size);
8384  Flush();
8385 }
8386 
8387 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
8388  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
8389 {
8390  CallParams callParams;
8391  GetBasicParams(callParams);
8392 
8393  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8394  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
8395  allocation,
8396  offset,
8397  size);
8398  Flush();
8399 }
8400 
8401 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
8402  const VkBufferCreateInfo& bufCreateInfo,
8403  const VmaAllocationCreateInfo& allocCreateInfo,
8404  VmaAllocation allocation)
8405 {
8406  CallParams callParams;
8407  GetBasicParams(callParams);
8408 
8409  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8410  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
8411  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8412  bufCreateInfo.flags,
8413  bufCreateInfo.size,
8414  bufCreateInfo.usage,
8415  bufCreateInfo.sharingMode,
8416  allocCreateInfo.flags,
8417  allocCreateInfo.usage,
8418  allocCreateInfo.requiredFlags,
8419  allocCreateInfo.preferredFlags,
8420  allocCreateInfo.memoryTypeBits,
8421  allocCreateInfo.pool,
8422  allocation,
8423  userDataStr.GetString());
8424  Flush();
8425 }
8426 
8427 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
8428  const VkImageCreateInfo& imageCreateInfo,
8429  const VmaAllocationCreateInfo& allocCreateInfo,
8430  VmaAllocation allocation)
8431 {
8432  CallParams callParams;
8433  GetBasicParams(callParams);
8434 
8435  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8436  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
8437  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8438  imageCreateInfo.flags,
8439  imageCreateInfo.imageType,
8440  imageCreateInfo.format,
8441  imageCreateInfo.extent.width,
8442  imageCreateInfo.extent.height,
8443  imageCreateInfo.extent.depth,
8444  imageCreateInfo.mipLevels,
8445  imageCreateInfo.arrayLayers,
8446  imageCreateInfo.samples,
8447  imageCreateInfo.tiling,
8448  imageCreateInfo.usage,
8449  imageCreateInfo.sharingMode,
8450  imageCreateInfo.initialLayout,
8451  allocCreateInfo.flags,
8452  allocCreateInfo.usage,
8453  allocCreateInfo.requiredFlags,
8454  allocCreateInfo.preferredFlags,
8455  allocCreateInfo.memoryTypeBits,
8456  allocCreateInfo.pool,
8457  allocation,
8458  userDataStr.GetString());
8459  Flush();
8460 }
8461 
8462 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
8463  VmaAllocation allocation)
8464 {
8465  CallParams callParams;
8466  GetBasicParams(callParams);
8467 
8468  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8469  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
8470  allocation);
8471  Flush();
8472 }
8473 
8474 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
8475  VmaAllocation allocation)
8476 {
8477  CallParams callParams;
8478  GetBasicParams(callParams);
8479 
8480  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8481  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
8482  allocation);
8483  Flush();
8484 }
8485 
8486 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
8487  VmaAllocation allocation)
8488 {
8489  CallParams callParams;
8490  GetBasicParams(callParams);
8491 
8492  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8493  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
8494  allocation);
8495  Flush();
8496 }
8497 
8498 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
8499  VmaAllocation allocation)
8500 {
8501  CallParams callParams;
8502  GetBasicParams(callParams);
8503 
8504  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8505  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
8506  allocation);
8507  Flush();
8508 }
8509 
8510 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
8511 {
8512  if(pUserData != VMA_NULL)
8513  {
8514  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
8515  {
8516  m_Str = (const char*)pUserData;
8517  }
8518  else
8519  {
8520  sprintf_s(m_PtrStr, "%p", pUserData);
8521  m_Str = m_PtrStr;
8522  }
8523  }
8524  else
8525  {
8526  m_Str = "";
8527  }
8528 }
8529 
8530 void VmaRecorder::GetBasicParams(CallParams& outParams)
8531 {
8532  outParams.threadId = GetCurrentThreadId();
8533 
8534  LARGE_INTEGER counter;
8535  QueryPerformanceCounter(&counter);
8536  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
8537 }
8538 
8539 void VmaRecorder::Flush()
8540 {
8541  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
8542  {
8543  fflush(m_File);
8544  }
8545 }
8546 
8547 #endif // #if VMA_RECORDING_ENABLED
8548 
8550 // VmaAllocator_T
8551 
8552 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
8553  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
8554  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
8555  m_hDevice(pCreateInfo->device),
8556  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
8557  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
8558  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
8559  m_PreferredLargeHeapBlockSize(0),
8560  m_PhysicalDevice(pCreateInfo->physicalDevice),
8561  m_CurrentFrameIndex(0),
8562  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
8563  m_NextPoolId(0)
8564 #if VMA_RECORDING_ENABLED
8565  ,m_pRecorder(VMA_NULL)
8566 #endif
8567 {
8568  if(VMA_DEBUG_DETECT_CORRUPTION)
8569  {
8570  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
8571  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
8572  }
8573 
8574  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
8575 
8576 #if !(VMA_DEDICATED_ALLOCATION)
8577  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
8578  {
8579  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
8580  }
8581 #endif
8582 
8583  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
8584  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
8585  memset(&m_MemProps, 0, sizeof(m_MemProps));
8586 
8587  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
8588  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
8589 
8590  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
8591  {
8592  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
8593  }
8594 
8595  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
8596  {
8597  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
8598  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
8599  }
8600 
8601  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
8602 
8603  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
8604  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
8605 
8606  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
8607  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
8608 
8609  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
8610  {
8611  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
8612  {
8613  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
8614  if(limit != VK_WHOLE_SIZE)
8615  {
8616  m_HeapSizeLimit[heapIndex] = limit;
8617  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
8618  {
8619  m_MemProps.memoryHeaps[heapIndex].size = limit;
8620  }
8621  }
8622  }
8623  }
8624 
8625  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8626  {
8627  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
8628 
8629  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
8630  this,
8631  memTypeIndex,
8632  preferredBlockSize,
8633  0,
8634  SIZE_MAX,
8635  GetBufferImageGranularity(),
8636  pCreateInfo->frameInUseCount,
8637  false); // isCustomPool
8638  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
8639  // becase minBlockCount is 0.
8640  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
8641 
8642  }
8643 }
8644 
8645 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
8646 {
8647  VkResult res = VK_SUCCESS;
8648 
8649  if(pCreateInfo->pRecordSettings != VMA_NULL &&
8650  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
8651  {
8652 #if VMA_RECORDING_ENABLED
8653  m_pRecorder = vma_new(this, VmaRecorder)();
8654  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
8655  if(res != VK_SUCCESS)
8656  {
8657  return res;
8658  }
8659  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
8660 #else
8661  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
8662  return VK_ERROR_FEATURE_NOT_PRESENT;
8663 #endif
8664  }
8665 
8666  return res;
8667 }
8668 
8669 VmaAllocator_T::~VmaAllocator_T()
8670 {
8671 #if VMA_RECORDING_ENABLED
8672  if(m_pRecorder != VMA_NULL)
8673  {
8674  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
8675  vma_delete(this, m_pRecorder);
8676  }
8677 #endif
8678 
8679  VMA_ASSERT(m_Pools.empty());
8680 
8681  for(size_t i = GetMemoryTypeCount(); i--; )
8682  {
8683  vma_delete(this, m_pDedicatedAllocations[i]);
8684  vma_delete(this, m_pBlockVectors[i]);
8685  }
8686 }
8687 
8688 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
8689 {
8690 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
8691  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
8692  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
8693  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
8694  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
8695  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
8696  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
8697  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
8698  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
8699  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
8700  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
8701  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
8702  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
8703  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
8704  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
8705  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
8706  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
8707 #if VMA_DEDICATED_ALLOCATION
8708  if(m_UseKhrDedicatedAllocation)
8709  {
8710  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
8711  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
8712  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
8713  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
8714  }
8715 #endif // #if VMA_DEDICATED_ALLOCATION
8716 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
8717 
8718 #define VMA_COPY_IF_NOT_NULL(funcName) \
8719  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
8720 
8721  if(pVulkanFunctions != VMA_NULL)
8722  {
8723  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
8724  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
8725  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
8726  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
8727  VMA_COPY_IF_NOT_NULL(vkMapMemory);
8728  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
8729  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
8730  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
8731  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
8732  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
8733  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
8734  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
8735  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
8736  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
8737  VMA_COPY_IF_NOT_NULL(vkCreateImage);
8738  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
8739 #if VMA_DEDICATED_ALLOCATION
8740  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
8741  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
8742 #endif
8743  }
8744 
8745 #undef VMA_COPY_IF_NOT_NULL
8746 
8747  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
8748  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
8749  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
8750  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
8751  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
8752  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
8753  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
8754  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
8755  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
8756  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
8757  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
8758  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
8759  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
8760  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
8761  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
8762  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
8763  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
8764  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
8765 #if VMA_DEDICATED_ALLOCATION
8766  if(m_UseKhrDedicatedAllocation)
8767  {
8768  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
8769  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
8770  }
8771 #endif
8772 }
8773 
8774 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
8775 {
8776  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
8777  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
8778  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
8779  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
8780 }
8781 
8782 VkResult VmaAllocator_T::AllocateMemoryOfType(
8783  VkDeviceSize size,
8784  VkDeviceSize alignment,
8785  bool dedicatedAllocation,
8786  VkBuffer dedicatedBuffer,
8787  VkImage dedicatedImage,
8788  const VmaAllocationCreateInfo& createInfo,
8789  uint32_t memTypeIndex,
8790  VmaSuballocationType suballocType,
8791  VmaAllocation* pAllocation)
8792 {
8793  VMA_ASSERT(pAllocation != VMA_NULL);
8794  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
8795 
8796  VmaAllocationCreateInfo finalCreateInfo = createInfo;
8797 
8798  // If memory type is not HOST_VISIBLE, disable MAPPED.
8799  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
8800  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
8801  {
8802  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
8803  }
8804 
8805  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
8806  VMA_ASSERT(blockVector);
8807 
8808  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
8809  bool preferDedicatedMemory =
8810  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
8811  dedicatedAllocation ||
8812  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
8813  size > preferredBlockSize / 2;
8814 
8815  if(preferDedicatedMemory &&
8816  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
8817  finalCreateInfo.pool == VK_NULL_HANDLE)
8818  {
8819  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
8820  }
8821 
8822  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
8823  {
8824  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
8825  {
8826  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
8827  }
8828  else
8829  {
8830  return AllocateDedicatedMemory(
8831  size,
8832  suballocType,
8833  memTypeIndex,
8834  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
8835  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
8836  finalCreateInfo.pUserData,
8837  dedicatedBuffer,
8838  dedicatedImage,
8839  pAllocation);
8840  }
8841  }
8842  else
8843  {
8844  VkResult res = blockVector->Allocate(
8845  VK_NULL_HANDLE, // hCurrentPool
8846  m_CurrentFrameIndex.load(),
8847  size,
8848  alignment,
8849  finalCreateInfo,
8850  suballocType,
8851  pAllocation);
8852  if(res == VK_SUCCESS)
8853  {
8854  return res;
8855  }
8856 
8857  // 5. Try dedicated memory.
8858  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
8859  {
8860  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
8861  }
8862  else
8863  {
8864  res = AllocateDedicatedMemory(
8865  size,
8866  suballocType,
8867  memTypeIndex,
8868  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
8869  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
8870  finalCreateInfo.pUserData,
8871  dedicatedBuffer,
8872  dedicatedImage,
8873  pAllocation);
8874  if(res == VK_SUCCESS)
8875  {
8876  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
8877  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
8878  return VK_SUCCESS;
8879  }
8880  else
8881  {
8882  // Everything failed: Return error code.
8883  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
8884  return res;
8885  }
8886  }
8887  }
8888 }
8889 
8890 VkResult VmaAllocator_T::AllocateDedicatedMemory(
8891  VkDeviceSize size,
8892  VmaSuballocationType suballocType,
8893  uint32_t memTypeIndex,
8894  bool map,
8895  bool isUserDataString,
8896  void* pUserData,
8897  VkBuffer dedicatedBuffer,
8898  VkImage dedicatedImage,
8899  VmaAllocation* pAllocation)
8900 {
8901  VMA_ASSERT(pAllocation);
8902 
8903  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
8904  allocInfo.memoryTypeIndex = memTypeIndex;
8905  allocInfo.allocationSize = size;
8906 
8907 #if VMA_DEDICATED_ALLOCATION
8908  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
8909  if(m_UseKhrDedicatedAllocation)
8910  {
8911  if(dedicatedBuffer != VK_NULL_HANDLE)
8912  {
8913  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
8914  dedicatedAllocInfo.buffer = dedicatedBuffer;
8915  allocInfo.pNext = &dedicatedAllocInfo;
8916  }
8917  else if(dedicatedImage != VK_NULL_HANDLE)
8918  {
8919  dedicatedAllocInfo.image = dedicatedImage;
8920  allocInfo.pNext = &dedicatedAllocInfo;
8921  }
8922  }
8923 #endif // #if VMA_DEDICATED_ALLOCATION
8924 
8925  // Allocate VkDeviceMemory.
8926  VkDeviceMemory hMemory = VK_NULL_HANDLE;
8927  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
8928  if(res < 0)
8929  {
8930  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
8931  return res;
8932  }
8933 
8934  void* pMappedData = VMA_NULL;
8935  if(map)
8936  {
8937  res = (*m_VulkanFunctions.vkMapMemory)(
8938  m_hDevice,
8939  hMemory,
8940  0,
8941  VK_WHOLE_SIZE,
8942  0,
8943  &pMappedData);
8944  if(res < 0)
8945  {
8946  VMA_DEBUG_LOG(" vkMapMemory FAILED");
8947  FreeVulkanMemory(memTypeIndex, size, hMemory);
8948  return res;
8949  }
8950  }
8951 
8952  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
8953  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
8954  (*pAllocation)->SetUserData(this, pUserData);
8955  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
8956  {
8957  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
8958  }
8959 
8960  // Register it in m_pDedicatedAllocations.
8961  {
8962  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8963  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
8964  VMA_ASSERT(pDedicatedAllocations);
8965  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
8966  }
8967 
8968  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
8969 
8970  return VK_SUCCESS;
8971 }
8972 
8973 void VmaAllocator_T::GetBufferMemoryRequirements(
8974  VkBuffer hBuffer,
8975  VkMemoryRequirements& memReq,
8976  bool& requiresDedicatedAllocation,
8977  bool& prefersDedicatedAllocation) const
8978 {
8979 #if VMA_DEDICATED_ALLOCATION
8980  if(m_UseKhrDedicatedAllocation)
8981  {
8982  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
8983  memReqInfo.buffer = hBuffer;
8984 
8985  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
8986 
8987  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
8988  memReq2.pNext = &memDedicatedReq;
8989 
8990  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
8991 
8992  memReq = memReq2.memoryRequirements;
8993  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
8994  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
8995  }
8996  else
8997 #endif // #if VMA_DEDICATED_ALLOCATION
8998  {
8999  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
9000  requiresDedicatedAllocation = false;
9001  prefersDedicatedAllocation = false;
9002  }
9003 }
9004 
9005 void VmaAllocator_T::GetImageMemoryRequirements(
9006  VkImage hImage,
9007  VkMemoryRequirements& memReq,
9008  bool& requiresDedicatedAllocation,
9009  bool& prefersDedicatedAllocation) const
9010 {
9011 #if VMA_DEDICATED_ALLOCATION
9012  if(m_UseKhrDedicatedAllocation)
9013  {
9014  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
9015  memReqInfo.image = hImage;
9016 
9017  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
9018 
9019  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
9020  memReq2.pNext = &memDedicatedReq;
9021 
9022  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
9023 
9024  memReq = memReq2.memoryRequirements;
9025  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
9026  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
9027  }
9028  else
9029 #endif // #if VMA_DEDICATED_ALLOCATION
9030  {
9031  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
9032  requiresDedicatedAllocation = false;
9033  prefersDedicatedAllocation = false;
9034  }
9035 }
9036 
9037 VkResult VmaAllocator_T::AllocateMemory(
9038  const VkMemoryRequirements& vkMemReq,
9039  bool requiresDedicatedAllocation,
9040  bool prefersDedicatedAllocation,
9041  VkBuffer dedicatedBuffer,
9042  VkImage dedicatedImage,
9043  const VmaAllocationCreateInfo& createInfo,
9044  VmaSuballocationType suballocType,
9045  VmaAllocation* pAllocation)
9046 {
9047  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
9048  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
9049  {
9050  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
9051  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9052  }
9053  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
9054  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
9055  {
9056  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
9057  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9058  }
9059  if(requiresDedicatedAllocation)
9060  {
9061  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
9062  {
9063  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
9064  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9065  }
9066  if(createInfo.pool != VK_NULL_HANDLE)
9067  {
9068  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
9069  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9070  }
9071  }
9072  if((createInfo.pool != VK_NULL_HANDLE) &&
9073  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
9074  {
9075  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
9076  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9077  }
9078 
9079  if(createInfo.pool != VK_NULL_HANDLE)
9080  {
9081  const VkDeviceSize alignmentForPool = VMA_MAX(
9082  vkMemReq.alignment,
9083  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
9084  return createInfo.pool->m_BlockVector.Allocate(
9085  createInfo.pool,
9086  m_CurrentFrameIndex.load(),
9087  vkMemReq.size,
9088  alignmentForPool,
9089  createInfo,
9090  suballocType,
9091  pAllocation);
9092  }
9093  else
9094  {
9095  // Bit mask of memory Vulkan types acceptable for this allocation.
9096  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
9097  uint32_t memTypeIndex = UINT32_MAX;
9098  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
9099  if(res == VK_SUCCESS)
9100  {
9101  VkDeviceSize alignmentForMemType = VMA_MAX(
9102  vkMemReq.alignment,
9103  GetMemoryTypeMinAlignment(memTypeIndex));
9104 
9105  res = AllocateMemoryOfType(
9106  vkMemReq.size,
9107  alignmentForMemType,
9108  requiresDedicatedAllocation || prefersDedicatedAllocation,
9109  dedicatedBuffer,
9110  dedicatedImage,
9111  createInfo,
9112  memTypeIndex,
9113  suballocType,
9114  pAllocation);
9115  // Succeeded on first try.
9116  if(res == VK_SUCCESS)
9117  {
9118  return res;
9119  }
9120  // Allocation from this memory type failed. Try other compatible memory types.
9121  else
9122  {
9123  for(;;)
9124  {
9125  // Remove old memTypeIndex from list of possibilities.
9126  memoryTypeBits &= ~(1u << memTypeIndex);
9127  // Find alternative memTypeIndex.
9128  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
9129  if(res == VK_SUCCESS)
9130  {
9131  alignmentForMemType = VMA_MAX(
9132  vkMemReq.alignment,
9133  GetMemoryTypeMinAlignment(memTypeIndex));
9134 
9135  res = AllocateMemoryOfType(
9136  vkMemReq.size,
9137  alignmentForMemType,
9138  requiresDedicatedAllocation || prefersDedicatedAllocation,
9139  dedicatedBuffer,
9140  dedicatedImage,
9141  createInfo,
9142  memTypeIndex,
9143  suballocType,
9144  pAllocation);
9145  // Allocation from this alternative memory type succeeded.
9146  if(res == VK_SUCCESS)
9147  {
9148  return res;
9149  }
9150  // else: Allocation from this memory type failed. Try next one - next loop iteration.
9151  }
9152  // No other matching memory type index could be found.
9153  else
9154  {
9155  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
9156  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9157  }
9158  }
9159  }
9160  }
9161  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
9162  else
9163  return res;
9164  }
9165 }
9166 
9167 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
9168 {
9169  VMA_ASSERT(allocation);
9170 
9171  if(allocation->CanBecomeLost() == false ||
9172  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
9173  {
9174  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9175  {
9176  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
9177  }
9178 
9179  switch(allocation->GetType())
9180  {
9181  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9182  {
9183  VmaBlockVector* pBlockVector = VMA_NULL;
9184  VmaPool hPool = allocation->GetPool();
9185  if(hPool != VK_NULL_HANDLE)
9186  {
9187  pBlockVector = &hPool->m_BlockVector;
9188  }
9189  else
9190  {
9191  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
9192  pBlockVector = m_pBlockVectors[memTypeIndex];
9193  }
9194  pBlockVector->Free(allocation);
9195  }
9196  break;
9197  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9198  FreeDedicatedMemory(allocation);
9199  break;
9200  default:
9201  VMA_ASSERT(0);
9202  }
9203  }
9204 
9205  allocation->SetUserData(this, VMA_NULL);
9206  vma_delete(this, allocation);
9207 }
9208 
9209 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
9210 {
9211  // Initialize.
9212  InitStatInfo(pStats->total);
9213  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
9214  InitStatInfo(pStats->memoryType[i]);
9215  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
9216  InitStatInfo(pStats->memoryHeap[i]);
9217 
9218  // Process default pools.
9219  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9220  {
9221  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
9222  VMA_ASSERT(pBlockVector);
9223  pBlockVector->AddStats(pStats);
9224  }
9225 
9226  // Process custom pools.
9227  {
9228  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9229  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
9230  {
9231  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
9232  }
9233  }
9234 
9235  // Process dedicated allocations.
9236  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9237  {
9238  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
9239  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9240  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
9241  VMA_ASSERT(pDedicatedAllocVector);
9242  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
9243  {
9244  VmaStatInfo allocationStatInfo;
9245  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
9246  VmaAddStatInfo(pStats->total, allocationStatInfo);
9247  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
9248  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
9249  }
9250  }
9251 
9252  // Postprocess.
9253  VmaPostprocessCalcStatInfo(pStats->total);
9254  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
9255  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
9256  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
9257  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
9258 }
9259 
9260 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
9261 
9262 VkResult VmaAllocator_T::Defragment(
9263  VmaAllocation* pAllocations,
9264  size_t allocationCount,
9265  VkBool32* pAllocationsChanged,
9266  const VmaDefragmentationInfo* pDefragmentationInfo,
9267  VmaDefragmentationStats* pDefragmentationStats)
9268 {
9269  if(pAllocationsChanged != VMA_NULL)
9270  {
9271  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
9272  }
9273  if(pDefragmentationStats != VMA_NULL)
9274  {
9275  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
9276  }
9277 
9278  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
9279 
9280  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
9281 
9282  const size_t poolCount = m_Pools.size();
9283 
9284  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
9285  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
9286  {
9287  VmaAllocation hAlloc = pAllocations[allocIndex];
9288  VMA_ASSERT(hAlloc);
9289  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
9290  // DedicatedAlloc cannot be defragmented.
9291  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
9292  // Only HOST_VISIBLE memory types can be defragmented.
9293  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
9294  // Lost allocation cannot be defragmented.
9295  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
9296  {
9297  VmaBlockVector* pAllocBlockVector = VMA_NULL;
9298 
9299  const VmaPool hAllocPool = hAlloc->GetPool();
9300  // This allocation belongs to custom pool.
9301  if(hAllocPool != VK_NULL_HANDLE)
9302  {
9303  pAllocBlockVector = &hAllocPool->GetBlockVector();
9304  }
9305  // This allocation belongs to general pool.
9306  else
9307  {
9308  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
9309  }
9310 
9311  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
9312 
9313  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
9314  &pAllocationsChanged[allocIndex] : VMA_NULL;
9315  pDefragmentator->AddAllocation(hAlloc, pChanged);
9316  }
9317  }
9318 
9319  VkResult result = VK_SUCCESS;
9320 
9321  // ======== Main processing.
9322 
9323  VkDeviceSize maxBytesToMove = SIZE_MAX;
9324  uint32_t maxAllocationsToMove = UINT32_MAX;
9325  if(pDefragmentationInfo != VMA_NULL)
9326  {
9327  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
9328  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
9329  }
9330 
9331  // Process standard memory.
9332  for(uint32_t memTypeIndex = 0;
9333  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
9334  ++memTypeIndex)
9335  {
9336  // Only HOST_VISIBLE memory types can be defragmented.
9337  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9338  {
9339  result = m_pBlockVectors[memTypeIndex]->Defragment(
9340  pDefragmentationStats,
9341  maxBytesToMove,
9342  maxAllocationsToMove);
9343  }
9344  }
9345 
9346  // Process custom pools.
9347  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
9348  {
9349  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
9350  pDefragmentationStats,
9351  maxBytesToMove,
9352  maxAllocationsToMove);
9353  }
9354 
9355  // ======== Destroy defragmentators.
9356 
9357  // Process custom pools.
9358  for(size_t poolIndex = poolCount; poolIndex--; )
9359  {
9360  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
9361  }
9362 
9363  // Process standard memory.
9364  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
9365  {
9366  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9367  {
9368  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
9369  }
9370  }
9371 
9372  return result;
9373 }
9374 
9375 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
9376 {
9377  if(hAllocation->CanBecomeLost())
9378  {
9379  /*
9380  Warning: This is a carefully designed algorithm.
9381  Do not modify unless you really know what you're doing :)
9382  */
9383  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9384  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9385  for(;;)
9386  {
9387  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
9388  {
9389  pAllocationInfo->memoryType = UINT32_MAX;
9390  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
9391  pAllocationInfo->offset = 0;
9392  pAllocationInfo->size = hAllocation->GetSize();
9393  pAllocationInfo->pMappedData = VMA_NULL;
9394  pAllocationInfo->pUserData = hAllocation->GetUserData();
9395  return;
9396  }
9397  else if(localLastUseFrameIndex == localCurrFrameIndex)
9398  {
9399  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
9400  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
9401  pAllocationInfo->offset = hAllocation->GetOffset();
9402  pAllocationInfo->size = hAllocation->GetSize();
9403  pAllocationInfo->pMappedData = VMA_NULL;
9404  pAllocationInfo->pUserData = hAllocation->GetUserData();
9405  return;
9406  }
9407  else // Last use time earlier than current time.
9408  {
9409  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9410  {
9411  localLastUseFrameIndex = localCurrFrameIndex;
9412  }
9413  }
9414  }
9415  }
9416  else
9417  {
9418 #if VMA_STATS_STRING_ENABLED
9419  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9420  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9421  for(;;)
9422  {
9423  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
9424  if(localLastUseFrameIndex == localCurrFrameIndex)
9425  {
9426  break;
9427  }
9428  else // Last use time earlier than current time.
9429  {
9430  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9431  {
9432  localLastUseFrameIndex = localCurrFrameIndex;
9433  }
9434  }
9435  }
9436 #endif
9437 
9438  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
9439  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
9440  pAllocationInfo->offset = hAllocation->GetOffset();
9441  pAllocationInfo->size = hAllocation->GetSize();
9442  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
9443  pAllocationInfo->pUserData = hAllocation->GetUserData();
9444  }
9445 }
9446 
9447 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
9448 {
9449  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
9450  if(hAllocation->CanBecomeLost())
9451  {
9452  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9453  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9454  for(;;)
9455  {
9456  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
9457  {
9458  return false;
9459  }
9460  else if(localLastUseFrameIndex == localCurrFrameIndex)
9461  {
9462  return true;
9463  }
9464  else // Last use time earlier than current time.
9465  {
9466  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9467  {
9468  localLastUseFrameIndex = localCurrFrameIndex;
9469  }
9470  }
9471  }
9472  }
9473  else
9474  {
9475 #if VMA_STATS_STRING_ENABLED
9476  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9477  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9478  for(;;)
9479  {
9480  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
9481  if(localLastUseFrameIndex == localCurrFrameIndex)
9482  {
9483  break;
9484  }
9485  else // Last use time earlier than current time.
9486  {
9487  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9488  {
9489  localLastUseFrameIndex = localCurrFrameIndex;
9490  }
9491  }
9492  }
9493 #endif
9494 
9495  return true;
9496  }
9497 }
9498 
9499 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
9500 {
9501  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
9502 
9503  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
9504 
9505  if(newCreateInfo.maxBlockCount == 0)
9506  {
9507  newCreateInfo.maxBlockCount = SIZE_MAX;
9508  }
9509  if(newCreateInfo.blockSize == 0)
9510  {
9511  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
9512  }
9513 
9514  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
9515 
9516  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
9517  if(res != VK_SUCCESS)
9518  {
9519  vma_delete(this, *pPool);
9520  *pPool = VMA_NULL;
9521  return res;
9522  }
9523 
9524  // Add to m_Pools.
9525  {
9526  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9527  (*pPool)->SetId(m_NextPoolId++);
9528  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
9529  }
9530 
9531  return VK_SUCCESS;
9532 }
9533 
9534 void VmaAllocator_T::DestroyPool(VmaPool pool)
9535 {
9536  // Remove from m_Pools.
9537  {
9538  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9539  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
9540  VMA_ASSERT(success && "Pool not found in Allocator.");
9541  }
9542 
9543  vma_delete(this, pool);
9544 }
9545 
9546 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
9547 {
9548  pool->m_BlockVector.GetPoolStats(pPoolStats);
9549 }
9550 
9551 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
9552 {
9553  m_CurrentFrameIndex.store(frameIndex);
9554 }
9555 
9556 void VmaAllocator_T::MakePoolAllocationsLost(
9557  VmaPool hPool,
9558  size_t* pLostAllocationCount)
9559 {
9560  hPool->m_BlockVector.MakePoolAllocationsLost(
9561  m_CurrentFrameIndex.load(),
9562  pLostAllocationCount);
9563 }
9564 
9565 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
9566 {
9567  return hPool->m_BlockVector.CheckCorruption();
9568 }
9569 
9570 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
9571 {
9572  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
9573 
9574  // Process default pools.
9575  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9576  {
9577  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
9578  {
9579  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
9580  VMA_ASSERT(pBlockVector);
9581  VkResult localRes = pBlockVector->CheckCorruption();
9582  switch(localRes)
9583  {
9584  case VK_ERROR_FEATURE_NOT_PRESENT:
9585  break;
9586  case VK_SUCCESS:
9587  finalRes = VK_SUCCESS;
9588  break;
9589  default:
9590  return localRes;
9591  }
9592  }
9593  }
9594 
9595  // Process custom pools.
9596  {
9597  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9598  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
9599  {
9600  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
9601  {
9602  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
9603  switch(localRes)
9604  {
9605  case VK_ERROR_FEATURE_NOT_PRESENT:
9606  break;
9607  case VK_SUCCESS:
9608  finalRes = VK_SUCCESS;
9609  break;
9610  default:
9611  return localRes;
9612  }
9613  }
9614  }
9615  }
9616 
9617  return finalRes;
9618 }
9619 
9620 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
9621 {
9622  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
9623  (*pAllocation)->InitLost();
9624 }
9625 
9626 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
9627 {
9628  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
9629 
9630  VkResult res;
9631  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
9632  {
9633  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
9634  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
9635  {
9636  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
9637  if(res == VK_SUCCESS)
9638  {
9639  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
9640  }
9641  }
9642  else
9643  {
9644  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
9645  }
9646  }
9647  else
9648  {
9649  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
9650  }
9651 
9652  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
9653  {
9654  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
9655  }
9656 
9657  return res;
9658 }
9659 
9660 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
9661 {
9662  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
9663  {
9664  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
9665  }
9666 
9667  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
9668 
9669  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
9670  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
9671  {
9672  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
9673  m_HeapSizeLimit[heapIndex] += size;
9674  }
9675 }
9676 
9677 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
9678 {
9679  if(hAllocation->CanBecomeLost())
9680  {
9681  return VK_ERROR_MEMORY_MAP_FAILED;
9682  }
9683 
9684  switch(hAllocation->GetType())
9685  {
9686  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9687  {
9688  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
9689  char *pBytes = VMA_NULL;
9690  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
9691  if(res == VK_SUCCESS)
9692  {
9693  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
9694  hAllocation->BlockAllocMap();
9695  }
9696  return res;
9697  }
9698  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9699  return hAllocation->DedicatedAllocMap(this, ppData);
9700  default:
9701  VMA_ASSERT(0);
9702  return VK_ERROR_MEMORY_MAP_FAILED;
9703  }
9704 }
9705 
9706 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
9707 {
9708  switch(hAllocation->GetType())
9709  {
9710  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9711  {
9712  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
9713  hAllocation->BlockAllocUnmap();
9714  pBlock->Unmap(this, 1);
9715  }
9716  break;
9717  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9718  hAllocation->DedicatedAllocUnmap(this);
9719  break;
9720  default:
9721  VMA_ASSERT(0);
9722  }
9723 }
9724 
9725 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
9726 {
9727  VkResult res = VK_SUCCESS;
9728  switch(hAllocation->GetType())
9729  {
9730  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9731  res = GetVulkanFunctions().vkBindBufferMemory(
9732  m_hDevice,
9733  hBuffer,
9734  hAllocation->GetMemory(),
9735  0); //memoryOffset
9736  break;
9737  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9738  {
9739  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9740  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
9741  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
9742  break;
9743  }
9744  default:
9745  VMA_ASSERT(0);
9746  }
9747  return res;
9748 }
9749 
9750 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
9751 {
9752  VkResult res = VK_SUCCESS;
9753  switch(hAllocation->GetType())
9754  {
9755  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9756  res = GetVulkanFunctions().vkBindImageMemory(
9757  m_hDevice,
9758  hImage,
9759  hAllocation->GetMemory(),
9760  0); //memoryOffset
9761  break;
9762  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9763  {
9764  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9765  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
9766  res = pBlock->BindImageMemory(this, hAllocation, hImage);
9767  break;
9768  }
9769  default:
9770  VMA_ASSERT(0);
9771  }
9772  return res;
9773 }
9774 
9775 void VmaAllocator_T::FlushOrInvalidateAllocation(
9776  VmaAllocation hAllocation,
9777  VkDeviceSize offset, VkDeviceSize size,
9778  VMA_CACHE_OPERATION op)
9779 {
9780  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
9781  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
9782  {
9783  const VkDeviceSize allocationSize = hAllocation->GetSize();
9784  VMA_ASSERT(offset <= allocationSize);
9785 
9786  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
9787 
9788  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
9789  memRange.memory = hAllocation->GetMemory();
9790 
9791  switch(hAllocation->GetType())
9792  {
9793  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9794  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
9795  if(size == VK_WHOLE_SIZE)
9796  {
9797  memRange.size = allocationSize - memRange.offset;
9798  }
9799  else
9800  {
9801  VMA_ASSERT(offset + size <= allocationSize);
9802  memRange.size = VMA_MIN(
9803  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
9804  allocationSize - memRange.offset);
9805  }
9806  break;
9807 
9808  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9809  {
9810  // 1. Still within this allocation.
9811  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
9812  if(size == VK_WHOLE_SIZE)
9813  {
9814  size = allocationSize - offset;
9815  }
9816  else
9817  {
9818  VMA_ASSERT(offset + size <= allocationSize);
9819  }
9820  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
9821 
9822  // 2. Adjust to whole block.
9823  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
9824  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
9825  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
9826  memRange.offset += allocationOffset;
9827  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
9828 
9829  break;
9830  }
9831 
9832  default:
9833  VMA_ASSERT(0);
9834  }
9835 
9836  switch(op)
9837  {
9838  case VMA_CACHE_FLUSH:
9839  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
9840  break;
9841  case VMA_CACHE_INVALIDATE:
9842  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
9843  break;
9844  default:
9845  VMA_ASSERT(0);
9846  }
9847  }
9848  // else: Just ignore this call.
9849 }
9850 
9851 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
9852 {
9853  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
9854 
9855  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
9856  {
9857  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9858  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
9859  VMA_ASSERT(pDedicatedAllocations);
9860  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
9861  VMA_ASSERT(success);
9862  }
9863 
9864  VkDeviceMemory hMemory = allocation->GetMemory();
9865 
9866  if(allocation->GetMappedData() != VMA_NULL)
9867  {
9868  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
9869  }
9870 
9871  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
9872 
9873  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
9874 }
9875 
9876 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
9877 {
9878  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
9879  !hAllocation->CanBecomeLost() &&
9880  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9881  {
9882  void* pData = VMA_NULL;
9883  VkResult res = Map(hAllocation, &pData);
9884  if(res == VK_SUCCESS)
9885  {
9886  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
9887  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
9888  Unmap(hAllocation);
9889  }
9890  else
9891  {
9892  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
9893  }
9894  }
9895 }
9896 
9897 #if VMA_STATS_STRING_ENABLED
9898 
9899 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
9900 {
9901  bool dedicatedAllocationsStarted = false;
9902  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9903  {
9904  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9905  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
9906  VMA_ASSERT(pDedicatedAllocVector);
9907  if(pDedicatedAllocVector->empty() == false)
9908  {
9909  if(dedicatedAllocationsStarted == false)
9910  {
9911  dedicatedAllocationsStarted = true;
9912  json.WriteString("DedicatedAllocations");
9913  json.BeginObject();
9914  }
9915 
9916  json.BeginString("Type ");
9917  json.ContinueString(memTypeIndex);
9918  json.EndString();
9919 
9920  json.BeginArray();
9921 
9922  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
9923  {
9924  json.BeginObject(true);
9925  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
9926  hAlloc->PrintParameters(json);
9927  json.EndObject();
9928  }
9929 
9930  json.EndArray();
9931  }
9932  }
9933  if(dedicatedAllocationsStarted)
9934  {
9935  json.EndObject();
9936  }
9937 
9938  {
9939  bool allocationsStarted = false;
9940  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9941  {
9942  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
9943  {
9944  if(allocationsStarted == false)
9945  {
9946  allocationsStarted = true;
9947  json.WriteString("DefaultPools");
9948  json.BeginObject();
9949  }
9950 
9951  json.BeginString("Type ");
9952  json.ContinueString(memTypeIndex);
9953  json.EndString();
9954 
9955  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
9956  }
9957  }
9958  if(allocationsStarted)
9959  {
9960  json.EndObject();
9961  }
9962  }
9963 
9964  {
9965  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9966  const size_t poolCount = m_Pools.size();
9967  if(poolCount > 0)
9968  {
9969  json.WriteString("Pools");
9970  json.BeginObject();
9971  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
9972  {
9973  json.BeginString();
9974  json.ContinueString(m_Pools[poolIndex]->GetId());
9975  json.EndString();
9976 
9977  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
9978  }
9979  json.EndObject();
9980  }
9981  }
9982 }
9983 
9984 #endif // #if VMA_STATS_STRING_ENABLED
9985 
9987 // Public interface
9988 
9989 VkResult vmaCreateAllocator(
9990  const VmaAllocatorCreateInfo* pCreateInfo,
9991  VmaAllocator* pAllocator)
9992 {
9993  VMA_ASSERT(pCreateInfo && pAllocator);
9994  VMA_DEBUG_LOG("vmaCreateAllocator");
9995  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
9996  return (*pAllocator)->Init(pCreateInfo);
9997 }
9998 
9999 void vmaDestroyAllocator(
10000  VmaAllocator allocator)
10001 {
10002  if(allocator != VK_NULL_HANDLE)
10003  {
10004  VMA_DEBUG_LOG("vmaDestroyAllocator");
10005  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
10006  vma_delete(&allocationCallbacks, allocator);
10007  }
10008 }
10009 
10010 void vmaGetPhysicalDeviceProperties(
10011  VmaAllocator allocator,
10012  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
10013 {
10014  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
10015  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
10016 }
10017 
10018 void vmaGetMemoryProperties(
10019  VmaAllocator allocator,
10020  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
10021 {
10022  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
10023  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
10024 }
10025 
10026 void vmaGetMemoryTypeProperties(
10027  VmaAllocator allocator,
10028  uint32_t memoryTypeIndex,
10029  VkMemoryPropertyFlags* pFlags)
10030 {
10031  VMA_ASSERT(allocator && pFlags);
10032  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
10033  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
10034 }
10035 
10036 void vmaSetCurrentFrameIndex(
10037  VmaAllocator allocator,
10038  uint32_t frameIndex)
10039 {
10040  VMA_ASSERT(allocator);
10041  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
10042 
10043  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10044 
10045  allocator->SetCurrentFrameIndex(frameIndex);
10046 }
10047 
10048 void vmaCalculateStats(
10049  VmaAllocator allocator,
10050  VmaStats* pStats)
10051 {
10052  VMA_ASSERT(allocator && pStats);
10053  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10054  allocator->CalculateStats(pStats);
10055 }
10056 
10057 #if VMA_STATS_STRING_ENABLED
10058 
10059 void vmaBuildStatsString(
10060  VmaAllocator allocator,
10061  char** ppStatsString,
10062  VkBool32 detailedMap)
10063 {
10064  VMA_ASSERT(allocator && ppStatsString);
10065  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10066 
10067  VmaStringBuilder sb(allocator);
10068  {
10069  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
10070  json.BeginObject();
10071 
10072  VmaStats stats;
10073  allocator->CalculateStats(&stats);
10074 
10075  json.WriteString("Total");
10076  VmaPrintStatInfo(json, stats.total);
10077 
10078  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
10079  {
10080  json.BeginString("Heap ");
10081  json.ContinueString(heapIndex);
10082  json.EndString();
10083  json.BeginObject();
10084 
10085  json.WriteString("Size");
10086  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
10087 
10088  json.WriteString("Flags");
10089  json.BeginArray(true);
10090  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
10091  {
10092  json.WriteString("DEVICE_LOCAL");
10093  }
10094  json.EndArray();
10095 
10096  if(stats.memoryHeap[heapIndex].blockCount > 0)
10097  {
10098  json.WriteString("Stats");
10099  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
10100  }
10101 
10102  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
10103  {
10104  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
10105  {
10106  json.BeginString("Type ");
10107  json.ContinueString(typeIndex);
10108  json.EndString();
10109 
10110  json.BeginObject();
10111 
10112  json.WriteString("Flags");
10113  json.BeginArray(true);
10114  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
10115  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
10116  {
10117  json.WriteString("DEVICE_LOCAL");
10118  }
10119  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
10120  {
10121  json.WriteString("HOST_VISIBLE");
10122  }
10123  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
10124  {
10125  json.WriteString("HOST_COHERENT");
10126  }
10127  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
10128  {
10129  json.WriteString("HOST_CACHED");
10130  }
10131  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
10132  {
10133  json.WriteString("LAZILY_ALLOCATED");
10134  }
10135  json.EndArray();
10136 
10137  if(stats.memoryType[typeIndex].blockCount > 0)
10138  {
10139  json.WriteString("Stats");
10140  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
10141  }
10142 
10143  json.EndObject();
10144  }
10145  }
10146 
10147  json.EndObject();
10148  }
10149  if(detailedMap == VK_TRUE)
10150  {
10151  allocator->PrintDetailedMap(json);
10152  }
10153 
10154  json.EndObject();
10155  }
10156 
10157  const size_t len = sb.GetLength();
10158  char* const pChars = vma_new_array(allocator, char, len + 1);
10159  if(len > 0)
10160  {
10161  memcpy(pChars, sb.GetData(), len);
10162  }
10163  pChars[len] = '\0';
10164  *ppStatsString = pChars;
10165 }
10166 
10167 void vmaFreeStatsString(
10168  VmaAllocator allocator,
10169  char* pStatsString)
10170 {
10171  if(pStatsString != VMA_NULL)
10172  {
10173  VMA_ASSERT(allocator);
10174  size_t len = strlen(pStatsString);
10175  vma_delete_array(allocator, pStatsString, len + 1);
10176  }
10177 }
10178 
10179 #endif // #if VMA_STATS_STRING_ENABLED
10180 
10181 /*
10182 This function is not protected by any mutex because it just reads immutable data.
10183 */
10184 VkResult vmaFindMemoryTypeIndex(
10185  VmaAllocator allocator,
10186  uint32_t memoryTypeBits,
10187  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10188  uint32_t* pMemoryTypeIndex)
10189 {
10190  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10191  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10192  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10193 
10194  if(pAllocationCreateInfo->memoryTypeBits != 0)
10195  {
10196  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
10197  }
10198 
10199  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
10200  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
10201 
10202  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
10203  if(mapped)
10204  {
10205  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10206  }
10207 
10208  // Convert usage to requiredFlags and preferredFlags.
10209  switch(pAllocationCreateInfo->usage)
10210  {
10211  case VMA_MEMORY_USAGE_UNKNOWN:
10212  break;
10213  case VMA_MEMORY_USAGE_GPU_ONLY:
10214  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
10215  {
10216  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
10217  }
10218  break;
10219  case VMA_MEMORY_USAGE_CPU_ONLY:
10220  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
10221  break;
10222  case VMA_MEMORY_USAGE_CPU_TO_GPU:
10223  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10224  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
10225  {
10226  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
10227  }
10228  break;
10229  case VMA_MEMORY_USAGE_GPU_TO_CPU:
10230  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10231  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
10232  break;
10233  default:
10234  break;
10235  }
10236 
10237  *pMemoryTypeIndex = UINT32_MAX;
10238  uint32_t minCost = UINT32_MAX;
10239  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
10240  memTypeIndex < allocator->GetMemoryTypeCount();
10241  ++memTypeIndex, memTypeBit <<= 1)
10242  {
10243  // This memory type is acceptable according to memoryTypeBits bitmask.
10244  if((memTypeBit & memoryTypeBits) != 0)
10245  {
10246  const VkMemoryPropertyFlags currFlags =
10247  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
10248  // This memory type contains requiredFlags.
10249  if((requiredFlags & ~currFlags) == 0)
10250  {
10251  // Calculate cost as number of bits from preferredFlags not present in this memory type.
10252  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
10253  // Remember memory type with lowest cost.
10254  if(currCost < minCost)
10255  {
10256  *pMemoryTypeIndex = memTypeIndex;
10257  if(currCost == 0)
10258  {
10259  return VK_SUCCESS;
10260  }
10261  minCost = currCost;
10262  }
10263  }
10264  }
10265  }
10266  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
10267 }
10268 
10269 VkResult vmaFindMemoryTypeIndexForBufferInfo(
10270  VmaAllocator allocator,
10271  const VkBufferCreateInfo* pBufferCreateInfo,
10272  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10273  uint32_t* pMemoryTypeIndex)
10274 {
10275  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10276  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
10277  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10278  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10279 
10280  const VkDevice hDev = allocator->m_hDevice;
10281  VkBuffer hBuffer = VK_NULL_HANDLE;
10282  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
10283  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
10284  if(res == VK_SUCCESS)
10285  {
10286  VkMemoryRequirements memReq = {};
10287  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
10288  hDev, hBuffer, &memReq);
10289 
10290  res = vmaFindMemoryTypeIndex(
10291  allocator,
10292  memReq.memoryTypeBits,
10293  pAllocationCreateInfo,
10294  pMemoryTypeIndex);
10295 
10296  allocator->GetVulkanFunctions().vkDestroyBuffer(
10297  hDev, hBuffer, allocator->GetAllocationCallbacks());
10298  }
10299  return res;
10300 }
10301 
10302 VkResult vmaFindMemoryTypeIndexForImageInfo(
10303  VmaAllocator allocator,
10304  const VkImageCreateInfo* pImageCreateInfo,
10305  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10306  uint32_t* pMemoryTypeIndex)
10307 {
10308  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10309  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
10310  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10311  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10312 
10313  const VkDevice hDev = allocator->m_hDevice;
10314  VkImage hImage = VK_NULL_HANDLE;
10315  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
10316  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
10317  if(res == VK_SUCCESS)
10318  {
10319  VkMemoryRequirements memReq = {};
10320  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
10321  hDev, hImage, &memReq);
10322 
10323  res = vmaFindMemoryTypeIndex(
10324  allocator,
10325  memReq.memoryTypeBits,
10326  pAllocationCreateInfo,
10327  pMemoryTypeIndex);
10328 
10329  allocator->GetVulkanFunctions().vkDestroyImage(
10330  hDev, hImage, allocator->GetAllocationCallbacks());
10331  }
10332  return res;
10333 }
10334 
10335 VkResult vmaCreatePool(
10336  VmaAllocator allocator,
10337  const VmaPoolCreateInfo* pCreateInfo,
10338  VmaPool* pPool)
10339 {
10340  VMA_ASSERT(allocator && pCreateInfo && pPool);
10341 
10342  VMA_DEBUG_LOG("vmaCreatePool");
10343 
10344  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10345 
10346  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
10347 
10348 #if VMA_RECORDING_ENABLED
10349  if(allocator->GetRecorder() != VMA_NULL)
10350  {
10351  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
10352  }
10353 #endif
10354 
10355  return res;
10356 }
10357 
10358 void vmaDestroyPool(
10359  VmaAllocator allocator,
10360  VmaPool pool)
10361 {
10362  VMA_ASSERT(allocator);
10363 
10364  if(pool == VK_NULL_HANDLE)
10365  {
10366  return;
10367  }
10368 
10369  VMA_DEBUG_LOG("vmaDestroyPool");
10370 
10371  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10372 
10373 #if VMA_RECORDING_ENABLED
10374  if(allocator->GetRecorder() != VMA_NULL)
10375  {
10376  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
10377  }
10378 #endif
10379 
10380  allocator->DestroyPool(pool);
10381 }
10382 
10383 void vmaGetPoolStats(
10384  VmaAllocator allocator,
10385  VmaPool pool,
10386  VmaPoolStats* pPoolStats)
10387 {
10388  VMA_ASSERT(allocator && pool && pPoolStats);
10389 
10390  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10391 
10392  allocator->GetPoolStats(pool, pPoolStats);
10393 }
10394 
10395 void vmaMakePoolAllocationsLost(
10396  VmaAllocator allocator,
10397  VmaPool pool,
10398  size_t* pLostAllocationCount)
10399 {
10400  VMA_ASSERT(allocator && pool);
10401 
10402  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10403 
10404  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
10405 }
10406 
10407 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
10408 {
10409  VMA_ASSERT(allocator && pool);
10410 
10411  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10412 
10413  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
10414 
10415  return allocator->CheckPoolCorruption(pool);
10416 }
10417 
10418 VkResult vmaAllocateMemory(
10419  VmaAllocator allocator,
10420  const VkMemoryRequirements* pVkMemoryRequirements,
10421  const VmaAllocationCreateInfo* pCreateInfo,
10422  VmaAllocation* pAllocation,
10423  VmaAllocationInfo* pAllocationInfo)
10424 {
10425  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
10426 
10427  VMA_DEBUG_LOG("vmaAllocateMemory");
10428 
10429  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10430 
10431  VkResult result = allocator->AllocateMemory(
10432  *pVkMemoryRequirements,
10433  false, // requiresDedicatedAllocation
10434  false, // prefersDedicatedAllocation
10435  VK_NULL_HANDLE, // dedicatedBuffer
10436  VK_NULL_HANDLE, // dedicatedImage
10437  *pCreateInfo,
10438  VMA_SUBALLOCATION_TYPE_UNKNOWN,
10439  pAllocation);
10440 
10441 #if VMA_RECORDING_ENABLED
10442  if(allocator->GetRecorder() != VMA_NULL)
10443  {
10444  allocator->GetRecorder()->RecordAllocateMemory(
10445  allocator->GetCurrentFrameIndex(),
10446  *pVkMemoryRequirements,
10447  *pCreateInfo,
10448  *pAllocation);
10449  }
10450 #endif
10451 
10452  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
10453  {
10454  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10455  }
10456 
10457  return result;
10458 }
10459 
10460 VkResult vmaAllocateMemoryForBuffer(
10461  VmaAllocator allocator,
10462  VkBuffer buffer,
10463  const VmaAllocationCreateInfo* pCreateInfo,
10464  VmaAllocation* pAllocation,
10465  VmaAllocationInfo* pAllocationInfo)
10466 {
10467  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
10468 
10469  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
10470 
10471  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10472 
10473  VkMemoryRequirements vkMemReq = {};
10474  bool requiresDedicatedAllocation = false;
10475  bool prefersDedicatedAllocation = false;
10476  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
10477  requiresDedicatedAllocation,
10478  prefersDedicatedAllocation);
10479 
10480  VkResult result = allocator->AllocateMemory(
10481  vkMemReq,
10482  requiresDedicatedAllocation,
10483  prefersDedicatedAllocation,
10484  buffer, // dedicatedBuffer
10485  VK_NULL_HANDLE, // dedicatedImage
10486  *pCreateInfo,
10487  VMA_SUBALLOCATION_TYPE_BUFFER,
10488  pAllocation);
10489 
10490 #if VMA_RECORDING_ENABLED
10491  if(allocator->GetRecorder() != VMA_NULL)
10492  {
10493  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
10494  allocator->GetCurrentFrameIndex(),
10495  vkMemReq,
10496  requiresDedicatedAllocation,
10497  prefersDedicatedAllocation,
10498  *pCreateInfo,
10499  *pAllocation);
10500  }
10501 #endif
10502 
10503  if(pAllocationInfo && result == VK_SUCCESS)
10504  {
10505  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10506  }
10507 
10508  return result;
10509 }
10510 
10511 VkResult vmaAllocateMemoryForImage(
10512  VmaAllocator allocator,
10513  VkImage image,
10514  const VmaAllocationCreateInfo* pCreateInfo,
10515  VmaAllocation* pAllocation,
10516  VmaAllocationInfo* pAllocationInfo)
10517 {
10518  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
10519 
10520  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
10521 
10522  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10523 
10524  VkMemoryRequirements vkMemReq = {};
10525  bool requiresDedicatedAllocation = false;
10526  bool prefersDedicatedAllocation = false;
10527  allocator->GetImageMemoryRequirements(image, vkMemReq,
10528  requiresDedicatedAllocation, prefersDedicatedAllocation);
10529 
10530  VkResult result = allocator->AllocateMemory(
10531  vkMemReq,
10532  requiresDedicatedAllocation,
10533  prefersDedicatedAllocation,
10534  VK_NULL_HANDLE, // dedicatedBuffer
10535  image, // dedicatedImage
10536  *pCreateInfo,
10537  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
10538  pAllocation);
10539 
10540 #if VMA_RECORDING_ENABLED
10541  if(allocator->GetRecorder() != VMA_NULL)
10542  {
10543  allocator->GetRecorder()->RecordAllocateMemoryForImage(
10544  allocator->GetCurrentFrameIndex(),
10545  vkMemReq,
10546  requiresDedicatedAllocation,
10547  prefersDedicatedAllocation,
10548  *pCreateInfo,
10549  *pAllocation);
10550  }
10551 #endif
10552 
10553  if(pAllocationInfo && result == VK_SUCCESS)
10554  {
10555  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10556  }
10557 
10558  return result;
10559 }
10560 
10561 void vmaFreeMemory(
10562  VmaAllocator allocator,
10563  VmaAllocation allocation)
10564 {
10565  VMA_ASSERT(allocator);
10566 
10567  if(allocation == VK_NULL_HANDLE)
10568  {
10569  return;
10570  }
10571 
10572  VMA_DEBUG_LOG("vmaFreeMemory");
10573 
10574  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10575 
10576 #if VMA_RECORDING_ENABLED
10577  if(allocator->GetRecorder() != VMA_NULL)
10578  {
10579  allocator->GetRecorder()->RecordFreeMemory(
10580  allocator->GetCurrentFrameIndex(),
10581  allocation);
10582  }
10583 #endif
10584 
10585  allocator->FreeMemory(allocation);
10586 }
10587 
10588 void vmaGetAllocationInfo(
10589  VmaAllocator allocator,
10590  VmaAllocation allocation,
10591  VmaAllocationInfo* pAllocationInfo)
10592 {
10593  VMA_ASSERT(allocator && allocation && pAllocationInfo);
10594 
10595  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10596 
10597 #if VMA_RECORDING_ENABLED
10598  if(allocator->GetRecorder() != VMA_NULL)
10599  {
10600  allocator->GetRecorder()->RecordGetAllocationInfo(
10601  allocator->GetCurrentFrameIndex(),
10602  allocation);
10603  }
10604 #endif
10605 
10606  allocator->GetAllocationInfo(allocation, pAllocationInfo);
10607 }
10608 
10609 VkBool32 vmaTouchAllocation(
10610  VmaAllocator allocator,
10611  VmaAllocation allocation)
10612 {
10613  VMA_ASSERT(allocator && allocation);
10614 
10615  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10616 
10617 #if VMA_RECORDING_ENABLED
10618  if(allocator->GetRecorder() != VMA_NULL)
10619  {
10620  allocator->GetRecorder()->RecordTouchAllocation(
10621  allocator->GetCurrentFrameIndex(),
10622  allocation);
10623  }
10624 #endif
10625 
10626  return allocator->TouchAllocation(allocation);
10627 }
10628 
10629 void vmaSetAllocationUserData(
10630  VmaAllocator allocator,
10631  VmaAllocation allocation,
10632  void* pUserData)
10633 {
10634  VMA_ASSERT(allocator && allocation);
10635 
10636  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10637 
10638  allocation->SetUserData(allocator, pUserData);
10639 
10640 #if VMA_RECORDING_ENABLED
10641  if(allocator->GetRecorder() != VMA_NULL)
10642  {
10643  allocator->GetRecorder()->RecordSetAllocationUserData(
10644  allocator->GetCurrentFrameIndex(),
10645  allocation,
10646  pUserData);
10647  }
10648 #endif
10649 }
10650 
10651 void vmaCreateLostAllocation(
10652  VmaAllocator allocator,
10653  VmaAllocation* pAllocation)
10654 {
10655  VMA_ASSERT(allocator && pAllocation);
10656 
10657  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
10658 
10659  allocator->CreateLostAllocation(pAllocation);
10660 
10661 #if VMA_RECORDING_ENABLED
10662  if(allocator->GetRecorder() != VMA_NULL)
10663  {
10664  allocator->GetRecorder()->RecordCreateLostAllocation(
10665  allocator->GetCurrentFrameIndex(),
10666  *pAllocation);
10667  }
10668 #endif
10669 }
10670 
10671 VkResult vmaMapMemory(
10672  VmaAllocator allocator,
10673  VmaAllocation allocation,
10674  void** ppData)
10675 {
10676  VMA_ASSERT(allocator && allocation && ppData);
10677 
10678  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10679 
10680  VkResult res = allocator->Map(allocation, ppData);
10681 
10682 #if VMA_RECORDING_ENABLED
10683  if(allocator->GetRecorder() != VMA_NULL)
10684  {
10685  allocator->GetRecorder()->RecordMapMemory(
10686  allocator->GetCurrentFrameIndex(),
10687  allocation);
10688  }
10689 #endif
10690 
10691  return res;
10692 }
10693 
10694 void vmaUnmapMemory(
10695  VmaAllocator allocator,
10696  VmaAllocation allocation)
10697 {
10698  VMA_ASSERT(allocator && allocation);
10699 
10700  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10701 
10702 #if VMA_RECORDING_ENABLED
10703  if(allocator->GetRecorder() != VMA_NULL)
10704  {
10705  allocator->GetRecorder()->RecordUnmapMemory(
10706  allocator->GetCurrentFrameIndex(),
10707  allocation);
10708  }
10709 #endif
10710 
10711  allocator->Unmap(allocation);
10712 }
10713 
10714 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10715 {
10716  VMA_ASSERT(allocator && allocation);
10717 
10718  VMA_DEBUG_LOG("vmaFlushAllocation");
10719 
10720  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10721 
10722  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
10723 
10724 #if VMA_RECORDING_ENABLED
10725  if(allocator->GetRecorder() != VMA_NULL)
10726  {
10727  allocator->GetRecorder()->RecordFlushAllocation(
10728  allocator->GetCurrentFrameIndex(),
10729  allocation, offset, size);
10730  }
10731 #endif
10732 }
10733 
10734 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10735 {
10736  VMA_ASSERT(allocator && allocation);
10737 
10738  VMA_DEBUG_LOG("vmaInvalidateAllocation");
10739 
10740  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10741 
10742  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
10743 
10744 #if VMA_RECORDING_ENABLED
10745  if(allocator->GetRecorder() != VMA_NULL)
10746  {
10747  allocator->GetRecorder()->RecordInvalidateAllocation(
10748  allocator->GetCurrentFrameIndex(),
10749  allocation, offset, size);
10750  }
10751 #endif
10752 }
10753 
10754 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
10755 {
10756  VMA_ASSERT(allocator);
10757 
10758  VMA_DEBUG_LOG("vmaCheckCorruption");
10759 
10760  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10761 
10762  return allocator->CheckCorruption(memoryTypeBits);
10763 }
10764 
10765 VkResult vmaDefragment(
10766  VmaAllocator allocator,
10767  VmaAllocation* pAllocations,
10768  size_t allocationCount,
10769  VkBool32* pAllocationsChanged,
10770  const VmaDefragmentationInfo *pDefragmentationInfo,
10771  VmaDefragmentationStats* pDefragmentationStats)
10772 {
10773  VMA_ASSERT(allocator && pAllocations);
10774 
10775  VMA_DEBUG_LOG("vmaDefragment");
10776 
10777  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10778 
10779  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
10780 }
10781 
10782 VkResult vmaBindBufferMemory(
10783  VmaAllocator allocator,
10784  VmaAllocation allocation,
10785  VkBuffer buffer)
10786 {
10787  VMA_ASSERT(allocator && allocation && buffer);
10788 
10789  VMA_DEBUG_LOG("vmaBindBufferMemory");
10790 
10791  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10792 
10793  return allocator->BindBufferMemory(allocation, buffer);
10794 }
10795 
10796 VkResult vmaBindImageMemory(
10797  VmaAllocator allocator,
10798  VmaAllocation allocation,
10799  VkImage image)
10800 {
10801  VMA_ASSERT(allocator && allocation && image);
10802 
10803  VMA_DEBUG_LOG("vmaBindImageMemory");
10804 
10805  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10806 
10807  return allocator->BindImageMemory(allocation, image);
10808 }
10809 
10810 VkResult vmaCreateBuffer(
10811  VmaAllocator allocator,
10812  const VkBufferCreateInfo* pBufferCreateInfo,
10813  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10814  VkBuffer* pBuffer,
10815  VmaAllocation* pAllocation,
10816  VmaAllocationInfo* pAllocationInfo)
10817 {
10818  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
10819 
10820  VMA_DEBUG_LOG("vmaCreateBuffer");
10821 
10822  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10823 
10824  *pBuffer = VK_NULL_HANDLE;
10825  *pAllocation = VK_NULL_HANDLE;
10826 
10827  // 1. Create VkBuffer.
10828  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
10829  allocator->m_hDevice,
10830  pBufferCreateInfo,
10831  allocator->GetAllocationCallbacks(),
10832  pBuffer);
10833  if(res >= 0)
10834  {
10835  // 2. vkGetBufferMemoryRequirements.
10836  VkMemoryRequirements vkMemReq = {};
10837  bool requiresDedicatedAllocation = false;
10838  bool prefersDedicatedAllocation = false;
10839  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
10840  requiresDedicatedAllocation, prefersDedicatedAllocation);
10841 
10842  // Make sure alignment requirements for specific buffer usages reported
10843  // in Physical Device Properties are included in alignment reported by memory requirements.
10844  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
10845  {
10846  VMA_ASSERT(vkMemReq.alignment %
10847  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
10848  }
10849  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
10850  {
10851  VMA_ASSERT(vkMemReq.alignment %
10852  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
10853  }
10854  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
10855  {
10856  VMA_ASSERT(vkMemReq.alignment %
10857  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
10858  }
10859 
10860  // 3. Allocate memory using allocator.
10861  res = allocator->AllocateMemory(
10862  vkMemReq,
10863  requiresDedicatedAllocation,
10864  prefersDedicatedAllocation,
10865  *pBuffer, // dedicatedBuffer
10866  VK_NULL_HANDLE, // dedicatedImage
10867  *pAllocationCreateInfo,
10868  VMA_SUBALLOCATION_TYPE_BUFFER,
10869  pAllocation);
10870 
10871 #if VMA_RECORDING_ENABLED
10872  if(allocator->GetRecorder() != VMA_NULL)
10873  {
10874  allocator->GetRecorder()->RecordCreateBuffer(
10875  allocator->GetCurrentFrameIndex(),
10876  *pBufferCreateInfo,
10877  *pAllocationCreateInfo,
10878  *pAllocation);
10879  }
10880 #endif
10881 
10882  if(res >= 0)
10883  {
10884  // 3. Bind buffer with memory.
10885  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
10886  if(res >= 0)
10887  {
10888  // All steps succeeded.
10889  #if VMA_STATS_STRING_ENABLED
10890  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
10891  #endif
10892  if(pAllocationInfo != VMA_NULL)
10893  {
10894  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10895  }
10896 
10897  return VK_SUCCESS;
10898  }
10899  allocator->FreeMemory(*pAllocation);
10900  *pAllocation = VK_NULL_HANDLE;
10901  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
10902  *pBuffer = VK_NULL_HANDLE;
10903  return res;
10904  }
10905  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
10906  *pBuffer = VK_NULL_HANDLE;
10907  return res;
10908  }
10909  return res;
10910 }
10911 
10912 void vmaDestroyBuffer(
10913  VmaAllocator allocator,
10914  VkBuffer buffer,
10915  VmaAllocation allocation)
10916 {
10917  VMA_ASSERT(allocator);
10918 
10919  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
10920  {
10921  return;
10922  }
10923 
10924  VMA_DEBUG_LOG("vmaDestroyBuffer");
10925 
10926  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10927 
10928 #if VMA_RECORDING_ENABLED
10929  if(allocator->GetRecorder() != VMA_NULL)
10930  {
10931  allocator->GetRecorder()->RecordDestroyBuffer(
10932  allocator->GetCurrentFrameIndex(),
10933  allocation);
10934  }
10935 #endif
10936 
10937  if(buffer != VK_NULL_HANDLE)
10938  {
10939  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
10940  }
10941 
10942  if(allocation != VK_NULL_HANDLE)
10943  {
10944  allocator->FreeMemory(allocation);
10945  }
10946 }
10947 
10948 VkResult vmaCreateImage(
10949  VmaAllocator allocator,
10950  const VkImageCreateInfo* pImageCreateInfo,
10951  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10952  VkImage* pImage,
10953  VmaAllocation* pAllocation,
10954  VmaAllocationInfo* pAllocationInfo)
10955 {
10956  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
10957 
10958  VMA_DEBUG_LOG("vmaCreateImage");
10959 
10960  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10961 
10962  *pImage = VK_NULL_HANDLE;
10963  *pAllocation = VK_NULL_HANDLE;
10964 
10965  // 1. Create VkImage.
10966  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
10967  allocator->m_hDevice,
10968  pImageCreateInfo,
10969  allocator->GetAllocationCallbacks(),
10970  pImage);
10971  if(res >= 0)
10972  {
10973  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
10974  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
10975  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
10976 
10977  // 2. Allocate memory using allocator.
10978  VkMemoryRequirements vkMemReq = {};
10979  bool requiresDedicatedAllocation = false;
10980  bool prefersDedicatedAllocation = false;
10981  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
10982  requiresDedicatedAllocation, prefersDedicatedAllocation);
10983 
10984  res = allocator->AllocateMemory(
10985  vkMemReq,
10986  requiresDedicatedAllocation,
10987  prefersDedicatedAllocation,
10988  VK_NULL_HANDLE, // dedicatedBuffer
10989  *pImage, // dedicatedImage
10990  *pAllocationCreateInfo,
10991  suballocType,
10992  pAllocation);
10993 
10994 #if VMA_RECORDING_ENABLED
10995  if(allocator->GetRecorder() != VMA_NULL)
10996  {
10997  allocator->GetRecorder()->RecordCreateImage(
10998  allocator->GetCurrentFrameIndex(),
10999  *pImageCreateInfo,
11000  *pAllocationCreateInfo,
11001  *pAllocation);
11002  }
11003 #endif
11004 
11005  if(res >= 0)
11006  {
11007  // 3. Bind image with memory.
11008  res = allocator->BindImageMemory(*pAllocation, *pImage);
11009  if(res >= 0)
11010  {
11011  // All steps succeeded.
11012  #if VMA_STATS_STRING_ENABLED
11013  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
11014  #endif
11015  if(pAllocationInfo != VMA_NULL)
11016  {
11017  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
11018  }
11019 
11020  return VK_SUCCESS;
11021  }
11022  allocator->FreeMemory(*pAllocation);
11023  *pAllocation = VK_NULL_HANDLE;
11024  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
11025  *pImage = VK_NULL_HANDLE;
11026  return res;
11027  }
11028  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
11029  *pImage = VK_NULL_HANDLE;
11030  return res;
11031  }
11032  return res;
11033 }
11034 
11035 void vmaDestroyImage(
11036  VmaAllocator allocator,
11037  VkImage image,
11038  VmaAllocation allocation)
11039 {
11040  VMA_ASSERT(allocator);
11041 
11042  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
11043  {
11044  return;
11045  }
11046 
11047  VMA_DEBUG_LOG("vmaDestroyImage");
11048 
11049  VMA_DEBUG_GLOBAL_MUTEX_LOCK
11050 
11051 #if VMA_RECORDING_ENABLED
11052  if(allocator->GetRecorder() != VMA_NULL)
11053  {
11054  allocator->GetRecorder()->RecordDestroyImage(
11055  allocator->GetCurrentFrameIndex(),
11056  allocation);
11057  }
11058 #endif
11059 
11060  if(image != VK_NULL_HANDLE)
11061  {
11062  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
11063  }
11064  if(allocation != VK_NULL_HANDLE)
11065  {
11066  allocator->FreeMemory(allocation);
11067  }
11068 }
11069 
11070 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1345
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
1258 #include <vulkan/vulkan.h>
1259 
1260 #if !defined(VMA_DEDICATED_ALLOCATION)
1261  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
1262  #define VMA_DEDICATED_ALLOCATION 1
1263  #else
1264  #define VMA_DEDICATED_ALLOCATION 0
1265  #endif
1266 #endif
1267 
1277 VK_DEFINE_HANDLE(VmaAllocator)
1278 
1279 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
1281  VmaAllocator allocator,
1282  uint32_t memoryType,
1283  VkDeviceMemory memory,
1284  VkDeviceSize size);
1286 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
1287  VmaAllocator allocator,
1288  uint32_t memoryType,
1289  VkDeviceMemory memory,
1290  VkDeviceSize size);
1291 
1299 typedef struct VmaDeviceMemoryCallbacks {
1301  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
1303  PFN_vmaFreeDeviceMemoryFunction pfnFree;
1304 } VmaDeviceMemoryCallbacks;
1305 
1307 typedef enum VmaAllocatorCreateFlagBits {
1312  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1334  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1335 
1336  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1337 } VmaAllocatorCreateFlagBits;
1338 typedef VkFlags VmaAllocatorCreateFlags;
1339 
1344 typedef struct VmaVulkanFunctions {
1345  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1346  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1347  PFN_vkAllocateMemory vkAllocateMemory;
1348  PFN_vkFreeMemory vkFreeMemory;
1349  PFN_vkMapMemory vkMapMemory;
1350  PFN_vkUnmapMemory vkUnmapMemory;
1351  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
1352  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
1353  PFN_vkBindBufferMemory vkBindBufferMemory;
1354  PFN_vkBindImageMemory vkBindImageMemory;
1355  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1356  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1357  PFN_vkCreateBuffer vkCreateBuffer;
1358  PFN_vkDestroyBuffer vkDestroyBuffer;
1359  PFN_vkCreateImage vkCreateImage;
1360  PFN_vkDestroyImage vkDestroyImage;
1361 #if VMA_DEDICATED_ALLOCATION
1362  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1363  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1364 #endif
1365 } VmaVulkanFunctions;
1366 
1368 typedef enum VmaRecordFlagBits {
1374  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
1375 
1376  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1377 } VmaRecordFlagBits;
1378 typedef VkFlags VmaRecordFlags;
1379 
1380 /*
1381 Define this macro to 0/1 to disable/enable support for recording functionality,
1382 available through VmaAllocatorCreateInfo::pRecordSettings.
1383 */
1384 #ifndef VMA_RECORDING_ENABLED
1385  #ifdef _WIN32
1386  #define VMA_RECORDING_ENABLED 1
1387  #else
1388  #define VMA_RECORDING_ENABLED 0
1389  #endif
1390 #endif
1391 
1393 typedef struct VmaRecordSettings
1394 {
1396  VmaRecordFlags flags;
1404  const char* pFilePath;
1405 } VmaRecordSettings;
1406 
1408 typedef struct VmaAllocatorCreateInfo
1409 {
1411  VmaAllocatorCreateFlags flags;
1413 
1414  VkPhysicalDevice physicalDevice;
1416 
1417  VkDevice device;
1419 
1420  VkDeviceSize preferredLargeHeapBlockSize;
1422 
1423  const VkAllocationCallbacks* pAllocationCallbacks;
1425 
1426  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1440  uint32_t frameInUseCount;
1464  const VkDeviceSize* pHeapSizeLimit;
1476  const VmaVulkanFunctions* pVulkanFunctions;
1483  const VmaRecordSettings* pRecordSettings;
1484 } VmaAllocatorCreateInfo;
1485 
1487 VkResult vmaCreateAllocator(
1488  const VmaAllocatorCreateInfo* pCreateInfo,
1489  VmaAllocator* pAllocator);
1490 
1492 void vmaDestroyAllocator(
1493  VmaAllocator allocator);
1494 
1499 void vmaGetPhysicalDeviceProperties(
1500  VmaAllocator allocator,
1501  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1502 
1507 void vmaGetMemoryProperties(
1508  VmaAllocator allocator,
1509  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1510 
1517 void vmaGetMemoryTypeProperties(
1518  VmaAllocator allocator,
1519  uint32_t memoryTypeIndex,
1520  VkMemoryPropertyFlags* pFlags);
1521 
1530 void vmaSetCurrentFrameIndex(
1531  VmaAllocator allocator,
1532  uint32_t frameIndex);
1533 
1536 typedef struct VmaStatInfo
1537 {
1539  uint32_t blockCount;
1541  uint32_t allocationCount;
1543  uint32_t unusedRangeCount;
1545  VkDeviceSize usedBytes;
1547  VkDeviceSize unusedBytes;
1548  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1549  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1550 } VmaStatInfo;
1551 
1553 typedef struct VmaStats
1554 {
1555  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1556  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1557  VmaStatInfo total;
1558 } VmaStats;
1559 
1561 void vmaCalculateStats(
1562  VmaAllocator allocator,
1563  VmaStats* pStats);
1564 
1565 #define VMA_STATS_STRING_ENABLED 1
1566 
1567 #if VMA_STATS_STRING_ENABLED
1568 
1570 
1572 void vmaBuildStatsString(
1573  VmaAllocator allocator,
1574  char** ppStatsString,
1575  VkBool32 detailedMap);
1576 
1577 void vmaFreeStatsString(
1578  VmaAllocator allocator,
1579  char* pStatsString);
1580 
1581 #endif // #if VMA_STATS_STRING_ENABLED
1582 
1591 VK_DEFINE_HANDLE(VmaPool)
1592 
1593 typedef enum VmaMemoryUsage
1594 {
1598  VMA_MEMORY_USAGE_UNKNOWN = 0,
1615  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1625  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1632  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1641  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1642  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1643 } VmaMemoryUsage;
1644 
1646 typedef enum VmaAllocationCreateFlagBits {
1658  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1659 
1669  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1682  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1695  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1702  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1708  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1709 
1710  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1711 } VmaAllocationCreateFlagBits;
1712 typedef VkFlags VmaAllocationCreateFlags;
1713 
1714 typedef struct VmaAllocationCreateInfo
1715 {
1717  VmaAllocationCreateFlags flags;
1723  VmaMemoryUsage usage;
1728  VkMemoryPropertyFlags requiredFlags;
1733  VkMemoryPropertyFlags preferredFlags;
1741  uint32_t memoryTypeBits;
1747  VmaPool pool;
1754  void* pUserData;
1755 } VmaAllocationCreateInfo;
1756 
1773 VkResult vmaFindMemoryTypeIndex(
1774  VmaAllocator allocator,
1775  uint32_t memoryTypeBits,
1776  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1777  uint32_t* pMemoryTypeIndex);
1778 
1791 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1792  VmaAllocator allocator,
1793  const VkBufferCreateInfo* pBufferCreateInfo,
1794  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1795  uint32_t* pMemoryTypeIndex);
1796 
1809 VkResult vmaFindMemoryTypeIndexForImageInfo(
1810  VmaAllocator allocator,
1811  const VkImageCreateInfo* pImageCreateInfo,
1812  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1813  uint32_t* pMemoryTypeIndex);
1814 
1816 typedef enum VmaPoolCreateFlagBits {
1834  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1835 
1836  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1837 } VmaPoolCreateFlagBits;
1838 typedef VkFlags VmaPoolCreateFlags;
1839 
1842 typedef struct VmaPoolCreateInfo {
1845  uint32_t memoryTypeIndex;
1848  VmaPoolCreateFlags flags;
1853  VkDeviceSize blockSize;
1858  size_t minBlockCount;
1866  size_t maxBlockCount;
1880  uint32_t frameInUseCount;
1881 } VmaPoolCreateInfo;
1882 
1885 typedef struct VmaPoolStats {
1888  VkDeviceSize size;
1891  VkDeviceSize unusedSize;
1894  size_t allocationCount;
1897  size_t unusedRangeCount;
1904  VkDeviceSize unusedRangeSizeMax;
1905 } VmaPoolStats;
1906 
1913 VkResult vmaCreatePool(
1914  VmaAllocator allocator,
1915  const VmaPoolCreateInfo* pCreateInfo,
1916  VmaPool* pPool);
1917 
1920 void vmaDestroyPool(
1921  VmaAllocator allocator,
1922  VmaPool pool);
1923 
1930 void vmaGetPoolStats(
1931  VmaAllocator allocator,
1932  VmaPool pool,
1933  VmaPoolStats* pPoolStats);
1934 
1941 void vmaMakePoolAllocationsLost(
1942  VmaAllocator allocator,
1943  VmaPool pool,
1944  size_t* pLostAllocationCount);
1945 
1960 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
1961 
1986 VK_DEFINE_HANDLE(VmaAllocation)
1987 
1988 
1990 typedef struct VmaAllocationInfo {
1995  uint32_t memoryType;
2004  VkDeviceMemory deviceMemory;
2009  VkDeviceSize offset;
2014  VkDeviceSize size;
2023  void* pMappedData;
2028  void* pUserData;
2029 } VmaAllocationInfo;
2030 
2041 VkResult vmaAllocateMemory(
2042  VmaAllocator allocator,
2043  const VkMemoryRequirements* pVkMemoryRequirements,
2044  const VmaAllocationCreateInfo* pCreateInfo,
2045  VmaAllocation* pAllocation,
2046  VmaAllocationInfo* pAllocationInfo);
2047 
2054 VkResult vmaAllocateMemoryForBuffer(
2055  VmaAllocator allocator,
2056  VkBuffer buffer,
2057  const VmaAllocationCreateInfo* pCreateInfo,
2058  VmaAllocation* pAllocation,
2059  VmaAllocationInfo* pAllocationInfo);
2060 
2062 VkResult vmaAllocateMemoryForImage(
2063  VmaAllocator allocator,
2064  VkImage image,
2065  const VmaAllocationCreateInfo* pCreateInfo,
2066  VmaAllocation* pAllocation,
2067  VmaAllocationInfo* pAllocationInfo);
2068 
2070 void vmaFreeMemory(
2071  VmaAllocator allocator,
2072  VmaAllocation allocation);
2073 
2090 void vmaGetAllocationInfo(
2091  VmaAllocator allocator,
2092  VmaAllocation allocation,
2093  VmaAllocationInfo* pAllocationInfo);
2094 
2109 VkBool32 vmaTouchAllocation(
2110  VmaAllocator allocator,
2111  VmaAllocation allocation);
2112 
2126 void vmaSetAllocationUserData(
2127  VmaAllocator allocator,
2128  VmaAllocation allocation,
2129  void* pUserData);
2130 
2141 void vmaCreateLostAllocation(
2142  VmaAllocator allocator,
2143  VmaAllocation* pAllocation);
2144 
2179 VkResult vmaMapMemory(
2180  VmaAllocator allocator,
2181  VmaAllocation allocation,
2182  void** ppData);
2183 
2188 void vmaUnmapMemory(
2189  VmaAllocator allocator,
2190  VmaAllocation allocation);
2191 
2204 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2205 
2218 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2219 
2236 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
2237 
2239 typedef struct VmaDefragmentationInfo {
2244  VkDeviceSize maxBytesToMove;
2249  uint32_t maxAllocationsToMove;
2250 } VmaDefragmentationInfo;
2251 
2253 typedef struct VmaDefragmentationStats {
2255  VkDeviceSize bytesMoved;
2257  VkDeviceSize bytesFreed;
2259  uint32_t allocationsMoved;
2261  uint32_t deviceMemoryBlocksFreed;
2262 } VmaDefragmentationStats;
2263 
2346 VkResult vmaDefragment(
2347  VmaAllocator allocator,
2348  VmaAllocation* pAllocations,
2349  size_t allocationCount,
2350  VkBool32* pAllocationsChanged,
2351  const VmaDefragmentationInfo *pDefragmentationInfo,
2352  VmaDefragmentationStats* pDefragmentationStats);
2353 
2366 VkResult vmaBindBufferMemory(
2367  VmaAllocator allocator,
2368  VmaAllocation allocation,
2369  VkBuffer buffer);
2370 
2383 VkResult vmaBindImageMemory(
2384  VmaAllocator allocator,
2385  VmaAllocation allocation,
2386  VkImage image);
2387 
2414 VkResult vmaCreateBuffer(
2415  VmaAllocator allocator,
2416  const VkBufferCreateInfo* pBufferCreateInfo,
2417  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2418  VkBuffer* pBuffer,
2419  VmaAllocation* pAllocation,
2420  VmaAllocationInfo* pAllocationInfo);
2421 
2433 void vmaDestroyBuffer(
2434  VmaAllocator allocator,
2435  VkBuffer buffer,
2436  VmaAllocation allocation);
2437 
2439 VkResult vmaCreateImage(
2440  VmaAllocator allocator,
2441  const VkImageCreateInfo* pImageCreateInfo,
2442  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2443  VkImage* pImage,
2444  VmaAllocation* pAllocation,
2445  VmaAllocationInfo* pAllocationInfo);
2446 
2458 void vmaDestroyImage(
2459  VmaAllocator allocator,
2460  VkImage image,
2461  VmaAllocation allocation);
2462 
2463 #ifdef __cplusplus
2464 }
2465 #endif
2466 
2467 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
2468 
2469 // For Visual Studio IntelliSense.
2470 #if defined(__cplusplus) && defined(__INTELLISENSE__)
2471 #define VMA_IMPLEMENTATION
2472 #endif
2473 
2474 #ifdef VMA_IMPLEMENTATION
2475 #undef VMA_IMPLEMENTATION
2476 
2477 #include <cstdint>
2478 #include <cstdlib>
2479 #include <cstring>
2480 
2481 /*******************************************************************************
2482 CONFIGURATION SECTION
2483 
2484 Define some of these macros before each #include of this header or change them
2485 here if you need other then default behavior depending on your environment.
2486 */
2487 
2488 /*
2489 Define this macro to 1 to make the library fetch pointers to Vulkan functions
2490 internally, like:
2491 
2492  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
2493 
2494 Define to 0 if you are going to provide you own pointers to Vulkan functions via
2495 VmaAllocatorCreateInfo::pVulkanFunctions.
2496 */
2497 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
2498 #define VMA_STATIC_VULKAN_FUNCTIONS 1
2499 #endif
2500 
2501 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
2502 //#define VMA_USE_STL_CONTAINERS 1
2503 
2504 /* Set this macro to 1 to make the library including and using STL containers:
2505 std::pair, std::vector, std::list, std::unordered_map.
2506 
2507 Set it to 0 or undefined to make the library using its own implementation of
2508 the containers.
2509 */
2510 #if VMA_USE_STL_CONTAINERS
2511  #define VMA_USE_STL_VECTOR 1
2512  #define VMA_USE_STL_UNORDERED_MAP 1
2513  #define VMA_USE_STL_LIST 1
2514 #endif
2515 
2516 #if VMA_USE_STL_VECTOR
2517  #include <vector>
2518 #endif
2519 
2520 #if VMA_USE_STL_UNORDERED_MAP
2521  #include <unordered_map>
2522 #endif
2523 
2524 #if VMA_USE_STL_LIST
2525  #include <list>
2526 #endif
2527 
2528 /*
2529 Following headers are used in this CONFIGURATION section only, so feel free to
2530 remove them if not needed.
2531 */
2532 #include <cassert> // for assert
2533 #include <algorithm> // for min, max
2534 #include <mutex> // for std::mutex
2535 #include <atomic> // for std::atomic
2536 
2537 #ifndef VMA_NULL
2538  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2539  #define VMA_NULL nullptr
2540 #endif
2541 
2542 #if defined(__APPLE__) || defined(__ANDROID__)
2543 #include <cstdlib>
2544 void *aligned_alloc(size_t alignment, size_t size)
2545 {
2546  // alignment must be >= sizeof(void*)
2547  if(alignment < sizeof(void*))
2548  {
2549  alignment = sizeof(void*);
2550  }
2551 
2552  void *pointer;
2553  if(posix_memalign(&pointer, alignment, size) == 0)
2554  return pointer;
2555  return VMA_NULL;
2556 }
2557 #endif
2558 
2559 // If your compiler is not compatible with C++11 and definition of
2560 // aligned_alloc() function is missing, uncommeting following line may help:
2561 
2562 //#include <malloc.h>
2563 
2564 // Normal assert to check for programmer's errors, especially in Debug configuration.
2565 #ifndef VMA_ASSERT
2566  #ifdef _DEBUG
2567  #define VMA_ASSERT(expr) assert(expr)
2568  #else
2569  #define VMA_ASSERT(expr)
2570  #endif
2571 #endif
2572 
2573 // Assert that will be called very often, like inside data structures e.g. operator[].
2574 // Making it non-empty can make program slow.
2575 #ifndef VMA_HEAVY_ASSERT
2576  #ifdef _DEBUG
2577  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2578  #else
2579  #define VMA_HEAVY_ASSERT(expr)
2580  #endif
2581 #endif
2582 
2583 #ifndef VMA_ALIGN_OF
2584  #define VMA_ALIGN_OF(type) (__alignof(type))
2585 #endif
2586 
2587 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2588  #if defined(_WIN32)
2589  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2590  #else
2591  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2592  #endif
2593 #endif
2594 
2595 #ifndef VMA_SYSTEM_FREE
2596  #if defined(_WIN32)
2597  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2598  #else
2599  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2600  #endif
2601 #endif
2602 
2603 #ifndef VMA_MIN
2604  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2605 #endif
2606 
2607 #ifndef VMA_MAX
2608  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2609 #endif
2610 
2611 #ifndef VMA_SWAP
2612  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2613 #endif
2614 
2615 #ifndef VMA_SORT
2616  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2617 #endif
2618 
2619 #ifndef VMA_DEBUG_LOG
2620  #define VMA_DEBUG_LOG(format, ...)
2621  /*
2622  #define VMA_DEBUG_LOG(format, ...) do { \
2623  printf(format, __VA_ARGS__); \
2624  printf("\n"); \
2625  } while(false)
2626  */
2627 #endif
2628 
2629 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2630 #if VMA_STATS_STRING_ENABLED
2631  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2632  {
2633  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2634  }
2635  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2636  {
2637  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2638  }
2639  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2640  {
2641  snprintf(outStr, strLen, "%p", ptr);
2642  }
2643 #endif
2644 
2645 #ifndef VMA_MUTEX
2646  class VmaMutex
2647  {
2648  public:
2649  VmaMutex() { }
2650  ~VmaMutex() { }
2651  void Lock() { m_Mutex.lock(); }
2652  void Unlock() { m_Mutex.unlock(); }
2653  private:
2654  std::mutex m_Mutex;
2655  };
2656  #define VMA_MUTEX VmaMutex
2657 #endif
2658 
2659 /*
2660 If providing your own implementation, you need to implement a subset of std::atomic:
2661 
2662 - Constructor(uint32_t desired)
2663 - uint32_t load() const
2664 - void store(uint32_t desired)
2665 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2666 */
2667 #ifndef VMA_ATOMIC_UINT32
2668  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2669 #endif
2670 
2671 #ifndef VMA_BEST_FIT
2672 
2684  #define VMA_BEST_FIT (1)
2685 #endif
2686 
2687 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2688 
2692  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2693 #endif
2694 
2695 #ifndef VMA_DEBUG_ALIGNMENT
2696 
2700  #define VMA_DEBUG_ALIGNMENT (1)
2701 #endif
2702 
2703 #ifndef VMA_DEBUG_MARGIN
2704 
2708  #define VMA_DEBUG_MARGIN (0)
2709 #endif
2710 
2711 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
2712 
2716  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
2717 #endif
2718 
2719 #ifndef VMA_DEBUG_DETECT_CORRUPTION
2720 
2725  #define VMA_DEBUG_DETECT_CORRUPTION (0)
2726 #endif
2727 
2728 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2729 
2733  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2734 #endif
2735 
2736 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2737 
2741  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2742 #endif
2743 
2744 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2745  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2747 #endif
2748 
2749 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2750  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2752 #endif
2753 
2754 #ifndef VMA_CLASS_NO_COPY
2755  #define VMA_CLASS_NO_COPY(className) \
2756  private: \
2757  className(const className&) = delete; \
2758  className& operator=(const className&) = delete;
2759 #endif
2760 
2761 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2762 
2763 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
2764 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
2765 
2766 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
2767 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
2768 
2769 /*******************************************************************************
2770 END OF CONFIGURATION
2771 */
2772 
2773 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2774  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2775 
2776 // Returns number of bits set to 1 in (v).
2777 static inline uint32_t VmaCountBitsSet(uint32_t v)
2778 {
2779  uint32_t c = v - ((v >> 1) & 0x55555555);
2780  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2781  c = ((c >> 4) + c) & 0x0F0F0F0F;
2782  c = ((c >> 8) + c) & 0x00FF00FF;
2783  c = ((c >> 16) + c) & 0x0000FFFF;
2784  return c;
2785 }
2786 
2787 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2788 // Use types like uint32_t, uint64_t as T.
2789 template <typename T>
2790 static inline T VmaAlignUp(T val, T align)
2791 {
2792  return (val + align - 1) / align * align;
2793 }
2794 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
2795 // Use types like uint32_t, uint64_t as T.
2796 template <typename T>
2797 static inline T VmaAlignDown(T val, T align)
2798 {
2799  return val / align * align;
2800 }
2801 
2802 // Division with mathematical rounding to nearest number.
2803 template <typename T>
2804 inline T VmaRoundDiv(T x, T y)
2805 {
2806  return (x + (y / (T)2)) / y;
2807 }
2808 
2809 static inline bool VmaStrIsEmpty(const char* pStr)
2810 {
2811  return pStr == VMA_NULL || *pStr == '\0';
2812 }
2813 
2814 #ifndef VMA_SORT
2815 
2816 template<typename Iterator, typename Compare>
2817 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2818 {
2819  Iterator centerValue = end; --centerValue;
2820  Iterator insertIndex = beg;
2821  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2822  {
2823  if(cmp(*memTypeIndex, *centerValue))
2824  {
2825  if(insertIndex != memTypeIndex)
2826  {
2827  VMA_SWAP(*memTypeIndex, *insertIndex);
2828  }
2829  ++insertIndex;
2830  }
2831  }
2832  if(insertIndex != centerValue)
2833  {
2834  VMA_SWAP(*insertIndex, *centerValue);
2835  }
2836  return insertIndex;
2837 }
2838 
2839 template<typename Iterator, typename Compare>
2840 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2841 {
2842  if(beg < end)
2843  {
2844  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2845  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2846  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2847  }
2848 }
2849 
2850 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2851 
2852 #endif // #ifndef VMA_SORT
2853 
2854 /*
2855 Returns true if two memory blocks occupy overlapping pages.
2856 ResourceA must be in less memory offset than ResourceB.
2857 
2858 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2859 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2860 */
2861 static inline bool VmaBlocksOnSamePage(
2862  VkDeviceSize resourceAOffset,
2863  VkDeviceSize resourceASize,
2864  VkDeviceSize resourceBOffset,
2865  VkDeviceSize pageSize)
2866 {
2867  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2868  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2869  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2870  VkDeviceSize resourceBStart = resourceBOffset;
2871  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2872  return resourceAEndPage == resourceBStartPage;
2873 }
2874 
2875 enum VmaSuballocationType
2876 {
2877  VMA_SUBALLOCATION_TYPE_FREE = 0,
2878  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2879  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2880  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2881  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2882  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2883  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2884 };
2885 
2886 /*
2887 Returns true if given suballocation types could conflict and must respect
2888 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2889 or linear image and another one is optimal image. If type is unknown, behave
2890 conservatively.
2891 */
2892 static inline bool VmaIsBufferImageGranularityConflict(
2893  VmaSuballocationType suballocType1,
2894  VmaSuballocationType suballocType2)
2895 {
2896  if(suballocType1 > suballocType2)
2897  {
2898  VMA_SWAP(suballocType1, suballocType2);
2899  }
2900 
2901  switch(suballocType1)
2902  {
2903  case VMA_SUBALLOCATION_TYPE_FREE:
2904  return false;
2905  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2906  return true;
2907  case VMA_SUBALLOCATION_TYPE_BUFFER:
2908  return
2909  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2910  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2911  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2912  return
2913  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2914  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2915  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2916  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2917  return
2918  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2919  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2920  return false;
2921  default:
2922  VMA_ASSERT(0);
2923  return true;
2924  }
2925 }
2926 
2927 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
2928 {
2929  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
2930  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
2931  for(size_t i = 0; i < numberCount; ++i, ++pDst)
2932  {
2933  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
2934  }
2935 }
2936 
2937 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
2938 {
2939  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
2940  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
2941  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
2942  {
2943  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
2944  {
2945  return false;
2946  }
2947  }
2948  return true;
2949 }
2950 
2951 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2952 struct VmaMutexLock
2953 {
2954  VMA_CLASS_NO_COPY(VmaMutexLock)
2955 public:
2956  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2957  m_pMutex(useMutex ? &mutex : VMA_NULL)
2958  {
2959  if(m_pMutex)
2960  {
2961  m_pMutex->Lock();
2962  }
2963  }
2964 
2965  ~VmaMutexLock()
2966  {
2967  if(m_pMutex)
2968  {
2969  m_pMutex->Unlock();
2970  }
2971  }
2972 
2973 private:
2974  VMA_MUTEX* m_pMutex;
2975 };
2976 
2977 #if VMA_DEBUG_GLOBAL_MUTEX
2978  static VMA_MUTEX gDebugGlobalMutex;
2979  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2980 #else
2981  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2982 #endif
2983 
2984 // Minimum size of a free suballocation to register it in the free suballocation collection.
2985 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2986 
2987 /*
2988 Performs binary search and returns iterator to first element that is greater or
2989 equal to (key), according to comparison (cmp).
2990 
2991 Cmp should return true if first argument is less than second argument.
2992 
2993 Returned value is the found element, if present in the collection or place where
2994 new element with value (key) should be inserted.
2995 */
2996 template <typename IterT, typename KeyT, typename CmpT>
2997 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2998 {
2999  size_t down = 0, up = (end - beg);
3000  while(down < up)
3001  {
3002  const size_t mid = (down + up) / 2;
3003  if(cmp(*(beg+mid), key))
3004  {
3005  down = mid + 1;
3006  }
3007  else
3008  {
3009  up = mid;
3010  }
3011  }
3012  return beg + down;
3013 }
3014 
3016 // Memory allocation
3017 
3018 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
3019 {
3020  if((pAllocationCallbacks != VMA_NULL) &&
3021  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
3022  {
3023  return (*pAllocationCallbacks->pfnAllocation)(
3024  pAllocationCallbacks->pUserData,
3025  size,
3026  alignment,
3027  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3028  }
3029  else
3030  {
3031  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
3032  }
3033 }
3034 
3035 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
3036 {
3037  if((pAllocationCallbacks != VMA_NULL) &&
3038  (pAllocationCallbacks->pfnFree != VMA_NULL))
3039  {
3040  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
3041  }
3042  else
3043  {
3044  VMA_SYSTEM_FREE(ptr);
3045  }
3046 }
3047 
3048 template<typename T>
3049 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
3050 {
3051  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
3052 }
3053 
3054 template<typename T>
3055 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
3056 {
3057  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
3058 }
3059 
3060 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
3061 
3062 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
3063 
3064 template<typename T>
3065 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
3066 {
3067  ptr->~T();
3068  VmaFree(pAllocationCallbacks, ptr);
3069 }
3070 
3071 template<typename T>
3072 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
3073 {
3074  if(ptr != VMA_NULL)
3075  {
3076  for(size_t i = count; i--; )
3077  {
3078  ptr[i].~T();
3079  }
3080  VmaFree(pAllocationCallbacks, ptr);
3081  }
3082 }
3083 
3084 // STL-compatible allocator.
3085 template<typename T>
3086 class VmaStlAllocator
3087 {
3088 public:
3089  const VkAllocationCallbacks* const m_pCallbacks;
3090  typedef T value_type;
3091 
3092  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
3093  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
3094 
3095  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
3096  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
3097 
3098  template<typename U>
3099  bool operator==(const VmaStlAllocator<U>& rhs) const
3100  {
3101  return m_pCallbacks == rhs.m_pCallbacks;
3102  }
3103  template<typename U>
3104  bool operator!=(const VmaStlAllocator<U>& rhs) const
3105  {
3106  return m_pCallbacks != rhs.m_pCallbacks;
3107  }
3108 
3109  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
3110 };
3111 
3112 #if VMA_USE_STL_VECTOR
3113 
3114 #define VmaVector std::vector
3115 
3116 template<typename T, typename allocatorT>
3117 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
3118 {
3119  vec.insert(vec.begin() + index, item);
3120 }
3121 
3122 template<typename T, typename allocatorT>
3123 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
3124 {
3125  vec.erase(vec.begin() + index);
3126 }
3127 
3128 #else // #if VMA_USE_STL_VECTOR
3129 
3130 /* Class with interface compatible with subset of std::vector.
3131 T must be POD because constructors and destructors are not called and memcpy is
3132 used for these objects. */
3133 template<typename T, typename AllocatorT>
3134 class VmaVector
3135 {
3136 public:
3137  typedef T value_type;
3138 
3139  VmaVector(const AllocatorT& allocator) :
3140  m_Allocator(allocator),
3141  m_pArray(VMA_NULL),
3142  m_Count(0),
3143  m_Capacity(0)
3144  {
3145  }
3146 
3147  VmaVector(size_t count, const AllocatorT& allocator) :
3148  m_Allocator(allocator),
3149  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
3150  m_Count(count),
3151  m_Capacity(count)
3152  {
3153  }
3154 
3155  VmaVector(const VmaVector<T, AllocatorT>& src) :
3156  m_Allocator(src.m_Allocator),
3157  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
3158  m_Count(src.m_Count),
3159  m_Capacity(src.m_Count)
3160  {
3161  if(m_Count != 0)
3162  {
3163  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
3164  }
3165  }
3166 
3167  ~VmaVector()
3168  {
3169  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3170  }
3171 
3172  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
3173  {
3174  if(&rhs != this)
3175  {
3176  resize(rhs.m_Count);
3177  if(m_Count != 0)
3178  {
3179  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
3180  }
3181  }
3182  return *this;
3183  }
3184 
3185  bool empty() const { return m_Count == 0; }
3186  size_t size() const { return m_Count; }
3187  T* data() { return m_pArray; }
3188  const T* data() const { return m_pArray; }
3189 
3190  T& operator[](size_t index)
3191  {
3192  VMA_HEAVY_ASSERT(index < m_Count);
3193  return m_pArray[index];
3194  }
3195  const T& operator[](size_t index) const
3196  {
3197  VMA_HEAVY_ASSERT(index < m_Count);
3198  return m_pArray[index];
3199  }
3200 
3201  T& front()
3202  {
3203  VMA_HEAVY_ASSERT(m_Count > 0);
3204  return m_pArray[0];
3205  }
3206  const T& front() const
3207  {
3208  VMA_HEAVY_ASSERT(m_Count > 0);
3209  return m_pArray[0];
3210  }
3211  T& back()
3212  {
3213  VMA_HEAVY_ASSERT(m_Count > 0);
3214  return m_pArray[m_Count - 1];
3215  }
3216  const T& back() const
3217  {
3218  VMA_HEAVY_ASSERT(m_Count > 0);
3219  return m_pArray[m_Count - 1];
3220  }
3221 
3222  void reserve(size_t newCapacity, bool freeMemory = false)
3223  {
3224  newCapacity = VMA_MAX(newCapacity, m_Count);
3225 
3226  if((newCapacity < m_Capacity) && !freeMemory)
3227  {
3228  newCapacity = m_Capacity;
3229  }
3230 
3231  if(newCapacity != m_Capacity)
3232  {
3233  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
3234  if(m_Count != 0)
3235  {
3236  memcpy(newArray, m_pArray, m_Count * sizeof(T));
3237  }
3238  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3239  m_Capacity = newCapacity;
3240  m_pArray = newArray;
3241  }
3242  }
3243 
3244  void resize(size_t newCount, bool freeMemory = false)
3245  {
3246  size_t newCapacity = m_Capacity;
3247  if(newCount > m_Capacity)
3248  {
3249  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
3250  }
3251  else if(freeMemory)
3252  {
3253  newCapacity = newCount;
3254  }
3255 
3256  if(newCapacity != m_Capacity)
3257  {
3258  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
3259  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
3260  if(elementsToCopy != 0)
3261  {
3262  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
3263  }
3264  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3265  m_Capacity = newCapacity;
3266  m_pArray = newArray;
3267  }
3268 
3269  m_Count = newCount;
3270  }
3271 
3272  void clear(bool freeMemory = false)
3273  {
3274  resize(0, freeMemory);
3275  }
3276 
3277  void insert(size_t index, const T& src)
3278  {
3279  VMA_HEAVY_ASSERT(index <= m_Count);
3280  const size_t oldCount = size();
3281  resize(oldCount + 1);
3282  if(index < oldCount)
3283  {
3284  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
3285  }
3286  m_pArray[index] = src;
3287  }
3288 
3289  void remove(size_t index)
3290  {
3291  VMA_HEAVY_ASSERT(index < m_Count);
3292  const size_t oldCount = size();
3293  if(index < oldCount - 1)
3294  {
3295  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
3296  }
3297  resize(oldCount - 1);
3298  }
3299 
3300  void push_back(const T& src)
3301  {
3302  const size_t newIndex = size();
3303  resize(newIndex + 1);
3304  m_pArray[newIndex] = src;
3305  }
3306 
3307  void pop_back()
3308  {
3309  VMA_HEAVY_ASSERT(m_Count > 0);
3310  resize(size() - 1);
3311  }
3312 
3313  void push_front(const T& src)
3314  {
3315  insert(0, src);
3316  }
3317 
3318  void pop_front()
3319  {
3320  VMA_HEAVY_ASSERT(m_Count > 0);
3321  remove(0);
3322  }
3323 
3324  typedef T* iterator;
3325 
3326  iterator begin() { return m_pArray; }
3327  iterator end() { return m_pArray + m_Count; }
3328 
3329 private:
3330  AllocatorT m_Allocator;
3331  T* m_pArray;
3332  size_t m_Count;
3333  size_t m_Capacity;
3334 };
3335 
3336 template<typename T, typename allocatorT>
3337 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
3338 {
3339  vec.insert(index, item);
3340 }
3341 
3342 template<typename T, typename allocatorT>
3343 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
3344 {
3345  vec.remove(index);
3346 }
3347 
3348 #endif // #if VMA_USE_STL_VECTOR
3349 
3350 template<typename CmpLess, typename VectorT>
3351 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
3352 {
3353  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3354  vector.data(),
3355  vector.data() + vector.size(),
3356  value,
3357  CmpLess()) - vector.data();
3358  VmaVectorInsert(vector, indexToInsert, value);
3359  return indexToInsert;
3360 }
3361 
3362 template<typename CmpLess, typename VectorT>
3363 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
3364 {
3365  CmpLess comparator;
3366  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3367  vector.begin(),
3368  vector.end(),
3369  value,
3370  comparator);
3371  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
3372  {
3373  size_t indexToRemove = it - vector.begin();
3374  VmaVectorRemove(vector, indexToRemove);
3375  return true;
3376  }
3377  return false;
3378 }
3379 
3380 template<typename CmpLess, typename VectorT>
3381 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
3382 {
3383  CmpLess comparator;
3384  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3385  vector.data(),
3386  vector.data() + vector.size(),
3387  value,
3388  comparator);
3389  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
3390  {
3391  return it - vector.begin();
3392  }
3393  else
3394  {
3395  return vector.size();
3396  }
3397 }
3398 
3400 // class VmaPoolAllocator
3401 
3402 /*
3403 Allocator for objects of type T using a list of arrays (pools) to speed up
3404 allocation. Number of elements that can be allocated is not bounded because
3405 allocator can create multiple blocks.
3406 */
3407 template<typename T>
3408 class VmaPoolAllocator
3409 {
3410  VMA_CLASS_NO_COPY(VmaPoolAllocator)
3411 public:
3412  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
3413  ~VmaPoolAllocator();
3414  void Clear();
3415  T* Alloc();
3416  void Free(T* ptr);
3417 
3418 private:
3419  union Item
3420  {
3421  uint32_t NextFreeIndex;
3422  T Value;
3423  };
3424 
3425  struct ItemBlock
3426  {
3427  Item* pItems;
3428  uint32_t FirstFreeIndex;
3429  };
3430 
3431  const VkAllocationCallbacks* m_pAllocationCallbacks;
3432  size_t m_ItemsPerBlock;
3433  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
3434 
3435  ItemBlock& CreateNewBlock();
3436 };
3437 
3438 template<typename T>
3439 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
3440  m_pAllocationCallbacks(pAllocationCallbacks),
3441  m_ItemsPerBlock(itemsPerBlock),
3442  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
3443 {
3444  VMA_ASSERT(itemsPerBlock > 0);
3445 }
3446 
3447 template<typename T>
3448 VmaPoolAllocator<T>::~VmaPoolAllocator()
3449 {
3450  Clear();
3451 }
3452 
3453 template<typename T>
3454 void VmaPoolAllocator<T>::Clear()
3455 {
3456  for(size_t i = m_ItemBlocks.size(); i--; )
3457  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
3458  m_ItemBlocks.clear();
3459 }
3460 
3461 template<typename T>
3462 T* VmaPoolAllocator<T>::Alloc()
3463 {
3464  for(size_t i = m_ItemBlocks.size(); i--; )
3465  {
3466  ItemBlock& block = m_ItemBlocks[i];
3467  // This block has some free items: Use first one.
3468  if(block.FirstFreeIndex != UINT32_MAX)
3469  {
3470  Item* const pItem = &block.pItems[block.FirstFreeIndex];
3471  block.FirstFreeIndex = pItem->NextFreeIndex;
3472  return &pItem->Value;
3473  }
3474  }
3475 
3476  // No block has free item: Create new one and use it.
3477  ItemBlock& newBlock = CreateNewBlock();
3478  Item* const pItem = &newBlock.pItems[0];
3479  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
3480  return &pItem->Value;
3481 }
3482 
3483 template<typename T>
3484 void VmaPoolAllocator<T>::Free(T* ptr)
3485 {
3486  // Search all memory blocks to find ptr.
3487  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
3488  {
3489  ItemBlock& block = m_ItemBlocks[i];
3490 
3491  // Casting to union.
3492  Item* pItemPtr;
3493  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
3494 
3495  // Check if pItemPtr is in address range of this block.
3496  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
3497  {
3498  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
3499  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
3500  block.FirstFreeIndex = index;
3501  return;
3502  }
3503  }
3504  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
3505 }
3506 
3507 template<typename T>
3508 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
3509 {
3510  ItemBlock newBlock = {
3511  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
3512 
3513  m_ItemBlocks.push_back(newBlock);
3514 
3515  // Setup singly-linked list of all free items in this block.
3516  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
3517  newBlock.pItems[i].NextFreeIndex = i + 1;
3518  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
3519  return m_ItemBlocks.back();
3520 }
3521 
3523 // class VmaRawList, VmaList
3524 
3525 #if VMA_USE_STL_LIST
3526 
3527 #define VmaList std::list
3528 
3529 #else // #if VMA_USE_STL_LIST
3530 
3531 template<typename T>
3532 struct VmaListItem
3533 {
3534  VmaListItem* pPrev;
3535  VmaListItem* pNext;
3536  T Value;
3537 };
3538 
3539 // Doubly linked list.
3540 template<typename T>
3541 class VmaRawList
3542 {
3543  VMA_CLASS_NO_COPY(VmaRawList)
3544 public:
3545  typedef VmaListItem<T> ItemType;
3546 
3547  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
3548  ~VmaRawList();
3549  void Clear();
3550 
3551  size_t GetCount() const { return m_Count; }
3552  bool IsEmpty() const { return m_Count == 0; }
3553 
3554  ItemType* Front() { return m_pFront; }
3555  const ItemType* Front() const { return m_pFront; }
3556  ItemType* Back() { return m_pBack; }
3557  const ItemType* Back() const { return m_pBack; }
3558 
3559  ItemType* PushBack();
3560  ItemType* PushFront();
3561  ItemType* PushBack(const T& value);
3562  ItemType* PushFront(const T& value);
3563  void PopBack();
3564  void PopFront();
3565 
3566  // Item can be null - it means PushBack.
3567  ItemType* InsertBefore(ItemType* pItem);
3568  // Item can be null - it means PushFront.
3569  ItemType* InsertAfter(ItemType* pItem);
3570 
3571  ItemType* InsertBefore(ItemType* pItem, const T& value);
3572  ItemType* InsertAfter(ItemType* pItem, const T& value);
3573 
3574  void Remove(ItemType* pItem);
3575 
3576 private:
3577  const VkAllocationCallbacks* const m_pAllocationCallbacks;
3578  VmaPoolAllocator<ItemType> m_ItemAllocator;
3579  ItemType* m_pFront;
3580  ItemType* m_pBack;
3581  size_t m_Count;
3582 };
3583 
3584 template<typename T>
3585 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3586  m_pAllocationCallbacks(pAllocationCallbacks),
3587  m_ItemAllocator(pAllocationCallbacks, 128),
3588  m_pFront(VMA_NULL),
3589  m_pBack(VMA_NULL),
3590  m_Count(0)
3591 {
3592 }
3593 
3594 template<typename T>
3595 VmaRawList<T>::~VmaRawList()
3596 {
3597  // Intentionally not calling Clear, because that would be unnecessary
3598  // computations to return all items to m_ItemAllocator as free.
3599 }
3600 
3601 template<typename T>
3602 void VmaRawList<T>::Clear()
3603 {
3604  if(IsEmpty() == false)
3605  {
3606  ItemType* pItem = m_pBack;
3607  while(pItem != VMA_NULL)
3608  {
3609  ItemType* const pPrevItem = pItem->pPrev;
3610  m_ItemAllocator.Free(pItem);
3611  pItem = pPrevItem;
3612  }
3613  m_pFront = VMA_NULL;
3614  m_pBack = VMA_NULL;
3615  m_Count = 0;
3616  }
3617 }
3618 
3619 template<typename T>
3620 VmaListItem<T>* VmaRawList<T>::PushBack()
3621 {
3622  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3623  pNewItem->pNext = VMA_NULL;
3624  if(IsEmpty())
3625  {
3626  pNewItem->pPrev = VMA_NULL;
3627  m_pFront = pNewItem;
3628  m_pBack = pNewItem;
3629  m_Count = 1;
3630  }
3631  else
3632  {
3633  pNewItem->pPrev = m_pBack;
3634  m_pBack->pNext = pNewItem;
3635  m_pBack = pNewItem;
3636  ++m_Count;
3637  }
3638  return pNewItem;
3639 }
3640 
3641 template<typename T>
3642 VmaListItem<T>* VmaRawList<T>::PushFront()
3643 {
3644  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3645  pNewItem->pPrev = VMA_NULL;
3646  if(IsEmpty())
3647  {
3648  pNewItem->pNext = VMA_NULL;
3649  m_pFront = pNewItem;
3650  m_pBack = pNewItem;
3651  m_Count = 1;
3652  }
3653  else
3654  {
3655  pNewItem->pNext = m_pFront;
3656  m_pFront->pPrev = pNewItem;
3657  m_pFront = pNewItem;
3658  ++m_Count;
3659  }
3660  return pNewItem;
3661 }
3662 
3663 template<typename T>
3664 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3665 {
3666  ItemType* const pNewItem = PushBack();
3667  pNewItem->Value = value;
3668  return pNewItem;
3669 }
3670 
3671 template<typename T>
3672 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3673 {
3674  ItemType* const pNewItem = PushFront();
3675  pNewItem->Value = value;
3676  return pNewItem;
3677 }
3678 
3679 template<typename T>
3680 void VmaRawList<T>::PopBack()
3681 {
3682  VMA_HEAVY_ASSERT(m_Count > 0);
3683  ItemType* const pBackItem = m_pBack;
3684  ItemType* const pPrevItem = pBackItem->pPrev;
3685  if(pPrevItem != VMA_NULL)
3686  {
3687  pPrevItem->pNext = VMA_NULL;
3688  }
3689  m_pBack = pPrevItem;
3690  m_ItemAllocator.Free(pBackItem);
3691  --m_Count;
3692 }
3693 
3694 template<typename T>
3695 void VmaRawList<T>::PopFront()
3696 {
3697  VMA_HEAVY_ASSERT(m_Count > 0);
3698  ItemType* const pFrontItem = m_pFront;
3699  ItemType* const pNextItem = pFrontItem->pNext;
3700  if(pNextItem != VMA_NULL)
3701  {
3702  pNextItem->pPrev = VMA_NULL;
3703  }
3704  m_pFront = pNextItem;
3705  m_ItemAllocator.Free(pFrontItem);
3706  --m_Count;
3707 }
3708 
3709 template<typename T>
3710 void VmaRawList<T>::Remove(ItemType* pItem)
3711 {
3712  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3713  VMA_HEAVY_ASSERT(m_Count > 0);
3714 
3715  if(pItem->pPrev != VMA_NULL)
3716  {
3717  pItem->pPrev->pNext = pItem->pNext;
3718  }
3719  else
3720  {
3721  VMA_HEAVY_ASSERT(m_pFront == pItem);
3722  m_pFront = pItem->pNext;
3723  }
3724 
3725  if(pItem->pNext != VMA_NULL)
3726  {
3727  pItem->pNext->pPrev = pItem->pPrev;
3728  }
3729  else
3730  {
3731  VMA_HEAVY_ASSERT(m_pBack == pItem);
3732  m_pBack = pItem->pPrev;
3733  }
3734 
3735  m_ItemAllocator.Free(pItem);
3736  --m_Count;
3737 }
3738 
3739 template<typename T>
3740 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3741 {
3742  if(pItem != VMA_NULL)
3743  {
3744  ItemType* const prevItem = pItem->pPrev;
3745  ItemType* const newItem = m_ItemAllocator.Alloc();
3746  newItem->pPrev = prevItem;
3747  newItem->pNext = pItem;
3748  pItem->pPrev = newItem;
3749  if(prevItem != VMA_NULL)
3750  {
3751  prevItem->pNext = newItem;
3752  }
3753  else
3754  {
3755  VMA_HEAVY_ASSERT(m_pFront == pItem);
3756  m_pFront = newItem;
3757  }
3758  ++m_Count;
3759  return newItem;
3760  }
3761  else
3762  return PushBack();
3763 }
3764 
3765 template<typename T>
3766 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3767 {
3768  if(pItem != VMA_NULL)
3769  {
3770  ItemType* const nextItem = pItem->pNext;
3771  ItemType* const newItem = m_ItemAllocator.Alloc();
3772  newItem->pNext = nextItem;
3773  newItem->pPrev = pItem;
3774  pItem->pNext = newItem;
3775  if(nextItem != VMA_NULL)
3776  {
3777  nextItem->pPrev = newItem;
3778  }
3779  else
3780  {
3781  VMA_HEAVY_ASSERT(m_pBack == pItem);
3782  m_pBack = newItem;
3783  }
3784  ++m_Count;
3785  return newItem;
3786  }
3787  else
3788  return PushFront();
3789 }
3790 
3791 template<typename T>
3792 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3793 {
3794  ItemType* const newItem = InsertBefore(pItem);
3795  newItem->Value = value;
3796  return newItem;
3797 }
3798 
3799 template<typename T>
3800 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3801 {
3802  ItemType* const newItem = InsertAfter(pItem);
3803  newItem->Value = value;
3804  return newItem;
3805 }
3806 
3807 template<typename T, typename AllocatorT>
3808 class VmaList
3809 {
3810  VMA_CLASS_NO_COPY(VmaList)
3811 public:
3812  class iterator
3813  {
3814  public:
3815  iterator() :
3816  m_pList(VMA_NULL),
3817  m_pItem(VMA_NULL)
3818  {
3819  }
3820 
3821  T& operator*() const
3822  {
3823  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3824  return m_pItem->Value;
3825  }
3826  T* operator->() const
3827  {
3828  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3829  return &m_pItem->Value;
3830  }
3831 
3832  iterator& operator++()
3833  {
3834  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3835  m_pItem = m_pItem->pNext;
3836  return *this;
3837  }
3838  iterator& operator--()
3839  {
3840  if(m_pItem != VMA_NULL)
3841  {
3842  m_pItem = m_pItem->pPrev;
3843  }
3844  else
3845  {
3846  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3847  m_pItem = m_pList->Back();
3848  }
3849  return *this;
3850  }
3851 
3852  iterator operator++(int)
3853  {
3854  iterator result = *this;
3855  ++*this;
3856  return result;
3857  }
3858  iterator operator--(int)
3859  {
3860  iterator result = *this;
3861  --*this;
3862  return result;
3863  }
3864 
3865  bool operator==(const iterator& rhs) const
3866  {
3867  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3868  return m_pItem == rhs.m_pItem;
3869  }
3870  bool operator!=(const iterator& rhs) const
3871  {
3872  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3873  return m_pItem != rhs.m_pItem;
3874  }
3875 
3876  private:
3877  VmaRawList<T>* m_pList;
3878  VmaListItem<T>* m_pItem;
3879 
3880  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3881  m_pList(pList),
3882  m_pItem(pItem)
3883  {
3884  }
3885 
3886  friend class VmaList<T, AllocatorT>;
3887  };
3888 
3889  class const_iterator
3890  {
3891  public:
3892  const_iterator() :
3893  m_pList(VMA_NULL),
3894  m_pItem(VMA_NULL)
3895  {
3896  }
3897 
3898  const_iterator(const iterator& src) :
3899  m_pList(src.m_pList),
3900  m_pItem(src.m_pItem)
3901  {
3902  }
3903 
3904  const T& operator*() const
3905  {
3906  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3907  return m_pItem->Value;
3908  }
3909  const T* operator->() const
3910  {
3911  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3912  return &m_pItem->Value;
3913  }
3914 
3915  const_iterator& operator++()
3916  {
3917  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3918  m_pItem = m_pItem->pNext;
3919  return *this;
3920  }
3921  const_iterator& operator--()
3922  {
3923  if(m_pItem != VMA_NULL)
3924  {
3925  m_pItem = m_pItem->pPrev;
3926  }
3927  else
3928  {
3929  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3930  m_pItem = m_pList->Back();
3931  }
3932  return *this;
3933  }
3934 
3935  const_iterator operator++(int)
3936  {
3937  const_iterator result = *this;
3938  ++*this;
3939  return result;
3940  }
3941  const_iterator operator--(int)
3942  {
3943  const_iterator result = *this;
3944  --*this;
3945  return result;
3946  }
3947 
3948  bool operator==(const const_iterator& rhs) const
3949  {
3950  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3951  return m_pItem == rhs.m_pItem;
3952  }
3953  bool operator!=(const const_iterator& rhs) const
3954  {
3955  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3956  return m_pItem != rhs.m_pItem;
3957  }
3958 
3959  private:
3960  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3961  m_pList(pList),
3962  m_pItem(pItem)
3963  {
3964  }
3965 
3966  const VmaRawList<T>* m_pList;
3967  const VmaListItem<T>* m_pItem;
3968 
3969  friend class VmaList<T, AllocatorT>;
3970  };
3971 
3972  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3973 
3974  bool empty() const { return m_RawList.IsEmpty(); }
3975  size_t size() const { return m_RawList.GetCount(); }
3976 
3977  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3978  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3979 
3980  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3981  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3982 
3983  void clear() { m_RawList.Clear(); }
3984  void push_back(const T& value) { m_RawList.PushBack(value); }
3985  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3986  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3987 
3988 private:
3989  VmaRawList<T> m_RawList;
3990 };
3991 
3992 #endif // #if VMA_USE_STL_LIST
3993 
3995 // class VmaMap
3996 
3997 // Unused in this version.
3998 #if 0
3999 
4000 #if VMA_USE_STL_UNORDERED_MAP
4001 
4002 #define VmaPair std::pair
4003 
4004 #define VMA_MAP_TYPE(KeyT, ValueT) \
4005  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
4006 
4007 #else // #if VMA_USE_STL_UNORDERED_MAP
4008 
4009 template<typename T1, typename T2>
4010 struct VmaPair
4011 {
4012  T1 first;
4013  T2 second;
4014 
4015  VmaPair() : first(), second() { }
4016  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
4017 };
4018 
4019 /* Class compatible with subset of interface of std::unordered_map.
4020 KeyT, ValueT must be POD because they will be stored in VmaVector.
4021 */
4022 template<typename KeyT, typename ValueT>
4023 class VmaMap
4024 {
4025 public:
4026  typedef VmaPair<KeyT, ValueT> PairType;
4027  typedef PairType* iterator;
4028 
4029  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
4030 
4031  iterator begin() { return m_Vector.begin(); }
4032  iterator end() { return m_Vector.end(); }
4033 
4034  void insert(const PairType& pair);
4035  iterator find(const KeyT& key);
4036  void erase(iterator it);
4037 
4038 private:
4039  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
4040 };
4041 
4042 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
4043 
4044 template<typename FirstT, typename SecondT>
4045 struct VmaPairFirstLess
4046 {
4047  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
4048  {
4049  return lhs.first < rhs.first;
4050  }
4051  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
4052  {
4053  return lhs.first < rhsFirst;
4054  }
4055 };
4056 
4057 template<typename KeyT, typename ValueT>
4058 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
4059 {
4060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4061  m_Vector.data(),
4062  m_Vector.data() + m_Vector.size(),
4063  pair,
4064  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
4065  VmaVectorInsert(m_Vector, indexToInsert, pair);
4066 }
4067 
4068 template<typename KeyT, typename ValueT>
4069 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
4070 {
4071  PairType* it = VmaBinaryFindFirstNotLess(
4072  m_Vector.data(),
4073  m_Vector.data() + m_Vector.size(),
4074  key,
4075  VmaPairFirstLess<KeyT, ValueT>());
4076  if((it != m_Vector.end()) && (it->first == key))
4077  {
4078  return it;
4079  }
4080  else
4081  {
4082  return m_Vector.end();
4083  }
4084 }
4085 
4086 template<typename KeyT, typename ValueT>
4087 void VmaMap<KeyT, ValueT>::erase(iterator it)
4088 {
4089  VmaVectorRemove(m_Vector, it - m_Vector.begin());
4090 }
4091 
4092 #endif // #if VMA_USE_STL_UNORDERED_MAP
4093 
4094 #endif // #if 0
4095 
4097 
4098 class VmaDeviceMemoryBlock;
4099 
4100 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
4101 
4102 struct VmaAllocation_T
4103 {
4104  VMA_CLASS_NO_COPY(VmaAllocation_T)
4105 private:
4106  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
4107 
4108  enum FLAGS
4109  {
4110  FLAG_USER_DATA_STRING = 0x01,
4111  };
4112 
4113 public:
4114  enum ALLOCATION_TYPE
4115  {
4116  ALLOCATION_TYPE_NONE,
4117  ALLOCATION_TYPE_BLOCK,
4118  ALLOCATION_TYPE_DEDICATED,
4119  };
4120 
4121  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
4122  m_Alignment(1),
4123  m_Size(0),
4124  m_pUserData(VMA_NULL),
4125  m_LastUseFrameIndex(currentFrameIndex),
4126  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
4127  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
4128  m_MapCount(0),
4129  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
4130  {
4131 #if VMA_STATS_STRING_ENABLED
4132  m_CreationFrameIndex = currentFrameIndex;
4133  m_BufferImageUsage = 0;
4134 #endif
4135  }
4136 
4137  ~VmaAllocation_T()
4138  {
4139  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
4140 
4141  // Check if owned string was freed.
4142  VMA_ASSERT(m_pUserData == VMA_NULL);
4143  }
4144 
4145  void InitBlockAllocation(
4146  VmaPool hPool,
4147  VmaDeviceMemoryBlock* block,
4148  VkDeviceSize offset,
4149  VkDeviceSize alignment,
4150  VkDeviceSize size,
4151  VmaSuballocationType suballocationType,
4152  bool mapped,
4153  bool canBecomeLost)
4154  {
4155  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4156  VMA_ASSERT(block != VMA_NULL);
4157  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4158  m_Alignment = alignment;
4159  m_Size = size;
4160  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4161  m_SuballocationType = (uint8_t)suballocationType;
4162  m_BlockAllocation.m_hPool = hPool;
4163  m_BlockAllocation.m_Block = block;
4164  m_BlockAllocation.m_Offset = offset;
4165  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
4166  }
4167 
4168  void InitLost()
4169  {
4170  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4171  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
4172  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4173  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
4174  m_BlockAllocation.m_Block = VMA_NULL;
4175  m_BlockAllocation.m_Offset = 0;
4176  m_BlockAllocation.m_CanBecomeLost = true;
4177  }
4178 
4179  void ChangeBlockAllocation(
4180  VmaAllocator hAllocator,
4181  VmaDeviceMemoryBlock* block,
4182  VkDeviceSize offset);
4183 
4184  // pMappedData not null means allocation is created with MAPPED flag.
4185  void InitDedicatedAllocation(
4186  uint32_t memoryTypeIndex,
4187  VkDeviceMemory hMemory,
4188  VmaSuballocationType suballocationType,
4189  void* pMappedData,
4190  VkDeviceSize size)
4191  {
4192  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4193  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
4194  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
4195  m_Alignment = 0;
4196  m_Size = size;
4197  m_SuballocationType = (uint8_t)suballocationType;
4198  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4199  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
4200  m_DedicatedAllocation.m_hMemory = hMemory;
4201  m_DedicatedAllocation.m_pMappedData = pMappedData;
4202  }
4203 
4204  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
4205  VkDeviceSize GetAlignment() const { return m_Alignment; }
4206  VkDeviceSize GetSize() const { return m_Size; }
4207  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
4208  void* GetUserData() const { return m_pUserData; }
4209  void SetUserData(VmaAllocator hAllocator, void* pUserData);
4210  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
4211 
4212  VmaDeviceMemoryBlock* GetBlock() const
4213  {
4214  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4215  return m_BlockAllocation.m_Block;
4216  }
4217  VkDeviceSize GetOffset() const;
4218  VkDeviceMemory GetMemory() const;
4219  uint32_t GetMemoryTypeIndex() const;
4220  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
4221  void* GetMappedData() const;
4222  bool CanBecomeLost() const;
4223  VmaPool GetPool() const;
4224 
4225  uint32_t GetLastUseFrameIndex() const
4226  {
4227  return m_LastUseFrameIndex.load();
4228  }
4229  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
4230  {
4231  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
4232  }
4233  /*
4234  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
4235  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
4236  - Else, returns false.
4237 
4238  If hAllocation is already lost, assert - you should not call it then.
4239  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
4240  */
4241  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4242 
4243  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
4244  {
4245  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
4246  outInfo.blockCount = 1;
4247  outInfo.allocationCount = 1;
4248  outInfo.unusedRangeCount = 0;
4249  outInfo.usedBytes = m_Size;
4250  outInfo.unusedBytes = 0;
4251  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
4252  outInfo.unusedRangeSizeMin = UINT64_MAX;
4253  outInfo.unusedRangeSizeMax = 0;
4254  }
4255 
4256  void BlockAllocMap();
4257  void BlockAllocUnmap();
4258  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
4259  void DedicatedAllocUnmap(VmaAllocator hAllocator);
4260 
4261 #if VMA_STATS_STRING_ENABLED
4262  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
4263  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
4264 
4265  void InitBufferImageUsage(uint32_t bufferImageUsage)
4266  {
4267  VMA_ASSERT(m_BufferImageUsage == 0);
4268  m_BufferImageUsage = bufferImageUsage;
4269  }
4270 
4271  void PrintParameters(class VmaJsonWriter& json) const;
4272 #endif
4273 
4274 private:
4275  VkDeviceSize m_Alignment;
4276  VkDeviceSize m_Size;
4277  void* m_pUserData;
4278  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
4279  uint8_t m_Type; // ALLOCATION_TYPE
4280  uint8_t m_SuballocationType; // VmaSuballocationType
4281  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
4282  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
4283  uint8_t m_MapCount;
4284  uint8_t m_Flags; // enum FLAGS
4285 
4286  // Allocation out of VmaDeviceMemoryBlock.
4287  struct BlockAllocation
4288  {
4289  VmaPool m_hPool; // Null if belongs to general memory.
4290  VmaDeviceMemoryBlock* m_Block;
4291  VkDeviceSize m_Offset;
4292  bool m_CanBecomeLost;
4293  };
4294 
4295  // Allocation for an object that has its own private VkDeviceMemory.
4296  struct DedicatedAllocation
4297  {
4298  uint32_t m_MemoryTypeIndex;
4299  VkDeviceMemory m_hMemory;
4300  void* m_pMappedData; // Not null means memory is mapped.
4301  };
4302 
4303  union
4304  {
4305  // Allocation out of VmaDeviceMemoryBlock.
4306  BlockAllocation m_BlockAllocation;
4307  // Allocation for an object that has its own private VkDeviceMemory.
4308  DedicatedAllocation m_DedicatedAllocation;
4309  };
4310 
4311 #if VMA_STATS_STRING_ENABLED
4312  uint32_t m_CreationFrameIndex;
4313  uint32_t m_BufferImageUsage; // 0 if unknown.
4314 #endif
4315 
4316  void FreeUserDataString(VmaAllocator hAllocator);
4317 };
4318 
4319 /*
4320 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
4321 allocated memory block or free.
4322 */
4323 struct VmaSuballocation
4324 {
4325  VkDeviceSize offset;
4326  VkDeviceSize size;
4327  VmaAllocation hAllocation;
4328  VmaSuballocationType type;
4329 };
4330 
4331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
4332 
4333 // Cost of one additional allocation lost, as equivalent in bytes.
4334 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
4335 
4336 /*
4337 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
4338 
4339 If canMakeOtherLost was false:
4340 - item points to a FREE suballocation.
4341 - itemsToMakeLostCount is 0.
4342 
4343 If canMakeOtherLost was true:
4344 - item points to first of sequence of suballocations, which are either FREE,
4345  or point to VmaAllocations that can become lost.
4346 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
4347  the requested allocation to succeed.
4348 */
4349 struct VmaAllocationRequest
4350 {
4351  VkDeviceSize offset;
4352  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
4353  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
4354  VmaSuballocationList::iterator item;
4355  size_t itemsToMakeLostCount;
4356 
4357  VkDeviceSize CalcCost() const
4358  {
4359  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
4360  }
4361 };
4362 
4363 /*
4364 Data structure used for bookkeeping of allocations and unused ranges of memory
4365 in a single VkDeviceMemory block.
4366 */
4367 class VmaBlockMetadata
4368 {
4369  VMA_CLASS_NO_COPY(VmaBlockMetadata)
4370 public:
4371  VmaBlockMetadata(VmaAllocator hAllocator);
4372  ~VmaBlockMetadata();
4373  void Init(VkDeviceSize size);
4374 
4375  // Validates all data structures inside this object. If not valid, returns false.
4376  bool Validate() const;
4377  VkDeviceSize GetSize() const { return m_Size; }
4378  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
4379  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4380  VkDeviceSize GetUnusedRangeSizeMax() const;
4381  // Returns true if this block is empty - contains only single free suballocation.
4382  bool IsEmpty() const;
4383 
4384  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4385  void AddPoolStats(VmaPoolStats& inoutStats) const;
4386 
4387 #if VMA_STATS_STRING_ENABLED
4388  void PrintDetailedMap(class VmaJsonWriter& json) const;
4389 #endif
4390 
4391  // Tries to find a place for suballocation with given parameters inside this block.
4392  // If succeeded, fills pAllocationRequest and returns true.
4393  // If failed, returns false.
4394  bool CreateAllocationRequest(
4395  uint32_t currentFrameIndex,
4396  uint32_t frameInUseCount,
4397  VkDeviceSize bufferImageGranularity,
4398  VkDeviceSize allocSize,
4399  VkDeviceSize allocAlignment,
4400  VmaSuballocationType allocType,
4401  bool canMakeOtherLost,
4402  VmaAllocationRequest* pAllocationRequest);
4403 
4404  bool MakeRequestedAllocationsLost(
4405  uint32_t currentFrameIndex,
4406  uint32_t frameInUseCount,
4407  VmaAllocationRequest* pAllocationRequest);
4408 
4409  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4410 
4411  VkResult CheckCorruption(const void* pBlockData);
4412 
4413  // Makes actual allocation based on request. Request must already be checked and valid.
4414  void Alloc(
4415  const VmaAllocationRequest& request,
4416  VmaSuballocationType type,
4417  VkDeviceSize allocSize,
4418  VmaAllocation hAllocation);
4419 
4420  // Frees suballocation assigned to given memory region.
4421  void Free(const VmaAllocation allocation);
4422  void FreeAtOffset(VkDeviceSize offset);
4423 
4424 private:
4425  VkDeviceSize m_Size;
4426  uint32_t m_FreeCount;
4427  VkDeviceSize m_SumFreeSize;
4428  VmaSuballocationList m_Suballocations;
4429  // Suballocations that are free and have size greater than certain threshold.
4430  // Sorted by size, ascending.
4431  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
4432 
4433  bool ValidateFreeSuballocationList() const;
4434 
4435  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
4436  // If yes, fills pOffset and returns true. If no, returns false.
4437  bool CheckAllocation(
4438  uint32_t currentFrameIndex,
4439  uint32_t frameInUseCount,
4440  VkDeviceSize bufferImageGranularity,
4441  VkDeviceSize allocSize,
4442  VkDeviceSize allocAlignment,
4443  VmaSuballocationType allocType,
4444  VmaSuballocationList::const_iterator suballocItem,
4445  bool canMakeOtherLost,
4446  VkDeviceSize* pOffset,
4447  size_t* itemsToMakeLostCount,
4448  VkDeviceSize* pSumFreeSize,
4449  VkDeviceSize* pSumItemSize) const;
4450  // Given free suballocation, it merges it with following one, which must also be free.
4451  void MergeFreeWithNext(VmaSuballocationList::iterator item);
4452  // Releases given suballocation, making it free.
4453  // Merges it with adjacent free suballocations if applicable.
4454  // Returns iterator to new free suballocation at this place.
4455  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
4456  // Given free suballocation, it inserts it into sorted list of
4457  // m_FreeSuballocationsBySize if it's suitable.
4458  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
4459  // Given free suballocation, it removes it from sorted list of
4460  // m_FreeSuballocationsBySize if it's suitable.
4461  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
4462 };
4463 
4464 /*
4465 Represents a single block of device memory (`VkDeviceMemory`) with all the
4466 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
4467 
4468 Thread-safety: This class must be externally synchronized.
4469 */
4470 class VmaDeviceMemoryBlock
4471 {
4472  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
4473 public:
4474  VmaBlockMetadata m_Metadata;
4475 
4476  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
4477 
4478  ~VmaDeviceMemoryBlock()
4479  {
4480  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
4481  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
4482  }
4483 
4484  // Always call after construction.
4485  void Init(
4486  uint32_t newMemoryTypeIndex,
4487  VkDeviceMemory newMemory,
4488  VkDeviceSize newSize,
4489  uint32_t id);
4490  // Always call before destruction.
4491  void Destroy(VmaAllocator allocator);
4492 
4493  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
4494  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4495  uint32_t GetId() const { return m_Id; }
4496  void* GetMappedData() const { return m_pMappedData; }
4497 
4498  // Validates all data structures inside this object. If not valid, returns false.
4499  bool Validate() const;
4500 
4501  VkResult CheckCorruption(VmaAllocator hAllocator);
4502 
4503  // ppData can be null.
4504  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
4505  void Unmap(VmaAllocator hAllocator, uint32_t count);
4506 
4507  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4508  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4509 
4510  VkResult BindBufferMemory(
4511  const VmaAllocator hAllocator,
4512  const VmaAllocation hAllocation,
4513  VkBuffer hBuffer);
4514  VkResult BindImageMemory(
4515  const VmaAllocator hAllocator,
4516  const VmaAllocation hAllocation,
4517  VkImage hImage);
4518 
4519 private:
4520  uint32_t m_MemoryTypeIndex;
4521  uint32_t m_Id;
4522  VkDeviceMemory m_hMemory;
4523 
4524  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
4525  // Also protects m_MapCount, m_pMappedData.
4526  VMA_MUTEX m_Mutex;
4527  uint32_t m_MapCount;
4528  void* m_pMappedData;
4529 };
4530 
4531 struct VmaPointerLess
4532 {
4533  bool operator()(const void* lhs, const void* rhs) const
4534  {
4535  return lhs < rhs;
4536  }
4537 };
4538 
4539 class VmaDefragmentator;
4540 
4541 /*
4542 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
4543 Vulkan memory type.
4544 
4545 Synchronized internally with a mutex.
4546 */
4547 struct VmaBlockVector
4548 {
4549  VMA_CLASS_NO_COPY(VmaBlockVector)
4550 public:
4551  VmaBlockVector(
4552  VmaAllocator hAllocator,
4553  uint32_t memoryTypeIndex,
4554  VkDeviceSize preferredBlockSize,
4555  size_t minBlockCount,
4556  size_t maxBlockCount,
4557  VkDeviceSize bufferImageGranularity,
4558  uint32_t frameInUseCount,
4559  bool isCustomPool);
4560  ~VmaBlockVector();
4561 
4562  VkResult CreateMinBlocks();
4563 
4564  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4565  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
4566  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
4567  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
4568 
4569  void GetPoolStats(VmaPoolStats* pStats);
4570 
4571  bool IsEmpty() const { return m_Blocks.empty(); }
4572  bool IsCorruptionDetectionEnabled() const;
4573 
4574  VkResult Allocate(
4575  VmaPool hCurrentPool,
4576  uint32_t currentFrameIndex,
4577  VkDeviceSize size,
4578  VkDeviceSize alignment,
4579  const VmaAllocationCreateInfo& createInfo,
4580  VmaSuballocationType suballocType,
4581  VmaAllocation* pAllocation);
4582 
4583  void Free(
4584  VmaAllocation hAllocation);
4585 
4586  // Adds statistics of this BlockVector to pStats.
4587  void AddStats(VmaStats* pStats);
4588 
4589 #if VMA_STATS_STRING_ENABLED
4590  void PrintDetailedMap(class VmaJsonWriter& json);
4591 #endif
4592 
4593  void MakePoolAllocationsLost(
4594  uint32_t currentFrameIndex,
4595  size_t* pLostAllocationCount);
4596  VkResult CheckCorruption();
4597 
4598  VmaDefragmentator* EnsureDefragmentator(
4599  VmaAllocator hAllocator,
4600  uint32_t currentFrameIndex);
4601 
4602  VkResult Defragment(
4603  VmaDefragmentationStats* pDefragmentationStats,
4604  VkDeviceSize& maxBytesToMove,
4605  uint32_t& maxAllocationsToMove);
4606 
4607  void DestroyDefragmentator();
4608 
4609 private:
4610  friend class VmaDefragmentator;
4611 
4612  const VmaAllocator m_hAllocator;
4613  const uint32_t m_MemoryTypeIndex;
4614  const VkDeviceSize m_PreferredBlockSize;
4615  const size_t m_MinBlockCount;
4616  const size_t m_MaxBlockCount;
4617  const VkDeviceSize m_BufferImageGranularity;
4618  const uint32_t m_FrameInUseCount;
4619  const bool m_IsCustomPool;
4620  VMA_MUTEX m_Mutex;
4621  // Incrementally sorted by sumFreeSize, ascending.
4622  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4623  /* There can be at most one allocation that is completely empty - a
4624  hysteresis to avoid pessimistic case of alternating creation and destruction
4625  of a VkDeviceMemory. */
4626  bool m_HasEmptyBlock;
4627  VmaDefragmentator* m_pDefragmentator;
4628  uint32_t m_NextBlockId;
4629 
4630  VkDeviceSize CalcMaxBlockSize() const;
4631 
4632  // Finds and removes given block from vector.
4633  void Remove(VmaDeviceMemoryBlock* pBlock);
4634 
4635  // Performs single step in sorting m_Blocks. They may not be fully sorted
4636  // after this call.
4637  void IncrementallySortBlocks();
4638 
4639  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
4640 };
4641 
4642 struct VmaPool_T
4643 {
4644  VMA_CLASS_NO_COPY(VmaPool_T)
4645 public:
4646  VmaBlockVector m_BlockVector;
4647 
4648  VmaPool_T(
4649  VmaAllocator hAllocator,
4650  const VmaPoolCreateInfo& createInfo);
4651  ~VmaPool_T();
4652 
4653  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
4654  uint32_t GetId() const { return m_Id; }
4655  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
4656 
4657 #if VMA_STATS_STRING_ENABLED
4658  //void PrintDetailedMap(class VmaStringBuilder& sb);
4659 #endif
4660 
4661 private:
4662  uint32_t m_Id;
4663 };
4664 
4665 class VmaDefragmentator
4666 {
4667  VMA_CLASS_NO_COPY(VmaDefragmentator)
4668 private:
4669  const VmaAllocator m_hAllocator;
4670  VmaBlockVector* const m_pBlockVector;
4671  uint32_t m_CurrentFrameIndex;
4672  VkDeviceSize m_BytesMoved;
4673  uint32_t m_AllocationsMoved;
4674 
4675  struct AllocationInfo
4676  {
4677  VmaAllocation m_hAllocation;
4678  VkBool32* m_pChanged;
4679 
4680  AllocationInfo() :
4681  m_hAllocation(VK_NULL_HANDLE),
4682  m_pChanged(VMA_NULL)
4683  {
4684  }
4685  };
4686 
4687  struct AllocationInfoSizeGreater
4688  {
4689  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
4690  {
4691  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
4692  }
4693  };
4694 
4695  // Used between AddAllocation and Defragment.
4696  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4697 
4698  struct BlockInfo
4699  {
4700  VmaDeviceMemoryBlock* m_pBlock;
4701  bool m_HasNonMovableAllocations;
4702  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4703 
4704  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
4705  m_pBlock(VMA_NULL),
4706  m_HasNonMovableAllocations(true),
4707  m_Allocations(pAllocationCallbacks),
4708  m_pMappedDataForDefragmentation(VMA_NULL)
4709  {
4710  }
4711 
4712  void CalcHasNonMovableAllocations()
4713  {
4714  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
4715  const size_t defragmentAllocCount = m_Allocations.size();
4716  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
4717  }
4718 
4719  void SortAllocationsBySizeDescecnding()
4720  {
4721  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
4722  }
4723 
4724  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
4725  void Unmap(VmaAllocator hAllocator);
4726 
4727  private:
4728  // Not null if mapped for defragmentation only, not originally mapped.
4729  void* m_pMappedDataForDefragmentation;
4730  };
4731 
4732  struct BlockPointerLess
4733  {
4734  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
4735  {
4736  return pLhsBlockInfo->m_pBlock < pRhsBlock;
4737  }
4738  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4739  {
4740  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
4741  }
4742  };
4743 
4744  // 1. Blocks with some non-movable allocations go first.
4745  // 2. Blocks with smaller sumFreeSize go first.
4746  struct BlockInfoCompareMoveDestination
4747  {
4748  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4749  {
4750  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
4751  {
4752  return true;
4753  }
4754  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
4755  {
4756  return false;
4757  }
4758  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
4759  {
4760  return true;
4761  }
4762  return false;
4763  }
4764  };
4765 
4766  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
4767  BlockInfoVector m_Blocks;
4768 
4769  VkResult DefragmentRound(
4770  VkDeviceSize maxBytesToMove,
4771  uint32_t maxAllocationsToMove);
4772 
4773  static bool MoveMakesSense(
4774  size_t dstBlockIndex, VkDeviceSize dstOffset,
4775  size_t srcBlockIndex, VkDeviceSize srcOffset);
4776 
4777 public:
4778  VmaDefragmentator(
4779  VmaAllocator hAllocator,
4780  VmaBlockVector* pBlockVector,
4781  uint32_t currentFrameIndex);
4782 
4783  ~VmaDefragmentator();
4784 
4785  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
4786  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
4787 
4788  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
4789 
4790  VkResult Defragment(
4791  VkDeviceSize maxBytesToMove,
4792  uint32_t maxAllocationsToMove);
4793 };
4794 
4795 #if VMA_RECORDING_ENABLED
4796 
4797 class VmaRecorder
4798 {
4799 public:
4800  VmaRecorder();
4801  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
4802  void WriteConfiguration(
4803  const VkPhysicalDeviceProperties& devProps,
4804  const VkPhysicalDeviceMemoryProperties& memProps,
4805  bool dedicatedAllocationExtensionEnabled);
4806  ~VmaRecorder();
4807 
4808  void RecordCreateAllocator(uint32_t frameIndex);
4809  void RecordDestroyAllocator(uint32_t frameIndex);
4810  void RecordCreatePool(uint32_t frameIndex,
4811  const VmaPoolCreateInfo& createInfo,
4812  VmaPool pool);
4813  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
4814  void RecordAllocateMemory(uint32_t frameIndex,
4815  const VkMemoryRequirements& vkMemReq,
4816  const VmaAllocationCreateInfo& createInfo,
4817  VmaAllocation allocation);
4818  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
4819  const VkMemoryRequirements& vkMemReq,
4820  bool requiresDedicatedAllocation,
4821  bool prefersDedicatedAllocation,
4822  const VmaAllocationCreateInfo& createInfo,
4823  VmaAllocation allocation);
4824  void RecordAllocateMemoryForImage(uint32_t frameIndex,
4825  const VkMemoryRequirements& vkMemReq,
4826  bool requiresDedicatedAllocation,
4827  bool prefersDedicatedAllocation,
4828  const VmaAllocationCreateInfo& createInfo,
4829  VmaAllocation allocation);
4830  void RecordFreeMemory(uint32_t frameIndex,
4831  VmaAllocation allocation);
4832  void RecordSetAllocationUserData(uint32_t frameIndex,
4833  VmaAllocation allocation,
4834  const void* pUserData);
4835  void RecordCreateLostAllocation(uint32_t frameIndex,
4836  VmaAllocation allocation);
4837  void RecordMapMemory(uint32_t frameIndex,
4838  VmaAllocation allocation);
4839  void RecordUnmapMemory(uint32_t frameIndex,
4840  VmaAllocation allocation);
4841  void RecordFlushAllocation(uint32_t frameIndex,
4842  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
4843  void RecordInvalidateAllocation(uint32_t frameIndex,
4844  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
4845  void RecordCreateBuffer(uint32_t frameIndex,
4846  const VkBufferCreateInfo& bufCreateInfo,
4847  const VmaAllocationCreateInfo& allocCreateInfo,
4848  VmaAllocation allocation);
4849  void RecordCreateImage(uint32_t frameIndex,
4850  const VkImageCreateInfo& imageCreateInfo,
4851  const VmaAllocationCreateInfo& allocCreateInfo,
4852  VmaAllocation allocation);
4853  void RecordDestroyBuffer(uint32_t frameIndex,
4854  VmaAllocation allocation);
4855  void RecordDestroyImage(uint32_t frameIndex,
4856  VmaAllocation allocation);
4857  void RecordTouchAllocation(uint32_t frameIndex,
4858  VmaAllocation allocation);
4859  void RecordGetAllocationInfo(uint32_t frameIndex,
4860  VmaAllocation allocation);
4861  void RecordMakePoolAllocationsLost(uint32_t frameIndex,
4862  VmaPool pool);
4863 
4864 private:
4865  struct CallParams
4866  {
4867  uint32_t threadId;
4868  double time;
4869  };
4870 
4871  class UserDataString
4872  {
4873  public:
4874  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
4875  const char* GetString() const { return m_Str; }
4876 
4877  private:
4878  char m_PtrStr[17];
4879  const char* m_Str;
4880  };
4881 
4882  bool m_UseMutex;
4883  VmaRecordFlags m_Flags;
4884  FILE* m_File;
4885  VMA_MUTEX m_FileMutex;
4886  int64_t m_Freq;
4887  int64_t m_StartCounter;
4888 
4889  void GetBasicParams(CallParams& outParams);
4890  void Flush();
4891 };
4892 
4893 #endif // #if VMA_RECORDING_ENABLED
4894 
4895 // Main allocator object.
4896 struct VmaAllocator_T
4897 {
4898  VMA_CLASS_NO_COPY(VmaAllocator_T)
4899 public:
4900  bool m_UseMutex;
4901  bool m_UseKhrDedicatedAllocation;
4902  VkDevice m_hDevice;
4903  bool m_AllocationCallbacksSpecified;
4904  VkAllocationCallbacks m_AllocationCallbacks;
4905  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
4906 
4907  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
4908  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
4909  VMA_MUTEX m_HeapSizeLimitMutex;
4910 
4911  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
4912  VkPhysicalDeviceMemoryProperties m_MemProps;
4913 
4914  // Default pools.
4915  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
4916 
4917  // Each vector is sorted by memory (handle value).
4918  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4919  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4920  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4921 
4922  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4923  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
4924  ~VmaAllocator_T();
4925 
4926  const VkAllocationCallbacks* GetAllocationCallbacks() const
4927  {
4928  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4929  }
4930  const VmaVulkanFunctions& GetVulkanFunctions() const
4931  {
4932  return m_VulkanFunctions;
4933  }
4934 
4935  VkDeviceSize GetBufferImageGranularity() const
4936  {
4937  return VMA_MAX(
4938  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4939  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4940  }
4941 
4942  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4943  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4944 
4945  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4946  {
4947  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4948  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4949  }
4950  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
4951  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
4952  {
4953  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
4954  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
4955  }
4956  // Minimum alignment for all allocations in specific memory type.
4957  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
4958  {
4959  return IsMemoryTypeNonCoherent(memTypeIndex) ?
4960  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
4961  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
4962  }
4963 
4964  bool IsIntegratedGpu() const
4965  {
4966  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
4967  }
4968 
4969 #if VMA_RECORDING_ENABLED
4970  VmaRecorder* GetRecorder() const { return m_pRecorder; }
4971 #endif
4972 
4973  void GetBufferMemoryRequirements(
4974  VkBuffer hBuffer,
4975  VkMemoryRequirements& memReq,
4976  bool& requiresDedicatedAllocation,
4977  bool& prefersDedicatedAllocation) const;
4978  void GetImageMemoryRequirements(
4979  VkImage hImage,
4980  VkMemoryRequirements& memReq,
4981  bool& requiresDedicatedAllocation,
4982  bool& prefersDedicatedAllocation) const;
4983 
4984  // Main allocation function.
4985  VkResult AllocateMemory(
4986  const VkMemoryRequirements& vkMemReq,
4987  bool requiresDedicatedAllocation,
4988  bool prefersDedicatedAllocation,
4989  VkBuffer dedicatedBuffer,
4990  VkImage dedicatedImage,
4991  const VmaAllocationCreateInfo& createInfo,
4992  VmaSuballocationType suballocType,
4993  VmaAllocation* pAllocation);
4994 
4995  // Main deallocation function.
4996  void FreeMemory(const VmaAllocation allocation);
4997 
4998  void CalculateStats(VmaStats* pStats);
4999 
5000 #if VMA_STATS_STRING_ENABLED
5001  void PrintDetailedMap(class VmaJsonWriter& json);
5002 #endif
5003 
5004  VkResult Defragment(
5005  VmaAllocation* pAllocations,
5006  size_t allocationCount,
5007  VkBool32* pAllocationsChanged,
5008  const VmaDefragmentationInfo* pDefragmentationInfo,
5009  VmaDefragmentationStats* pDefragmentationStats);
5010 
5011  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
5012  bool TouchAllocation(VmaAllocation hAllocation);
5013 
5014  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
5015  void DestroyPool(VmaPool pool);
5016  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
5017 
5018  void SetCurrentFrameIndex(uint32_t frameIndex);
5019  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
5020 
5021  void MakePoolAllocationsLost(
5022  VmaPool hPool,
5023  size_t* pLostAllocationCount);
5024  VkResult CheckPoolCorruption(VmaPool hPool);
5025  VkResult CheckCorruption(uint32_t memoryTypeBits);
5026 
5027  void CreateLostAllocation(VmaAllocation* pAllocation);
5028 
5029  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
5030  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
5031 
5032  VkResult Map(VmaAllocation hAllocation, void** ppData);
5033  void Unmap(VmaAllocation hAllocation);
5034 
5035  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
5036  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
5037 
5038  void FlushOrInvalidateAllocation(
5039  VmaAllocation hAllocation,
5040  VkDeviceSize offset, VkDeviceSize size,
5041  VMA_CACHE_OPERATION op);
5042 
5043  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
5044 
5045 private:
5046  VkDeviceSize m_PreferredLargeHeapBlockSize;
5047 
5048  VkPhysicalDevice m_PhysicalDevice;
5049  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
5050 
5051  VMA_MUTEX m_PoolsMutex;
5052  // Protected by m_PoolsMutex. Sorted by pointer value.
5053  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
5054  uint32_t m_NextPoolId;
5055 
5056  VmaVulkanFunctions m_VulkanFunctions;
5057 
5058 #if VMA_RECORDING_ENABLED
5059  VmaRecorder* m_pRecorder;
5060 #endif
5061 
5062  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
5063 
5064  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
5065 
5066  VkResult AllocateMemoryOfType(
5067  VkDeviceSize size,
5068  VkDeviceSize alignment,
5069  bool dedicatedAllocation,
5070  VkBuffer dedicatedBuffer,
5071  VkImage dedicatedImage,
5072  const VmaAllocationCreateInfo& createInfo,
5073  uint32_t memTypeIndex,
5074  VmaSuballocationType suballocType,
5075  VmaAllocation* pAllocation);
5076 
5077  // Allocates and registers new VkDeviceMemory specifically for single allocation.
5078  VkResult AllocateDedicatedMemory(
5079  VkDeviceSize size,
5080  VmaSuballocationType suballocType,
5081  uint32_t memTypeIndex,
5082  bool map,
5083  bool isUserDataString,
5084  void* pUserData,
5085  VkBuffer dedicatedBuffer,
5086  VkImage dedicatedImage,
5087  VmaAllocation* pAllocation);
5088 
5089  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
5090  void FreeDedicatedMemory(VmaAllocation allocation);
5091 };
5092 
5094 // Memory allocation #2 after VmaAllocator_T definition
5095 
5096 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
5097 {
5098  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
5099 }
5100 
5101 static void VmaFree(VmaAllocator hAllocator, void* ptr)
5102 {
5103  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
5104 }
5105 
5106 template<typename T>
5107 static T* VmaAllocate(VmaAllocator hAllocator)
5108 {
5109  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
5110 }
5111 
5112 template<typename T>
5113 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
5114 {
5115  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
5116 }
5117 
5118 template<typename T>
5119 static void vma_delete(VmaAllocator hAllocator, T* ptr)
5120 {
5121  if(ptr != VMA_NULL)
5122  {
5123  ptr->~T();
5124  VmaFree(hAllocator, ptr);
5125  }
5126 }
5127 
5128 template<typename T>
5129 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
5130 {
5131  if(ptr != VMA_NULL)
5132  {
5133  for(size_t i = count; i--; )
5134  ptr[i].~T();
5135  VmaFree(hAllocator, ptr);
5136  }
5137 }
5138 
5140 // VmaStringBuilder
5141 
5142 #if VMA_STATS_STRING_ENABLED
5143 
5144 class VmaStringBuilder
5145 {
5146 public:
5147  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
5148  size_t GetLength() const { return m_Data.size(); }
5149  const char* GetData() const { return m_Data.data(); }
5150 
5151  void Add(char ch) { m_Data.push_back(ch); }
5152  void Add(const char* pStr);
5153  void AddNewLine() { Add('\n'); }
5154  void AddNumber(uint32_t num);
5155  void AddNumber(uint64_t num);
5156  void AddPointer(const void* ptr);
5157 
5158 private:
5159  VmaVector< char, VmaStlAllocator<char> > m_Data;
5160 };
5161 
5162 void VmaStringBuilder::Add(const char* pStr)
5163 {
5164  const size_t strLen = strlen(pStr);
5165  if(strLen > 0)
5166  {
5167  const size_t oldCount = m_Data.size();
5168  m_Data.resize(oldCount + strLen);
5169  memcpy(m_Data.data() + oldCount, pStr, strLen);
5170  }
5171 }
5172 
5173 void VmaStringBuilder::AddNumber(uint32_t num)
5174 {
5175  char buf[11];
5176  VmaUint32ToStr(buf, sizeof(buf), num);
5177  Add(buf);
5178 }
5179 
5180 void VmaStringBuilder::AddNumber(uint64_t num)
5181 {
5182  char buf[21];
5183  VmaUint64ToStr(buf, sizeof(buf), num);
5184  Add(buf);
5185 }
5186 
5187 void VmaStringBuilder::AddPointer(const void* ptr)
5188 {
5189  char buf[21];
5190  VmaPtrToStr(buf, sizeof(buf), ptr);
5191  Add(buf);
5192 }
5193 
5194 #endif // #if VMA_STATS_STRING_ENABLED
5195 
5197 // VmaJsonWriter
5198 
5199 #if VMA_STATS_STRING_ENABLED
5200 
5201 class VmaJsonWriter
5202 {
5203  VMA_CLASS_NO_COPY(VmaJsonWriter)
5204 public:
5205  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
5206  ~VmaJsonWriter();
5207 
5208  void BeginObject(bool singleLine = false);
5209  void EndObject();
5210 
5211  void BeginArray(bool singleLine = false);
5212  void EndArray();
5213 
5214  void WriteString(const char* pStr);
5215  void BeginString(const char* pStr = VMA_NULL);
5216  void ContinueString(const char* pStr);
5217  void ContinueString(uint32_t n);
5218  void ContinueString(uint64_t n);
5219  void ContinueString_Pointer(const void* ptr);
5220  void EndString(const char* pStr = VMA_NULL);
5221 
5222  void WriteNumber(uint32_t n);
5223  void WriteNumber(uint64_t n);
5224  void WriteBool(bool b);
5225  void WriteNull();
5226 
5227 private:
5228  static const char* const INDENT;
5229 
5230  enum COLLECTION_TYPE
5231  {
5232  COLLECTION_TYPE_OBJECT,
5233  COLLECTION_TYPE_ARRAY,
5234  };
5235  struct StackItem
5236  {
5237  COLLECTION_TYPE type;
5238  uint32_t valueCount;
5239  bool singleLineMode;
5240  };
5241 
5242  VmaStringBuilder& m_SB;
5243  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
5244  bool m_InsideString;
5245 
5246  void BeginValue(bool isString);
5247  void WriteIndent(bool oneLess = false);
5248 };
5249 
5250 const char* const VmaJsonWriter::INDENT = " ";
5251 
5252 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
5253  m_SB(sb),
5254  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
5255  m_InsideString(false)
5256 {
5257 }
5258 
5259 VmaJsonWriter::~VmaJsonWriter()
5260 {
5261  VMA_ASSERT(!m_InsideString);
5262  VMA_ASSERT(m_Stack.empty());
5263 }
5264 
5265 void VmaJsonWriter::BeginObject(bool singleLine)
5266 {
5267  VMA_ASSERT(!m_InsideString);
5268 
5269  BeginValue(false);
5270  m_SB.Add('{');
5271 
5272  StackItem item;
5273  item.type = COLLECTION_TYPE_OBJECT;
5274  item.valueCount = 0;
5275  item.singleLineMode = singleLine;
5276  m_Stack.push_back(item);
5277 }
5278 
5279 void VmaJsonWriter::EndObject()
5280 {
5281  VMA_ASSERT(!m_InsideString);
5282 
5283  WriteIndent(true);
5284  m_SB.Add('}');
5285 
5286  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
5287  m_Stack.pop_back();
5288 }
5289 
5290 void VmaJsonWriter::BeginArray(bool singleLine)
5291 {
5292  VMA_ASSERT(!m_InsideString);
5293 
5294  BeginValue(false);
5295  m_SB.Add('[');
5296 
5297  StackItem item;
5298  item.type = COLLECTION_TYPE_ARRAY;
5299  item.valueCount = 0;
5300  item.singleLineMode = singleLine;
5301  m_Stack.push_back(item);
5302 }
5303 
5304 void VmaJsonWriter::EndArray()
5305 {
5306  VMA_ASSERT(!m_InsideString);
5307 
5308  WriteIndent(true);
5309  m_SB.Add(']');
5310 
5311  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
5312  m_Stack.pop_back();
5313 }
5314 
5315 void VmaJsonWriter::WriteString(const char* pStr)
5316 {
5317  BeginString(pStr);
5318  EndString();
5319 }
5320 
5321 void VmaJsonWriter::BeginString(const char* pStr)
5322 {
5323  VMA_ASSERT(!m_InsideString);
5324 
5325  BeginValue(true);
5326  m_SB.Add('"');
5327  m_InsideString = true;
5328  if(pStr != VMA_NULL && pStr[0] != '\0')
5329  {
5330  ContinueString(pStr);
5331  }
5332 }
5333 
5334 void VmaJsonWriter::ContinueString(const char* pStr)
5335 {
5336  VMA_ASSERT(m_InsideString);
5337 
5338  const size_t strLen = strlen(pStr);
5339  for(size_t i = 0; i < strLen; ++i)
5340  {
5341  char ch = pStr[i];
5342  if(ch == '\\')
5343  {
5344  m_SB.Add("\\\\");
5345  }
5346  else if(ch == '"')
5347  {
5348  m_SB.Add("\\\"");
5349  }
5350  else if(ch >= 32)
5351  {
5352  m_SB.Add(ch);
5353  }
5354  else switch(ch)
5355  {
5356  case '\b':
5357  m_SB.Add("\\b");
5358  break;
5359  case '\f':
5360  m_SB.Add("\\f");
5361  break;
5362  case '\n':
5363  m_SB.Add("\\n");
5364  break;
5365  case '\r':
5366  m_SB.Add("\\r");
5367  break;
5368  case '\t':
5369  m_SB.Add("\\t");
5370  break;
5371  default:
5372  VMA_ASSERT(0 && "Character not currently supported.");
5373  break;
5374  }
5375  }
5376 }
5377 
5378 void VmaJsonWriter::ContinueString(uint32_t n)
5379 {
5380  VMA_ASSERT(m_InsideString);
5381  m_SB.AddNumber(n);
5382 }
5383 
5384 void VmaJsonWriter::ContinueString(uint64_t n)
5385 {
5386  VMA_ASSERT(m_InsideString);
5387  m_SB.AddNumber(n);
5388 }
5389 
5390 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
5391 {
5392  VMA_ASSERT(m_InsideString);
5393  m_SB.AddPointer(ptr);
5394 }
5395 
5396 void VmaJsonWriter::EndString(const char* pStr)
5397 {
5398  VMA_ASSERT(m_InsideString);
5399  if(pStr != VMA_NULL && pStr[0] != '\0')
5400  {
5401  ContinueString(pStr);
5402  }
5403  m_SB.Add('"');
5404  m_InsideString = false;
5405 }
5406 
5407 void VmaJsonWriter::WriteNumber(uint32_t n)
5408 {
5409  VMA_ASSERT(!m_InsideString);
5410  BeginValue(false);
5411  m_SB.AddNumber(n);
5412 }
5413 
5414 void VmaJsonWriter::WriteNumber(uint64_t n)
5415 {
5416  VMA_ASSERT(!m_InsideString);
5417  BeginValue(false);
5418  m_SB.AddNumber(n);
5419 }
5420 
5421 void VmaJsonWriter::WriteBool(bool b)
5422 {
5423  VMA_ASSERT(!m_InsideString);
5424  BeginValue(false);
5425  m_SB.Add(b ? "true" : "false");
5426 }
5427 
5428 void VmaJsonWriter::WriteNull()
5429 {
5430  VMA_ASSERT(!m_InsideString);
5431  BeginValue(false);
5432  m_SB.Add("null");
5433 }
5434 
5435 void VmaJsonWriter::BeginValue(bool isString)
5436 {
5437  if(!m_Stack.empty())
5438  {
5439  StackItem& currItem = m_Stack.back();
5440  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5441  currItem.valueCount % 2 == 0)
5442  {
5443  VMA_ASSERT(isString);
5444  }
5445 
5446  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5447  currItem.valueCount % 2 != 0)
5448  {
5449  m_SB.Add(": ");
5450  }
5451  else if(currItem.valueCount > 0)
5452  {
5453  m_SB.Add(", ");
5454  WriteIndent();
5455  }
5456  else
5457  {
5458  WriteIndent();
5459  }
5460  ++currItem.valueCount;
5461  }
5462 }
5463 
5464 void VmaJsonWriter::WriteIndent(bool oneLess)
5465 {
5466  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
5467  {
5468  m_SB.AddNewLine();
5469 
5470  size_t count = m_Stack.size();
5471  if(count > 0 && oneLess)
5472  {
5473  --count;
5474  }
5475  for(size_t i = 0; i < count; ++i)
5476  {
5477  m_SB.Add(INDENT);
5478  }
5479  }
5480 }
5481 
5482 #endif // #if VMA_STATS_STRING_ENABLED
5483 
5485 
5486 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
5487 {
5488  if(IsUserDataString())
5489  {
5490  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
5491 
5492  FreeUserDataString(hAllocator);
5493 
5494  if(pUserData != VMA_NULL)
5495  {
5496  const char* const newStrSrc = (char*)pUserData;
5497  const size_t newStrLen = strlen(newStrSrc);
5498  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
5499  memcpy(newStrDst, newStrSrc, newStrLen + 1);
5500  m_pUserData = newStrDst;
5501  }
5502  }
5503  else
5504  {
5505  m_pUserData = pUserData;
5506  }
5507 }
5508 
5509 void VmaAllocation_T::ChangeBlockAllocation(
5510  VmaAllocator hAllocator,
5511  VmaDeviceMemoryBlock* block,
5512  VkDeviceSize offset)
5513 {
5514  VMA_ASSERT(block != VMA_NULL);
5515  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5516 
5517  // Move mapping reference counter from old block to new block.
5518  if(block != m_BlockAllocation.m_Block)
5519  {
5520  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
5521  if(IsPersistentMap())
5522  ++mapRefCount;
5523  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
5524  block->Map(hAllocator, mapRefCount, VMA_NULL);
5525  }
5526 
5527  m_BlockAllocation.m_Block = block;
5528  m_BlockAllocation.m_Offset = offset;
5529 }
5530 
5531 VkDeviceSize VmaAllocation_T::GetOffset() const
5532 {
5533  switch(m_Type)
5534  {
5535  case ALLOCATION_TYPE_BLOCK:
5536  return m_BlockAllocation.m_Offset;
5537  case ALLOCATION_TYPE_DEDICATED:
5538  return 0;
5539  default:
5540  VMA_ASSERT(0);
5541  return 0;
5542  }
5543 }
5544 
5545 VkDeviceMemory VmaAllocation_T::GetMemory() const
5546 {
5547  switch(m_Type)
5548  {
5549  case ALLOCATION_TYPE_BLOCK:
5550  return m_BlockAllocation.m_Block->GetDeviceMemory();
5551  case ALLOCATION_TYPE_DEDICATED:
5552  return m_DedicatedAllocation.m_hMemory;
5553  default:
5554  VMA_ASSERT(0);
5555  return VK_NULL_HANDLE;
5556  }
5557 }
5558 
5559 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
5560 {
5561  switch(m_Type)
5562  {
5563  case ALLOCATION_TYPE_BLOCK:
5564  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
5565  case ALLOCATION_TYPE_DEDICATED:
5566  return m_DedicatedAllocation.m_MemoryTypeIndex;
5567  default:
5568  VMA_ASSERT(0);
5569  return UINT32_MAX;
5570  }
5571 }
5572 
5573 void* VmaAllocation_T::GetMappedData() const
5574 {
5575  switch(m_Type)
5576  {
5577  case ALLOCATION_TYPE_BLOCK:
5578  if(m_MapCount != 0)
5579  {
5580  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
5581  VMA_ASSERT(pBlockData != VMA_NULL);
5582  return (char*)pBlockData + m_BlockAllocation.m_Offset;
5583  }
5584  else
5585  {
5586  return VMA_NULL;
5587  }
5588  break;
5589  case ALLOCATION_TYPE_DEDICATED:
5590  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
5591  return m_DedicatedAllocation.m_pMappedData;
5592  default:
5593  VMA_ASSERT(0);
5594  return VMA_NULL;
5595  }
5596 }
5597 
5598 bool VmaAllocation_T::CanBecomeLost() const
5599 {
5600  switch(m_Type)
5601  {
5602  case ALLOCATION_TYPE_BLOCK:
5603  return m_BlockAllocation.m_CanBecomeLost;
5604  case ALLOCATION_TYPE_DEDICATED:
5605  return false;
5606  default:
5607  VMA_ASSERT(0);
5608  return false;
5609  }
5610 }
5611 
5612 VmaPool VmaAllocation_T::GetPool() const
5613 {
5614  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5615  return m_BlockAllocation.m_hPool;
5616 }
5617 
5618 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5619 {
5620  VMA_ASSERT(CanBecomeLost());
5621 
5622  /*
5623  Warning: This is a carefully designed algorithm.
5624  Do not modify unless you really know what you're doing :)
5625  */
5626  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
5627  for(;;)
5628  {
5629  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
5630  {
5631  VMA_ASSERT(0);
5632  return false;
5633  }
5634  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
5635  {
5636  return false;
5637  }
5638  else // Last use time earlier than current time.
5639  {
5640  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
5641  {
5642  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
5643  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
5644  return true;
5645  }
5646  }
5647  }
5648 }
5649 
5650 #if VMA_STATS_STRING_ENABLED
5651 
5652 // Correspond to values of enum VmaSuballocationType.
5653 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
5654  "FREE",
5655  "UNKNOWN",
5656  "BUFFER",
5657  "IMAGE_UNKNOWN",
5658  "IMAGE_LINEAR",
5659  "IMAGE_OPTIMAL",
5660 };
5661 
5662 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
5663 {
5664  json.WriteString("Type");
5665  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
5666 
5667  json.WriteString("Size");
5668  json.WriteNumber(m_Size);
5669 
5670  if(m_pUserData != VMA_NULL)
5671  {
5672  json.WriteString("UserData");
5673  if(IsUserDataString())
5674  {
5675  json.WriteString((const char*)m_pUserData);
5676  }
5677  else
5678  {
5679  json.BeginString();
5680  json.ContinueString_Pointer(m_pUserData);
5681  json.EndString();
5682  }
5683  }
5684 
5685  json.WriteString("CreationFrameIndex");
5686  json.WriteNumber(m_CreationFrameIndex);
5687 
5688  json.WriteString("LastUseFrameIndex");
5689  json.WriteNumber(GetLastUseFrameIndex());
5690 
5691  if(m_BufferImageUsage != 0)
5692  {
5693  json.WriteString("Usage");
5694  json.WriteNumber(m_BufferImageUsage);
5695  }
5696 }
5697 
5698 #endif
5699 
5700 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
5701 {
5702  VMA_ASSERT(IsUserDataString());
5703  if(m_pUserData != VMA_NULL)
5704  {
5705  char* const oldStr = (char*)m_pUserData;
5706  const size_t oldStrLen = strlen(oldStr);
5707  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
5708  m_pUserData = VMA_NULL;
5709  }
5710 }
5711 
5712 void VmaAllocation_T::BlockAllocMap()
5713 {
5714  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5715 
5716  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5717  {
5718  ++m_MapCount;
5719  }
5720  else
5721  {
5722  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
5723  }
5724 }
5725 
5726 void VmaAllocation_T::BlockAllocUnmap()
5727 {
5728  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5729 
5730  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5731  {
5732  --m_MapCount;
5733  }
5734  else
5735  {
5736  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
5737  }
5738 }
5739 
5740 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
5741 {
5742  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5743 
5744  if(m_MapCount != 0)
5745  {
5746  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5747  {
5748  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
5749  *ppData = m_DedicatedAllocation.m_pMappedData;
5750  ++m_MapCount;
5751  return VK_SUCCESS;
5752  }
5753  else
5754  {
5755  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
5756  return VK_ERROR_MEMORY_MAP_FAILED;
5757  }
5758  }
5759  else
5760  {
5761  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5762  hAllocator->m_hDevice,
5763  m_DedicatedAllocation.m_hMemory,
5764  0, // offset
5765  VK_WHOLE_SIZE,
5766  0, // flags
5767  ppData);
5768  if(result == VK_SUCCESS)
5769  {
5770  m_DedicatedAllocation.m_pMappedData = *ppData;
5771  m_MapCount = 1;
5772  }
5773  return result;
5774  }
5775 }
5776 
5777 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
5778 {
5779  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5780 
5781  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5782  {
5783  --m_MapCount;
5784  if(m_MapCount == 0)
5785  {
5786  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
5787  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
5788  hAllocator->m_hDevice,
5789  m_DedicatedAllocation.m_hMemory);
5790  }
5791  }
5792  else
5793  {
5794  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
5795  }
5796 }
5797 
5798 #if VMA_STATS_STRING_ENABLED
5799 
5800 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
5801 {
5802  json.BeginObject();
5803 
5804  json.WriteString("Blocks");
5805  json.WriteNumber(stat.blockCount);
5806 
5807  json.WriteString("Allocations");
5808  json.WriteNumber(stat.allocationCount);
5809 
5810  json.WriteString("UnusedRanges");
5811  json.WriteNumber(stat.unusedRangeCount);
5812 
5813  json.WriteString("UsedBytes");
5814  json.WriteNumber(stat.usedBytes);
5815 
5816  json.WriteString("UnusedBytes");
5817  json.WriteNumber(stat.unusedBytes);
5818 
5819  if(stat.allocationCount > 1)
5820  {
5821  json.WriteString("AllocationSize");
5822  json.BeginObject(true);
5823  json.WriteString("Min");
5824  json.WriteNumber(stat.allocationSizeMin);
5825  json.WriteString("Avg");
5826  json.WriteNumber(stat.allocationSizeAvg);
5827  json.WriteString("Max");
5828  json.WriteNumber(stat.allocationSizeMax);
5829  json.EndObject();
5830  }
5831 
5832  if(stat.unusedRangeCount > 1)
5833  {
5834  json.WriteString("UnusedRangeSize");
5835  json.BeginObject(true);
5836  json.WriteString("Min");
5837  json.WriteNumber(stat.unusedRangeSizeMin);
5838  json.WriteString("Avg");
5839  json.WriteNumber(stat.unusedRangeSizeAvg);
5840  json.WriteString("Max");
5841  json.WriteNumber(stat.unusedRangeSizeMax);
5842  json.EndObject();
5843  }
5844 
5845  json.EndObject();
5846 }
5847 
5848 #endif // #if VMA_STATS_STRING_ENABLED
5849 
5850 struct VmaSuballocationItemSizeLess
5851 {
5852  bool operator()(
5853  const VmaSuballocationList::iterator lhs,
5854  const VmaSuballocationList::iterator rhs) const
5855  {
5856  return lhs->size < rhs->size;
5857  }
5858  bool operator()(
5859  const VmaSuballocationList::iterator lhs,
5860  VkDeviceSize rhsSize) const
5861  {
5862  return lhs->size < rhsSize;
5863  }
5864 };
5865 
5867 // class VmaBlockMetadata
5868 
5869 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
5870  m_Size(0),
5871  m_FreeCount(0),
5872  m_SumFreeSize(0),
5873  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
5874  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
5875 {
5876 }
5877 
5878 VmaBlockMetadata::~VmaBlockMetadata()
5879 {
5880 }
5881 
5882 void VmaBlockMetadata::Init(VkDeviceSize size)
5883 {
5884  m_Size = size;
5885  m_FreeCount = 1;
5886  m_SumFreeSize = size;
5887 
5888  VmaSuballocation suballoc = {};
5889  suballoc.offset = 0;
5890  suballoc.size = size;
5891  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5892  suballoc.hAllocation = VK_NULL_HANDLE;
5893 
5894  m_Suballocations.push_back(suballoc);
5895  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
5896  --suballocItem;
5897  m_FreeSuballocationsBySize.push_back(suballocItem);
5898 }
5899 
5900 bool VmaBlockMetadata::Validate() const
5901 {
5902  if(m_Suballocations.empty())
5903  {
5904  return false;
5905  }
5906 
5907  // Expected offset of new suballocation as calculates from previous ones.
5908  VkDeviceSize calculatedOffset = 0;
5909  // Expected number of free suballocations as calculated from traversing their list.
5910  uint32_t calculatedFreeCount = 0;
5911  // Expected sum size of free suballocations as calculated from traversing their list.
5912  VkDeviceSize calculatedSumFreeSize = 0;
5913  // Expected number of free suballocations that should be registered in
5914  // m_FreeSuballocationsBySize calculated from traversing their list.
5915  size_t freeSuballocationsToRegister = 0;
5916  // True if previous visited suballocation was free.
5917  bool prevFree = false;
5918 
5919  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5920  suballocItem != m_Suballocations.cend();
5921  ++suballocItem)
5922  {
5923  const VmaSuballocation& subAlloc = *suballocItem;
5924 
5925  // Actual offset of this suballocation doesn't match expected one.
5926  if(subAlloc.offset != calculatedOffset)
5927  {
5928  return false;
5929  }
5930 
5931  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
5932  // Two adjacent free suballocations are invalid. They should be merged.
5933  if(prevFree && currFree)
5934  {
5935  return false;
5936  }
5937 
5938  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
5939  {
5940  return false;
5941  }
5942 
5943  if(currFree)
5944  {
5945  calculatedSumFreeSize += subAlloc.size;
5946  ++calculatedFreeCount;
5947  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5948  {
5949  ++freeSuballocationsToRegister;
5950  }
5951 
5952  // Margin required between allocations - every free space must be at least that large.
5953  if(subAlloc.size < VMA_DEBUG_MARGIN)
5954  {
5955  return false;
5956  }
5957  }
5958  else
5959  {
5960  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
5961  {
5962  return false;
5963  }
5964  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
5965  {
5966  return false;
5967  }
5968 
5969  // Margin required between allocations - previous allocation must be free.
5970  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
5971  {
5972  return false;
5973  }
5974  }
5975 
5976  calculatedOffset += subAlloc.size;
5977  prevFree = currFree;
5978  }
5979 
5980  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
5981  // match expected one.
5982  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
5983  {
5984  return false;
5985  }
5986 
5987  VkDeviceSize lastSize = 0;
5988  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
5989  {
5990  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
5991 
5992  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
5993  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5994  {
5995  return false;
5996  }
5997  // They must be sorted by size ascending.
5998  if(suballocItem->size < lastSize)
5999  {
6000  return false;
6001  }
6002 
6003  lastSize = suballocItem->size;
6004  }
6005 
6006  // Check if totals match calculacted values.
6007  if(!ValidateFreeSuballocationList() ||
6008  (calculatedOffset != m_Size) ||
6009  (calculatedSumFreeSize != m_SumFreeSize) ||
6010  (calculatedFreeCount != m_FreeCount))
6011  {
6012  return false;
6013  }
6014 
6015  return true;
6016 }
6017 
6018 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
6019 {
6020  if(!m_FreeSuballocationsBySize.empty())
6021  {
6022  return m_FreeSuballocationsBySize.back()->size;
6023  }
6024  else
6025  {
6026  return 0;
6027  }
6028 }
6029 
6030 bool VmaBlockMetadata::IsEmpty() const
6031 {
6032  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
6033 }
6034 
6035 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
6036 {
6037  outInfo.blockCount = 1;
6038 
6039  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6040  outInfo.allocationCount = rangeCount - m_FreeCount;
6041  outInfo.unusedRangeCount = m_FreeCount;
6042 
6043  outInfo.unusedBytes = m_SumFreeSize;
6044  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
6045 
6046  outInfo.allocationSizeMin = UINT64_MAX;
6047  outInfo.allocationSizeMax = 0;
6048  outInfo.unusedRangeSizeMin = UINT64_MAX;
6049  outInfo.unusedRangeSizeMax = 0;
6050 
6051  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6052  suballocItem != m_Suballocations.cend();
6053  ++suballocItem)
6054  {
6055  const VmaSuballocation& suballoc = *suballocItem;
6056  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
6057  {
6058  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
6059  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
6060  }
6061  else
6062  {
6063  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
6064  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
6065  }
6066  }
6067 }
6068 
6069 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
6070 {
6071  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6072 
6073  inoutStats.size += m_Size;
6074  inoutStats.unusedSize += m_SumFreeSize;
6075  inoutStats.allocationCount += rangeCount - m_FreeCount;
6076  inoutStats.unusedRangeCount += m_FreeCount;
6077  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
6078 }
6079 
6080 #if VMA_STATS_STRING_ENABLED
6081 
6082 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
6083 {
6084  json.BeginObject();
6085 
6086  json.WriteString("TotalBytes");
6087  json.WriteNumber(m_Size);
6088 
6089  json.WriteString("UnusedBytes");
6090  json.WriteNumber(m_SumFreeSize);
6091 
6092  json.WriteString("Allocations");
6093  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
6094 
6095  json.WriteString("UnusedRanges");
6096  json.WriteNumber(m_FreeCount);
6097 
6098  json.WriteString("Suballocations");
6099  json.BeginArray();
6100  size_t i = 0;
6101  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6102  suballocItem != m_Suballocations.cend();
6103  ++suballocItem, ++i)
6104  {
6105  json.BeginObject(true);
6106 
6107  json.WriteString("Offset");
6108  json.WriteNumber(suballocItem->offset);
6109 
6110  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6111  {
6112  json.WriteString("Type");
6113  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
6114 
6115  json.WriteString("Size");
6116  json.WriteNumber(suballocItem->size);
6117  }
6118  else
6119  {
6120  suballocItem->hAllocation->PrintParameters(json);
6121  }
6122 
6123  json.EndObject();
6124  }
6125  json.EndArray();
6126 
6127  json.EndObject();
6128 }
6129 
6130 #endif // #if VMA_STATS_STRING_ENABLED
6131 
6132 /*
6133 How many suitable free suballocations to analyze before choosing best one.
6134 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
6135  be chosen.
6136 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
6137  suballocations will be analized and best one will be chosen.
6138 - Any other value is also acceptable.
6139 */
6140 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
6141 
6142 bool VmaBlockMetadata::CreateAllocationRequest(
6143  uint32_t currentFrameIndex,
6144  uint32_t frameInUseCount,
6145  VkDeviceSize bufferImageGranularity,
6146  VkDeviceSize allocSize,
6147  VkDeviceSize allocAlignment,
6148  VmaSuballocationType allocType,
6149  bool canMakeOtherLost,
6150  VmaAllocationRequest* pAllocationRequest)
6151 {
6152  VMA_ASSERT(allocSize > 0);
6153  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6154  VMA_ASSERT(pAllocationRequest != VMA_NULL);
6155  VMA_HEAVY_ASSERT(Validate());
6156 
6157  // There is not enough total free space in this block to fullfill the request: Early return.
6158  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
6159  {
6160  return false;
6161  }
6162 
6163  // New algorithm, efficiently searching freeSuballocationsBySize.
6164  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
6165  if(freeSuballocCount > 0)
6166  {
6167  if(VMA_BEST_FIT)
6168  {
6169  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
6170  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6171  m_FreeSuballocationsBySize.data(),
6172  m_FreeSuballocationsBySize.data() + freeSuballocCount,
6173  allocSize + 2 * VMA_DEBUG_MARGIN,
6174  VmaSuballocationItemSizeLess());
6175  size_t index = it - m_FreeSuballocationsBySize.data();
6176  for(; index < freeSuballocCount; ++index)
6177  {
6178  if(CheckAllocation(
6179  currentFrameIndex,
6180  frameInUseCount,
6181  bufferImageGranularity,
6182  allocSize,
6183  allocAlignment,
6184  allocType,
6185  m_FreeSuballocationsBySize[index],
6186  false, // canMakeOtherLost
6187  &pAllocationRequest->offset,
6188  &pAllocationRequest->itemsToMakeLostCount,
6189  &pAllocationRequest->sumFreeSize,
6190  &pAllocationRequest->sumItemSize))
6191  {
6192  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6193  return true;
6194  }
6195  }
6196  }
6197  else
6198  {
6199  // Search staring from biggest suballocations.
6200  for(size_t index = freeSuballocCount; index--; )
6201  {
6202  if(CheckAllocation(
6203  currentFrameIndex,
6204  frameInUseCount,
6205  bufferImageGranularity,
6206  allocSize,
6207  allocAlignment,
6208  allocType,
6209  m_FreeSuballocationsBySize[index],
6210  false, // canMakeOtherLost
6211  &pAllocationRequest->offset,
6212  &pAllocationRequest->itemsToMakeLostCount,
6213  &pAllocationRequest->sumFreeSize,
6214  &pAllocationRequest->sumItemSize))
6215  {
6216  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6217  return true;
6218  }
6219  }
6220  }
6221  }
6222 
6223  if(canMakeOtherLost)
6224  {
6225  // Brute-force algorithm. TODO: Come up with something better.
6226 
6227  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
6228  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
6229 
6230  VmaAllocationRequest tmpAllocRequest = {};
6231  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
6232  suballocIt != m_Suballocations.end();
6233  ++suballocIt)
6234  {
6235  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
6236  suballocIt->hAllocation->CanBecomeLost())
6237  {
6238  if(CheckAllocation(
6239  currentFrameIndex,
6240  frameInUseCount,
6241  bufferImageGranularity,
6242  allocSize,
6243  allocAlignment,
6244  allocType,
6245  suballocIt,
6246  canMakeOtherLost,
6247  &tmpAllocRequest.offset,
6248  &tmpAllocRequest.itemsToMakeLostCount,
6249  &tmpAllocRequest.sumFreeSize,
6250  &tmpAllocRequest.sumItemSize))
6251  {
6252  tmpAllocRequest.item = suballocIt;
6253 
6254  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
6255  {
6256  *pAllocationRequest = tmpAllocRequest;
6257  }
6258  }
6259  }
6260  }
6261 
6262  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
6263  {
6264  return true;
6265  }
6266  }
6267 
6268  return false;
6269 }
6270 
6271 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
6272  uint32_t currentFrameIndex,
6273  uint32_t frameInUseCount,
6274  VmaAllocationRequest* pAllocationRequest)
6275 {
6276  while(pAllocationRequest->itemsToMakeLostCount > 0)
6277  {
6278  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
6279  {
6280  ++pAllocationRequest->item;
6281  }
6282  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6283  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
6284  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
6285  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6286  {
6287  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
6288  --pAllocationRequest->itemsToMakeLostCount;
6289  }
6290  else
6291  {
6292  return false;
6293  }
6294  }
6295 
6296  VMA_HEAVY_ASSERT(Validate());
6297  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6298  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
6299 
6300  return true;
6301 }
6302 
6303 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
6304 {
6305  uint32_t lostAllocationCount = 0;
6306  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6307  it != m_Suballocations.end();
6308  ++it)
6309  {
6310  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
6311  it->hAllocation->CanBecomeLost() &&
6312  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6313  {
6314  it = FreeSuballocation(it);
6315  ++lostAllocationCount;
6316  }
6317  }
6318  return lostAllocationCount;
6319 }
6320 
6321 VkResult VmaBlockMetadata::CheckCorruption(const void* pBlockData)
6322 {
6323  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6324  it != m_Suballocations.end();
6325  ++it)
6326  {
6327  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6328  {
6329  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
6330  {
6331  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
6332  return VK_ERROR_VALIDATION_FAILED_EXT;
6333  }
6334  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
6335  {
6336  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
6337  return VK_ERROR_VALIDATION_FAILED_EXT;
6338  }
6339  }
6340  }
6341 
6342  return VK_SUCCESS;
6343 }
6344 
6345 void VmaBlockMetadata::Alloc(
6346  const VmaAllocationRequest& request,
6347  VmaSuballocationType type,
6348  VkDeviceSize allocSize,
6349  VmaAllocation hAllocation)
6350 {
6351  VMA_ASSERT(request.item != m_Suballocations.end());
6352  VmaSuballocation& suballoc = *request.item;
6353  // Given suballocation is a free block.
6354  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6355  // Given offset is inside this suballocation.
6356  VMA_ASSERT(request.offset >= suballoc.offset);
6357  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
6358  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
6359  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
6360 
6361  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
6362  // it to become used.
6363  UnregisterFreeSuballocation(request.item);
6364 
6365  suballoc.offset = request.offset;
6366  suballoc.size = allocSize;
6367  suballoc.type = type;
6368  suballoc.hAllocation = hAllocation;
6369 
6370  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
6371  if(paddingEnd)
6372  {
6373  VmaSuballocation paddingSuballoc = {};
6374  paddingSuballoc.offset = request.offset + allocSize;
6375  paddingSuballoc.size = paddingEnd;
6376  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6377  VmaSuballocationList::iterator next = request.item;
6378  ++next;
6379  const VmaSuballocationList::iterator paddingEndItem =
6380  m_Suballocations.insert(next, paddingSuballoc);
6381  RegisterFreeSuballocation(paddingEndItem);
6382  }
6383 
6384  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
6385  if(paddingBegin)
6386  {
6387  VmaSuballocation paddingSuballoc = {};
6388  paddingSuballoc.offset = request.offset - paddingBegin;
6389  paddingSuballoc.size = paddingBegin;
6390  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6391  const VmaSuballocationList::iterator paddingBeginItem =
6392  m_Suballocations.insert(request.item, paddingSuballoc);
6393  RegisterFreeSuballocation(paddingBeginItem);
6394  }
6395 
6396  // Update totals.
6397  m_FreeCount = m_FreeCount - 1;
6398  if(paddingBegin > 0)
6399  {
6400  ++m_FreeCount;
6401  }
6402  if(paddingEnd > 0)
6403  {
6404  ++m_FreeCount;
6405  }
6406  m_SumFreeSize -= allocSize;
6407 }
6408 
6409 void VmaBlockMetadata::Free(const VmaAllocation allocation)
6410 {
6411  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6412  suballocItem != m_Suballocations.end();
6413  ++suballocItem)
6414  {
6415  VmaSuballocation& suballoc = *suballocItem;
6416  if(suballoc.hAllocation == allocation)
6417  {
6418  FreeSuballocation(suballocItem);
6419  VMA_HEAVY_ASSERT(Validate());
6420  return;
6421  }
6422  }
6423  VMA_ASSERT(0 && "Not found!");
6424 }
6425 
6426 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
6427 {
6428  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6429  suballocItem != m_Suballocations.end();
6430  ++suballocItem)
6431  {
6432  VmaSuballocation& suballoc = *suballocItem;
6433  if(suballoc.offset == offset)
6434  {
6435  FreeSuballocation(suballocItem);
6436  return;
6437  }
6438  }
6439  VMA_ASSERT(0 && "Not found!");
6440 }
6441 
6442 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
6443 {
6444  VkDeviceSize lastSize = 0;
6445  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
6446  {
6447  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
6448 
6449  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6450  {
6451  VMA_ASSERT(0);
6452  return false;
6453  }
6454  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6455  {
6456  VMA_ASSERT(0);
6457  return false;
6458  }
6459  if(it->size < lastSize)
6460  {
6461  VMA_ASSERT(0);
6462  return false;
6463  }
6464 
6465  lastSize = it->size;
6466  }
6467  return true;
6468 }
6469 
6470 bool VmaBlockMetadata::CheckAllocation(
6471  uint32_t currentFrameIndex,
6472  uint32_t frameInUseCount,
6473  VkDeviceSize bufferImageGranularity,
6474  VkDeviceSize allocSize,
6475  VkDeviceSize allocAlignment,
6476  VmaSuballocationType allocType,
6477  VmaSuballocationList::const_iterator suballocItem,
6478  bool canMakeOtherLost,
6479  VkDeviceSize* pOffset,
6480  size_t* itemsToMakeLostCount,
6481  VkDeviceSize* pSumFreeSize,
6482  VkDeviceSize* pSumItemSize) const
6483 {
6484  VMA_ASSERT(allocSize > 0);
6485  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6486  VMA_ASSERT(suballocItem != m_Suballocations.cend());
6487  VMA_ASSERT(pOffset != VMA_NULL);
6488 
6489  *itemsToMakeLostCount = 0;
6490  *pSumFreeSize = 0;
6491  *pSumItemSize = 0;
6492 
6493  if(canMakeOtherLost)
6494  {
6495  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6496  {
6497  *pSumFreeSize = suballocItem->size;
6498  }
6499  else
6500  {
6501  if(suballocItem->hAllocation->CanBecomeLost() &&
6502  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6503  {
6504  ++*itemsToMakeLostCount;
6505  *pSumItemSize = suballocItem->size;
6506  }
6507  else
6508  {
6509  return false;
6510  }
6511  }
6512 
6513  // Remaining size is too small for this request: Early return.
6514  if(m_Size - suballocItem->offset < allocSize)
6515  {
6516  return false;
6517  }
6518 
6519  // Start from offset equal to beginning of this suballocation.
6520  *pOffset = suballocItem->offset;
6521 
6522  // Apply VMA_DEBUG_MARGIN at the beginning.
6523  if(VMA_DEBUG_MARGIN > 0)
6524  {
6525  *pOffset += VMA_DEBUG_MARGIN;
6526  }
6527 
6528  // Apply alignment.
6529  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6530 
6531  // Check previous suballocations for BufferImageGranularity conflicts.
6532  // Make bigger alignment if necessary.
6533  if(bufferImageGranularity > 1)
6534  {
6535  bool bufferImageGranularityConflict = false;
6536  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6537  while(prevSuballocItem != m_Suballocations.cbegin())
6538  {
6539  --prevSuballocItem;
6540  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6541  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6542  {
6543  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6544  {
6545  bufferImageGranularityConflict = true;
6546  break;
6547  }
6548  }
6549  else
6550  // Already on previous page.
6551  break;
6552  }
6553  if(bufferImageGranularityConflict)
6554  {
6555  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6556  }
6557  }
6558 
6559  // Now that we have final *pOffset, check if we are past suballocItem.
6560  // If yes, return false - this function should be called for another suballocItem as starting point.
6561  if(*pOffset >= suballocItem->offset + suballocItem->size)
6562  {
6563  return false;
6564  }
6565 
6566  // Calculate padding at the beginning based on current offset.
6567  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
6568 
6569  // Calculate required margin at the end.
6570  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6571 
6572  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
6573  // Another early return check.
6574  if(suballocItem->offset + totalSize > m_Size)
6575  {
6576  return false;
6577  }
6578 
6579  // Advance lastSuballocItem until desired size is reached.
6580  // Update itemsToMakeLostCount.
6581  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
6582  if(totalSize > suballocItem->size)
6583  {
6584  VkDeviceSize remainingSize = totalSize - suballocItem->size;
6585  while(remainingSize > 0)
6586  {
6587  ++lastSuballocItem;
6588  if(lastSuballocItem == m_Suballocations.cend())
6589  {
6590  return false;
6591  }
6592  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6593  {
6594  *pSumFreeSize += lastSuballocItem->size;
6595  }
6596  else
6597  {
6598  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
6599  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
6600  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6601  {
6602  ++*itemsToMakeLostCount;
6603  *pSumItemSize += lastSuballocItem->size;
6604  }
6605  else
6606  {
6607  return false;
6608  }
6609  }
6610  remainingSize = (lastSuballocItem->size < remainingSize) ?
6611  remainingSize - lastSuballocItem->size : 0;
6612  }
6613  }
6614 
6615  // Check next suballocations for BufferImageGranularity conflicts.
6616  // If conflict exists, we must mark more allocations lost or fail.
6617  if(bufferImageGranularity > 1)
6618  {
6619  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
6620  ++nextSuballocItem;
6621  while(nextSuballocItem != m_Suballocations.cend())
6622  {
6623  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6624  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6625  {
6626  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6627  {
6628  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
6629  if(nextSuballoc.hAllocation->CanBecomeLost() &&
6630  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6631  {
6632  ++*itemsToMakeLostCount;
6633  }
6634  else
6635  {
6636  return false;
6637  }
6638  }
6639  }
6640  else
6641  {
6642  // Already on next page.
6643  break;
6644  }
6645  ++nextSuballocItem;
6646  }
6647  }
6648  }
6649  else
6650  {
6651  const VmaSuballocation& suballoc = *suballocItem;
6652  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6653 
6654  *pSumFreeSize = suballoc.size;
6655 
6656  // Size of this suballocation is too small for this request: Early return.
6657  if(suballoc.size < allocSize)
6658  {
6659  return false;
6660  }
6661 
6662  // Start from offset equal to beginning of this suballocation.
6663  *pOffset = suballoc.offset;
6664 
6665  // Apply VMA_DEBUG_MARGIN at the beginning.
6666  if(VMA_DEBUG_MARGIN > 0)
6667  {
6668  *pOffset += VMA_DEBUG_MARGIN;
6669  }
6670 
6671  // Apply alignment.
6672  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6673 
6674  // Check previous suballocations for BufferImageGranularity conflicts.
6675  // Make bigger alignment if necessary.
6676  if(bufferImageGranularity > 1)
6677  {
6678  bool bufferImageGranularityConflict = false;
6679  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6680  while(prevSuballocItem != m_Suballocations.cbegin())
6681  {
6682  --prevSuballocItem;
6683  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6684  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6685  {
6686  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6687  {
6688  bufferImageGranularityConflict = true;
6689  break;
6690  }
6691  }
6692  else
6693  // Already on previous page.
6694  break;
6695  }
6696  if(bufferImageGranularityConflict)
6697  {
6698  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6699  }
6700  }
6701 
6702  // Calculate padding at the beginning based on current offset.
6703  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
6704 
6705  // Calculate required margin at the end.
6706  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6707 
6708  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
6709  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
6710  {
6711  return false;
6712  }
6713 
6714  // Check next suballocations for BufferImageGranularity conflicts.
6715  // If conflict exists, allocation cannot be made here.
6716  if(bufferImageGranularity > 1)
6717  {
6718  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
6719  ++nextSuballocItem;
6720  while(nextSuballocItem != m_Suballocations.cend())
6721  {
6722  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6723  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6724  {
6725  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6726  {
6727  return false;
6728  }
6729  }
6730  else
6731  {
6732  // Already on next page.
6733  break;
6734  }
6735  ++nextSuballocItem;
6736  }
6737  }
6738  }
6739 
6740  // All tests passed: Success. pOffset is already filled.
6741  return true;
6742 }
6743 
6744 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
6745 {
6746  VMA_ASSERT(item != m_Suballocations.end());
6747  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6748 
6749  VmaSuballocationList::iterator nextItem = item;
6750  ++nextItem;
6751  VMA_ASSERT(nextItem != m_Suballocations.end());
6752  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
6753 
6754  item->size += nextItem->size;
6755  --m_FreeCount;
6756  m_Suballocations.erase(nextItem);
6757 }
6758 
6759 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
6760 {
6761  // Change this suballocation to be marked as free.
6762  VmaSuballocation& suballoc = *suballocItem;
6763  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6764  suballoc.hAllocation = VK_NULL_HANDLE;
6765 
6766  // Update totals.
6767  ++m_FreeCount;
6768  m_SumFreeSize += suballoc.size;
6769 
6770  // Merge with previous and/or next suballocation if it's also free.
6771  bool mergeWithNext = false;
6772  bool mergeWithPrev = false;
6773 
6774  VmaSuballocationList::iterator nextItem = suballocItem;
6775  ++nextItem;
6776  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
6777  {
6778  mergeWithNext = true;
6779  }
6780 
6781  VmaSuballocationList::iterator prevItem = suballocItem;
6782  if(suballocItem != m_Suballocations.begin())
6783  {
6784  --prevItem;
6785  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6786  {
6787  mergeWithPrev = true;
6788  }
6789  }
6790 
6791  if(mergeWithNext)
6792  {
6793  UnregisterFreeSuballocation(nextItem);
6794  MergeFreeWithNext(suballocItem);
6795  }
6796 
6797  if(mergeWithPrev)
6798  {
6799  UnregisterFreeSuballocation(prevItem);
6800  MergeFreeWithNext(prevItem);
6801  RegisterFreeSuballocation(prevItem);
6802  return prevItem;
6803  }
6804  else
6805  {
6806  RegisterFreeSuballocation(suballocItem);
6807  return suballocItem;
6808  }
6809 }
6810 
6811 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
6812 {
6813  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6814  VMA_ASSERT(item->size > 0);
6815 
6816  // You may want to enable this validation at the beginning or at the end of
6817  // this function, depending on what do you want to check.
6818  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6819 
6820  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6821  {
6822  if(m_FreeSuballocationsBySize.empty())
6823  {
6824  m_FreeSuballocationsBySize.push_back(item);
6825  }
6826  else
6827  {
6828  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
6829  }
6830  }
6831 
6832  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6833 }
6834 
6835 
6836 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
6837 {
6838  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6839  VMA_ASSERT(item->size > 0);
6840 
6841  // You may want to enable this validation at the beginning or at the end of
6842  // this function, depending on what do you want to check.
6843  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6844 
6845  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6846  {
6847  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6848  m_FreeSuballocationsBySize.data(),
6849  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
6850  item,
6851  VmaSuballocationItemSizeLess());
6852  for(size_t index = it - m_FreeSuballocationsBySize.data();
6853  index < m_FreeSuballocationsBySize.size();
6854  ++index)
6855  {
6856  if(m_FreeSuballocationsBySize[index] == item)
6857  {
6858  VmaVectorRemove(m_FreeSuballocationsBySize, index);
6859  return;
6860  }
6861  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
6862  }
6863  VMA_ASSERT(0 && "Not found.");
6864  }
6865 
6866  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6867 }
6868 
6870 // class VmaDeviceMemoryBlock
6871 
6872 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
6873  m_Metadata(hAllocator),
6874  m_MemoryTypeIndex(UINT32_MAX),
6875  m_Id(0),
6876  m_hMemory(VK_NULL_HANDLE),
6877  m_MapCount(0),
6878  m_pMappedData(VMA_NULL)
6879 {
6880 }
6881 
6882 void VmaDeviceMemoryBlock::Init(
6883  uint32_t newMemoryTypeIndex,
6884  VkDeviceMemory newMemory,
6885  VkDeviceSize newSize,
6886  uint32_t id)
6887 {
6888  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6889 
6890  m_MemoryTypeIndex = newMemoryTypeIndex;
6891  m_Id = id;
6892  m_hMemory = newMemory;
6893 
6894  m_Metadata.Init(newSize);
6895 }
6896 
6897 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
6898 {
6899  // This is the most important assert in the entire library.
6900  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
6901  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
6902 
6903  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
6904  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
6905  m_hMemory = VK_NULL_HANDLE;
6906 }
6907 
6908 bool VmaDeviceMemoryBlock::Validate() const
6909 {
6910  if((m_hMemory == VK_NULL_HANDLE) ||
6911  (m_Metadata.GetSize() == 0))
6912  {
6913  return false;
6914  }
6915 
6916  return m_Metadata.Validate();
6917 }
6918 
6919 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
6920 {
6921  void* pData = nullptr;
6922  VkResult res = Map(hAllocator, 1, &pData);
6923  if(res != VK_SUCCESS)
6924  {
6925  return res;
6926  }
6927 
6928  res = m_Metadata.CheckCorruption(pData);
6929 
6930  Unmap(hAllocator, 1);
6931 
6932  return res;
6933 }
6934 
6935 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
6936 {
6937  if(count == 0)
6938  {
6939  return VK_SUCCESS;
6940  }
6941 
6942  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6943  if(m_MapCount != 0)
6944  {
6945  m_MapCount += count;
6946  VMA_ASSERT(m_pMappedData != VMA_NULL);
6947  if(ppData != VMA_NULL)
6948  {
6949  *ppData = m_pMappedData;
6950  }
6951  return VK_SUCCESS;
6952  }
6953  else
6954  {
6955  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6956  hAllocator->m_hDevice,
6957  m_hMemory,
6958  0, // offset
6959  VK_WHOLE_SIZE,
6960  0, // flags
6961  &m_pMappedData);
6962  if(result == VK_SUCCESS)
6963  {
6964  if(ppData != VMA_NULL)
6965  {
6966  *ppData = m_pMappedData;
6967  }
6968  m_MapCount = count;
6969  }
6970  return result;
6971  }
6972 }
6973 
6974 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
6975 {
6976  if(count == 0)
6977  {
6978  return;
6979  }
6980 
6981  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6982  if(m_MapCount >= count)
6983  {
6984  m_MapCount -= count;
6985  if(m_MapCount == 0)
6986  {
6987  m_pMappedData = VMA_NULL;
6988  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
6989  }
6990  }
6991  else
6992  {
6993  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
6994  }
6995 }
6996 
6997 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
6998 {
6999  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
7000  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
7001 
7002  void* pData;
7003  VkResult res = Map(hAllocator, 1, &pData);
7004  if(res != VK_SUCCESS)
7005  {
7006  return res;
7007  }
7008 
7009  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
7010  VmaWriteMagicValue(pData, allocOffset + allocSize);
7011 
7012  Unmap(hAllocator, 1);
7013 
7014  return VK_SUCCESS;
7015 }
7016 
7017 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
7018 {
7019  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
7020  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
7021 
7022  void* pData;
7023  VkResult res = Map(hAllocator, 1, &pData);
7024  if(res != VK_SUCCESS)
7025  {
7026  return res;
7027  }
7028 
7029  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
7030  {
7031  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
7032  }
7033  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
7034  {
7035  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
7036  }
7037 
7038  Unmap(hAllocator, 1);
7039 
7040  return VK_SUCCESS;
7041 }
7042 
7043 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
7044  const VmaAllocator hAllocator,
7045  const VmaAllocation hAllocation,
7046  VkBuffer hBuffer)
7047 {
7048  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
7049  hAllocation->GetBlock() == this);
7050  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
7051  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
7052  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
7053  hAllocator->m_hDevice,
7054  hBuffer,
7055  m_hMemory,
7056  hAllocation->GetOffset());
7057 }
7058 
7059 VkResult VmaDeviceMemoryBlock::BindImageMemory(
7060  const VmaAllocator hAllocator,
7061  const VmaAllocation hAllocation,
7062  VkImage hImage)
7063 {
7064  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
7065  hAllocation->GetBlock() == this);
7066  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
7067  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
7068  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
7069  hAllocator->m_hDevice,
7070  hImage,
7071  m_hMemory,
7072  hAllocation->GetOffset());
7073 }
7074 
7075 static void InitStatInfo(VmaStatInfo& outInfo)
7076 {
7077  memset(&outInfo, 0, sizeof(outInfo));
7078  outInfo.allocationSizeMin = UINT64_MAX;
7079  outInfo.unusedRangeSizeMin = UINT64_MAX;
7080 }
7081 
7082 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
7083 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
7084 {
7085  inoutInfo.blockCount += srcInfo.blockCount;
7086  inoutInfo.allocationCount += srcInfo.allocationCount;
7087  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
7088  inoutInfo.usedBytes += srcInfo.usedBytes;
7089  inoutInfo.unusedBytes += srcInfo.unusedBytes;
7090  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
7091  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
7092  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
7093  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
7094 }
7095 
7096 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
7097 {
7098  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
7099  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
7100  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
7101  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
7102 }
7103 
7104 VmaPool_T::VmaPool_T(
7105  VmaAllocator hAllocator,
7106  const VmaPoolCreateInfo& createInfo) :
7107  m_BlockVector(
7108  hAllocator,
7109  createInfo.memoryTypeIndex,
7110  createInfo.blockSize,
7111  createInfo.minBlockCount,
7112  createInfo.maxBlockCount,
7113  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
7114  createInfo.frameInUseCount,
7115  true), // isCustomPool
7116  m_Id(0)
7117 {
7118 }
7119 
7120 VmaPool_T::~VmaPool_T()
7121 {
7122 }
7123 
7124 #if VMA_STATS_STRING_ENABLED
7125 
7126 #endif // #if VMA_STATS_STRING_ENABLED
7127 
7128 VmaBlockVector::VmaBlockVector(
7129  VmaAllocator hAllocator,
7130  uint32_t memoryTypeIndex,
7131  VkDeviceSize preferredBlockSize,
7132  size_t minBlockCount,
7133  size_t maxBlockCount,
7134  VkDeviceSize bufferImageGranularity,
7135  uint32_t frameInUseCount,
7136  bool isCustomPool) :
7137  m_hAllocator(hAllocator),
7138  m_MemoryTypeIndex(memoryTypeIndex),
7139  m_PreferredBlockSize(preferredBlockSize),
7140  m_MinBlockCount(minBlockCount),
7141  m_MaxBlockCount(maxBlockCount),
7142  m_BufferImageGranularity(bufferImageGranularity),
7143  m_FrameInUseCount(frameInUseCount),
7144  m_IsCustomPool(isCustomPool),
7145  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
7146  m_HasEmptyBlock(false),
7147  m_pDefragmentator(VMA_NULL),
7148  m_NextBlockId(0)
7149 {
7150 }
7151 
7152 VmaBlockVector::~VmaBlockVector()
7153 {
7154  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
7155 
7156  for(size_t i = m_Blocks.size(); i--; )
7157  {
7158  m_Blocks[i]->Destroy(m_hAllocator);
7159  vma_delete(m_hAllocator, m_Blocks[i]);
7160  }
7161 }
7162 
7163 VkResult VmaBlockVector::CreateMinBlocks()
7164 {
7165  for(size_t i = 0; i < m_MinBlockCount; ++i)
7166  {
7167  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
7168  if(res != VK_SUCCESS)
7169  {
7170  return res;
7171  }
7172  }
7173  return VK_SUCCESS;
7174 }
7175 
7176 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
7177 {
7178  pStats->size = 0;
7179  pStats->unusedSize = 0;
7180  pStats->allocationCount = 0;
7181  pStats->unusedRangeCount = 0;
7182  pStats->unusedRangeSizeMax = 0;
7183 
7184  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7185 
7186  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7187  {
7188  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7189  VMA_ASSERT(pBlock);
7190  VMA_HEAVY_ASSERT(pBlock->Validate());
7191  pBlock->m_Metadata.AddPoolStats(*pStats);
7192  }
7193 }
7194 
7195 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
7196 {
7197  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7198  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
7199  (VMA_DEBUG_MARGIN > 0) &&
7200  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
7201 }
7202 
7203 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
7204 
7205 VkResult VmaBlockVector::Allocate(
7206  VmaPool hCurrentPool,
7207  uint32_t currentFrameIndex,
7208  VkDeviceSize size,
7209  VkDeviceSize alignment,
7210  const VmaAllocationCreateInfo& createInfo,
7211  VmaSuballocationType suballocType,
7212  VmaAllocation* pAllocation)
7213 {
7214  // Early reject: requested allocation size is larger that maximum block size for this block vector.
7215  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
7216  {
7217  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7218  }
7219 
7220  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
7221  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
7222 
7223  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7224 
7225  // 1. Search existing allocations. Try to allocate without making other allocations lost.
7226  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
7227  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
7228  {
7229  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
7230  VMA_ASSERT(pCurrBlock);
7231  VmaAllocationRequest currRequest = {};
7232  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
7233  currentFrameIndex,
7234  m_FrameInUseCount,
7235  m_BufferImageGranularity,
7236  size,
7237  alignment,
7238  suballocType,
7239  false, // canMakeOtherLost
7240  &currRequest))
7241  {
7242  // Allocate from pCurrBlock.
7243  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
7244 
7245  if(mapped)
7246  {
7247  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
7248  if(res != VK_SUCCESS)
7249  {
7250  return res;
7251  }
7252  }
7253 
7254  // We no longer have an empty Allocation.
7255  if(pCurrBlock->m_Metadata.IsEmpty())
7256  {
7257  m_HasEmptyBlock = false;
7258  }
7259 
7260  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7261  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, size, *pAllocation);
7262  (*pAllocation)->InitBlockAllocation(
7263  hCurrentPool,
7264  pCurrBlock,
7265  currRequest.offset,
7266  alignment,
7267  size,
7268  suballocType,
7269  mapped,
7270  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7271  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
7272  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
7273  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7274  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7275  {
7276  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7277  }
7278  if(IsCorruptionDetectionEnabled())
7279  {
7280  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
7281  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7282  }
7283  return VK_SUCCESS;
7284  }
7285  }
7286 
7287  const bool canCreateNewBlock =
7288  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
7289  (m_Blocks.size() < m_MaxBlockCount);
7290 
7291  // 2. Try to create new block.
7292  if(canCreateNewBlock)
7293  {
7294  // Calculate optimal size for new block.
7295  VkDeviceSize newBlockSize = m_PreferredBlockSize;
7296  uint32_t newBlockSizeShift = 0;
7297  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
7298 
7299  // Allocating blocks of other sizes is allowed only in default pools.
7300  // In custom pools block size is fixed.
7301  if(m_IsCustomPool == false)
7302  {
7303  // Allocate 1/8, 1/4, 1/2 as first blocks.
7304  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
7305  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
7306  {
7307  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
7308  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
7309  {
7310  newBlockSize = smallerNewBlockSize;
7311  ++newBlockSizeShift;
7312  }
7313  else
7314  {
7315  break;
7316  }
7317  }
7318  }
7319 
7320  size_t newBlockIndex = 0;
7321  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
7322  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
7323  if(m_IsCustomPool == false)
7324  {
7325  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
7326  {
7327  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
7328  if(smallerNewBlockSize >= size)
7329  {
7330  newBlockSize = smallerNewBlockSize;
7331  ++newBlockSizeShift;
7332  res = CreateBlock(newBlockSize, &newBlockIndex);
7333  }
7334  else
7335  {
7336  break;
7337  }
7338  }
7339  }
7340 
7341  if(res == VK_SUCCESS)
7342  {
7343  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
7344  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= size);
7345 
7346  if(mapped)
7347  {
7348  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
7349  if(res != VK_SUCCESS)
7350  {
7351  return res;
7352  }
7353  }
7354 
7355  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
7356  VmaAllocationRequest allocRequest;
7357  if(pBlock->m_Metadata.CreateAllocationRequest(
7358  currentFrameIndex,
7359  m_FrameInUseCount,
7360  m_BufferImageGranularity,
7361  size,
7362  alignment,
7363  suballocType,
7364  false, // canMakeOtherLost
7365  &allocRequest))
7366  {
7367  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7368  pBlock->m_Metadata.Alloc(allocRequest, suballocType, size, *pAllocation);
7369  (*pAllocation)->InitBlockAllocation(
7370  hCurrentPool,
7371  pBlock,
7372  allocRequest.offset,
7373  alignment,
7374  size,
7375  suballocType,
7376  mapped,
7377  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7378  VMA_HEAVY_ASSERT(pBlock->Validate());
7379  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
7380  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7381  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7382  {
7383  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7384  }
7385  if(IsCorruptionDetectionEnabled())
7386  {
7387  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
7388  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7389  }
7390  return VK_SUCCESS;
7391  }
7392  else
7393  {
7394  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
7395  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7396  }
7397  }
7398  }
7399 
7400  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
7401 
7402  // 3. Try to allocate from existing blocks with making other allocations lost.
7403  if(canMakeOtherLost)
7404  {
7405  uint32_t tryIndex = 0;
7406  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
7407  {
7408  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
7409  VmaAllocationRequest bestRequest = {};
7410  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
7411 
7412  // 1. Search existing allocations.
7413  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
7414  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
7415  {
7416  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
7417  VMA_ASSERT(pCurrBlock);
7418  VmaAllocationRequest currRequest = {};
7419  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
7420  currentFrameIndex,
7421  m_FrameInUseCount,
7422  m_BufferImageGranularity,
7423  size,
7424  alignment,
7425  suballocType,
7426  canMakeOtherLost,
7427  &currRequest))
7428  {
7429  const VkDeviceSize currRequestCost = currRequest.CalcCost();
7430  if(pBestRequestBlock == VMA_NULL ||
7431  currRequestCost < bestRequestCost)
7432  {
7433  pBestRequestBlock = pCurrBlock;
7434  bestRequest = currRequest;
7435  bestRequestCost = currRequestCost;
7436 
7437  if(bestRequestCost == 0)
7438  {
7439  break;
7440  }
7441  }
7442  }
7443  }
7444 
7445  if(pBestRequestBlock != VMA_NULL)
7446  {
7447  if(mapped)
7448  {
7449  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
7450  if(res != VK_SUCCESS)
7451  {
7452  return res;
7453  }
7454  }
7455 
7456  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
7457  currentFrameIndex,
7458  m_FrameInUseCount,
7459  &bestRequest))
7460  {
7461  // We no longer have an empty Allocation.
7462  if(pBestRequestBlock->m_Metadata.IsEmpty())
7463  {
7464  m_HasEmptyBlock = false;
7465  }
7466  // Allocate from this pBlock.
7467  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
7468  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, size, *pAllocation);
7469  (*pAllocation)->InitBlockAllocation(
7470  hCurrentPool,
7471  pBestRequestBlock,
7472  bestRequest.offset,
7473  alignment,
7474  size,
7475  suballocType,
7476  mapped,
7477  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
7478  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
7479  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
7480  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
7481  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
7482  {
7483  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
7484  }
7485  if(IsCorruptionDetectionEnabled())
7486  {
7487  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
7488  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
7489  }
7490  return VK_SUCCESS;
7491  }
7492  // else: Some allocations must have been touched while we are here. Next try.
7493  }
7494  else
7495  {
7496  // Could not find place in any of the blocks - break outer loop.
7497  break;
7498  }
7499  }
7500  /* Maximum number of tries exceeded - a very unlike event when many other
7501  threads are simultaneously touching allocations making it impossible to make
7502  lost at the same time as we try to allocate. */
7503  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
7504  {
7505  return VK_ERROR_TOO_MANY_OBJECTS;
7506  }
7507  }
7508 
7509  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7510 }
7511 
7512 void VmaBlockVector::Free(
7513  VmaAllocation hAllocation)
7514 {
7515  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
7516 
7517  // Scope for lock.
7518  {
7519  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7520 
7521  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
7522 
7523  if(IsCorruptionDetectionEnabled())
7524  {
7525  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
7526  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
7527  }
7528 
7529  if(hAllocation->IsPersistentMap())
7530  {
7531  pBlock->Unmap(m_hAllocator, 1);
7532  }
7533 
7534  pBlock->m_Metadata.Free(hAllocation);
7535  VMA_HEAVY_ASSERT(pBlock->Validate());
7536 
7537  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
7538 
7539  // pBlock became empty after this deallocation.
7540  if(pBlock->m_Metadata.IsEmpty())
7541  {
7542  // Already has empty Allocation. We don't want to have two, so delete this one.
7543  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
7544  {
7545  pBlockToDelete = pBlock;
7546  Remove(pBlock);
7547  }
7548  // We now have first empty block.
7549  else
7550  {
7551  m_HasEmptyBlock = true;
7552  }
7553  }
7554  // pBlock didn't become empty, but we have another empty block - find and free that one.
7555  // (This is optional, heuristics.)
7556  else if(m_HasEmptyBlock)
7557  {
7558  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
7559  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
7560  {
7561  pBlockToDelete = pLastBlock;
7562  m_Blocks.pop_back();
7563  m_HasEmptyBlock = false;
7564  }
7565  }
7566 
7567  IncrementallySortBlocks();
7568  }
7569 
7570  // Destruction of a free Allocation. Deferred until this point, outside of mutex
7571  // lock, for performance reason.
7572  if(pBlockToDelete != VMA_NULL)
7573  {
7574  VMA_DEBUG_LOG(" Deleted empty allocation");
7575  pBlockToDelete->Destroy(m_hAllocator);
7576  vma_delete(m_hAllocator, pBlockToDelete);
7577  }
7578 }
7579 
7580 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
7581 {
7582  VkDeviceSize result = 0;
7583  for(size_t i = m_Blocks.size(); i--; )
7584  {
7585  result = VMA_MAX(result, m_Blocks[i]->m_Metadata.GetSize());
7586  if(result >= m_PreferredBlockSize)
7587  {
7588  break;
7589  }
7590  }
7591  return result;
7592 }
7593 
7594 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
7595 {
7596  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7597  {
7598  if(m_Blocks[blockIndex] == pBlock)
7599  {
7600  VmaVectorRemove(m_Blocks, blockIndex);
7601  return;
7602  }
7603  }
7604  VMA_ASSERT(0);
7605 }
7606 
7607 void VmaBlockVector::IncrementallySortBlocks()
7608 {
7609  // Bubble sort only until first swap.
7610  for(size_t i = 1; i < m_Blocks.size(); ++i)
7611  {
7612  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
7613  {
7614  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
7615  return;
7616  }
7617  }
7618 }
7619 
7620 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
7621 {
7622  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7623  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
7624  allocInfo.allocationSize = blockSize;
7625  VkDeviceMemory mem = VK_NULL_HANDLE;
7626  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
7627  if(res < 0)
7628  {
7629  return res;
7630  }
7631 
7632  // New VkDeviceMemory successfully created.
7633 
7634  // Create new Allocation for it.
7635  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
7636  pBlock->Init(
7637  m_MemoryTypeIndex,
7638  mem,
7639  allocInfo.allocationSize,
7640  m_NextBlockId++);
7641 
7642  m_Blocks.push_back(pBlock);
7643  if(pNewBlockIndex != VMA_NULL)
7644  {
7645  *pNewBlockIndex = m_Blocks.size() - 1;
7646  }
7647 
7648  return VK_SUCCESS;
7649 }
7650 
7651 #if VMA_STATS_STRING_ENABLED
7652 
7653 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
7654 {
7655  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7656 
7657  json.BeginObject();
7658 
7659  if(m_IsCustomPool)
7660  {
7661  json.WriteString("MemoryTypeIndex");
7662  json.WriteNumber(m_MemoryTypeIndex);
7663 
7664  json.WriteString("BlockSize");
7665  json.WriteNumber(m_PreferredBlockSize);
7666 
7667  json.WriteString("BlockCount");
7668  json.BeginObject(true);
7669  if(m_MinBlockCount > 0)
7670  {
7671  json.WriteString("Min");
7672  json.WriteNumber((uint64_t)m_MinBlockCount);
7673  }
7674  if(m_MaxBlockCount < SIZE_MAX)
7675  {
7676  json.WriteString("Max");
7677  json.WriteNumber((uint64_t)m_MaxBlockCount);
7678  }
7679  json.WriteString("Cur");
7680  json.WriteNumber((uint64_t)m_Blocks.size());
7681  json.EndObject();
7682 
7683  if(m_FrameInUseCount > 0)
7684  {
7685  json.WriteString("FrameInUseCount");
7686  json.WriteNumber(m_FrameInUseCount);
7687  }
7688  }
7689  else
7690  {
7691  json.WriteString("PreferredBlockSize");
7692  json.WriteNumber(m_PreferredBlockSize);
7693  }
7694 
7695  json.WriteString("Blocks");
7696  json.BeginObject();
7697  for(size_t i = 0; i < m_Blocks.size(); ++i)
7698  {
7699  json.BeginString();
7700  json.ContinueString(m_Blocks[i]->GetId());
7701  json.EndString();
7702 
7703  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
7704  }
7705  json.EndObject();
7706 
7707  json.EndObject();
7708 }
7709 
7710 #endif // #if VMA_STATS_STRING_ENABLED
7711 
7712 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
7713  VmaAllocator hAllocator,
7714  uint32_t currentFrameIndex)
7715 {
7716  if(m_pDefragmentator == VMA_NULL)
7717  {
7718  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
7719  hAllocator,
7720  this,
7721  currentFrameIndex);
7722  }
7723 
7724  return m_pDefragmentator;
7725 }
7726 
7727 VkResult VmaBlockVector::Defragment(
7728  VmaDefragmentationStats* pDefragmentationStats,
7729  VkDeviceSize& maxBytesToMove,
7730  uint32_t& maxAllocationsToMove)
7731 {
7732  if(m_pDefragmentator == VMA_NULL)
7733  {
7734  return VK_SUCCESS;
7735  }
7736 
7737  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7738 
7739  // Defragment.
7740  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
7741 
7742  // Accumulate statistics.
7743  if(pDefragmentationStats != VMA_NULL)
7744  {
7745  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
7746  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
7747  pDefragmentationStats->bytesMoved += bytesMoved;
7748  pDefragmentationStats->allocationsMoved += allocationsMoved;
7749  VMA_ASSERT(bytesMoved <= maxBytesToMove);
7750  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
7751  maxBytesToMove -= bytesMoved;
7752  maxAllocationsToMove -= allocationsMoved;
7753  }
7754 
7755  // Free empty blocks.
7756  m_HasEmptyBlock = false;
7757  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
7758  {
7759  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
7760  if(pBlock->m_Metadata.IsEmpty())
7761  {
7762  if(m_Blocks.size() > m_MinBlockCount)
7763  {
7764  if(pDefragmentationStats != VMA_NULL)
7765  {
7766  ++pDefragmentationStats->deviceMemoryBlocksFreed;
7767  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
7768  }
7769 
7770  VmaVectorRemove(m_Blocks, blockIndex);
7771  pBlock->Destroy(m_hAllocator);
7772  vma_delete(m_hAllocator, pBlock);
7773  }
7774  else
7775  {
7776  m_HasEmptyBlock = true;
7777  }
7778  }
7779  }
7780 
7781  return result;
7782 }
7783 
7784 void VmaBlockVector::DestroyDefragmentator()
7785 {
7786  if(m_pDefragmentator != VMA_NULL)
7787  {
7788  vma_delete(m_hAllocator, m_pDefragmentator);
7789  m_pDefragmentator = VMA_NULL;
7790  }
7791 }
7792 
7793 void VmaBlockVector::MakePoolAllocationsLost(
7794  uint32_t currentFrameIndex,
7795  size_t* pLostAllocationCount)
7796 {
7797  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7798  size_t lostAllocationCount = 0;
7799  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7800  {
7801  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7802  VMA_ASSERT(pBlock);
7803  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
7804  }
7805  if(pLostAllocationCount != VMA_NULL)
7806  {
7807  *pLostAllocationCount = lostAllocationCount;
7808  }
7809 }
7810 
7811 VkResult VmaBlockVector::CheckCorruption()
7812 {
7813  if(!IsCorruptionDetectionEnabled())
7814  {
7815  return VK_ERROR_FEATURE_NOT_PRESENT;
7816  }
7817 
7818  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7819  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7820  {
7821  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7822  VMA_ASSERT(pBlock);
7823  VkResult res = pBlock->CheckCorruption(m_hAllocator);
7824  if(res != VK_SUCCESS)
7825  {
7826  return res;
7827  }
7828  }
7829  return VK_SUCCESS;
7830 }
7831 
7832 void VmaBlockVector::AddStats(VmaStats* pStats)
7833 {
7834  const uint32_t memTypeIndex = m_MemoryTypeIndex;
7835  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
7836 
7837  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7838 
7839  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7840  {
7841  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7842  VMA_ASSERT(pBlock);
7843  VMA_HEAVY_ASSERT(pBlock->Validate());
7844  VmaStatInfo allocationStatInfo;
7845  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
7846  VmaAddStatInfo(pStats->total, allocationStatInfo);
7847  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7848  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7849  }
7850 }
7851 
7853 // VmaDefragmentator members definition
7854 
7855 VmaDefragmentator::VmaDefragmentator(
7856  VmaAllocator hAllocator,
7857  VmaBlockVector* pBlockVector,
7858  uint32_t currentFrameIndex) :
7859  m_hAllocator(hAllocator),
7860  m_pBlockVector(pBlockVector),
7861  m_CurrentFrameIndex(currentFrameIndex),
7862  m_BytesMoved(0),
7863  m_AllocationsMoved(0),
7864  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
7865  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
7866 {
7867 }
7868 
7869 VmaDefragmentator::~VmaDefragmentator()
7870 {
7871  for(size_t i = m_Blocks.size(); i--; )
7872  {
7873  vma_delete(m_hAllocator, m_Blocks[i]);
7874  }
7875 }
7876 
7877 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
7878 {
7879  AllocationInfo allocInfo;
7880  allocInfo.m_hAllocation = hAlloc;
7881  allocInfo.m_pChanged = pChanged;
7882  m_Allocations.push_back(allocInfo);
7883 }
7884 
7885 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
7886 {
7887  // It has already been mapped for defragmentation.
7888  if(m_pMappedDataForDefragmentation)
7889  {
7890  *ppMappedData = m_pMappedDataForDefragmentation;
7891  return VK_SUCCESS;
7892  }
7893 
7894  // It is originally mapped.
7895  if(m_pBlock->GetMappedData())
7896  {
7897  *ppMappedData = m_pBlock->GetMappedData();
7898  return VK_SUCCESS;
7899  }
7900 
7901  // Map on first usage.
7902  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
7903  *ppMappedData = m_pMappedDataForDefragmentation;
7904  return res;
7905 }
7906 
7907 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
7908 {
7909  if(m_pMappedDataForDefragmentation != VMA_NULL)
7910  {
7911  m_pBlock->Unmap(hAllocator, 1);
7912  }
7913 }
7914 
7915 VkResult VmaDefragmentator::DefragmentRound(
7916  VkDeviceSize maxBytesToMove,
7917  uint32_t maxAllocationsToMove)
7918 {
7919  if(m_Blocks.empty())
7920  {
7921  return VK_SUCCESS;
7922  }
7923 
7924  size_t srcBlockIndex = m_Blocks.size() - 1;
7925  size_t srcAllocIndex = SIZE_MAX;
7926  for(;;)
7927  {
7928  // 1. Find next allocation to move.
7929  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
7930  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
7931  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
7932  {
7933  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
7934  {
7935  // Finished: no more allocations to process.
7936  if(srcBlockIndex == 0)
7937  {
7938  return VK_SUCCESS;
7939  }
7940  else
7941  {
7942  --srcBlockIndex;
7943  srcAllocIndex = SIZE_MAX;
7944  }
7945  }
7946  else
7947  {
7948  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
7949  }
7950  }
7951 
7952  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
7953  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
7954 
7955  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
7956  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
7957  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
7958  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
7959 
7960  // 2. Try to find new place for this allocation in preceding or current block.
7961  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
7962  {
7963  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
7964  VmaAllocationRequest dstAllocRequest;
7965  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
7966  m_CurrentFrameIndex,
7967  m_pBlockVector->GetFrameInUseCount(),
7968  m_pBlockVector->GetBufferImageGranularity(),
7969  size,
7970  alignment,
7971  suballocType,
7972  false, // canMakeOtherLost
7973  &dstAllocRequest) &&
7974  MoveMakesSense(
7975  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
7976  {
7977  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
7978 
7979  // Reached limit on number of allocations or bytes to move.
7980  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
7981  (m_BytesMoved + size > maxBytesToMove))
7982  {
7983  return VK_INCOMPLETE;
7984  }
7985 
7986  void* pDstMappedData = VMA_NULL;
7987  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
7988  if(res != VK_SUCCESS)
7989  {
7990  return res;
7991  }
7992 
7993  void* pSrcMappedData = VMA_NULL;
7994  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
7995  if(res != VK_SUCCESS)
7996  {
7997  return res;
7998  }
7999 
8000  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
8001  memcpy(
8002  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
8003  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
8004  static_cast<size_t>(size));
8005 
8006  if(VMA_DEBUG_MARGIN > 0)
8007  {
8008  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
8009  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
8010  }
8011 
8012  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
8013  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
8014 
8015  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
8016 
8017  if(allocInfo.m_pChanged != VMA_NULL)
8018  {
8019  *allocInfo.m_pChanged = VK_TRUE;
8020  }
8021 
8022  ++m_AllocationsMoved;
8023  m_BytesMoved += size;
8024 
8025  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
8026 
8027  break;
8028  }
8029  }
8030 
8031  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
8032 
8033  if(srcAllocIndex > 0)
8034  {
8035  --srcAllocIndex;
8036  }
8037  else
8038  {
8039  if(srcBlockIndex > 0)
8040  {
8041  --srcBlockIndex;
8042  srcAllocIndex = SIZE_MAX;
8043  }
8044  else
8045  {
8046  return VK_SUCCESS;
8047  }
8048  }
8049  }
8050 }
8051 
8052 VkResult VmaDefragmentator::Defragment(
8053  VkDeviceSize maxBytesToMove,
8054  uint32_t maxAllocationsToMove)
8055 {
8056  if(m_Allocations.empty())
8057  {
8058  return VK_SUCCESS;
8059  }
8060 
8061  // Create block info for each block.
8062  const size_t blockCount = m_pBlockVector->m_Blocks.size();
8063  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8064  {
8065  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
8066  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
8067  m_Blocks.push_back(pBlockInfo);
8068  }
8069 
8070  // Sort them by m_pBlock pointer value.
8071  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
8072 
8073  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
8074  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
8075  {
8076  AllocationInfo& allocInfo = m_Allocations[blockIndex];
8077  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
8078  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
8079  {
8080  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
8081  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
8082  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
8083  {
8084  (*it)->m_Allocations.push_back(allocInfo);
8085  }
8086  else
8087  {
8088  VMA_ASSERT(0);
8089  }
8090  }
8091  }
8092  m_Allocations.clear();
8093 
8094  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8095  {
8096  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
8097  pBlockInfo->CalcHasNonMovableAllocations();
8098  pBlockInfo->SortAllocationsBySizeDescecnding();
8099  }
8100 
8101  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
8102  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
8103 
8104  // Execute defragmentation rounds (the main part).
8105  VkResult result = VK_SUCCESS;
8106  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
8107  {
8108  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
8109  }
8110 
8111  // Unmap blocks that were mapped for defragmentation.
8112  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
8113  {
8114  m_Blocks[blockIndex]->Unmap(m_hAllocator);
8115  }
8116 
8117  return result;
8118 }
8119 
8120 bool VmaDefragmentator::MoveMakesSense(
8121  size_t dstBlockIndex, VkDeviceSize dstOffset,
8122  size_t srcBlockIndex, VkDeviceSize srcOffset)
8123 {
8124  if(dstBlockIndex < srcBlockIndex)
8125  {
8126  return true;
8127  }
8128  if(dstBlockIndex > srcBlockIndex)
8129  {
8130  return false;
8131  }
8132  if(dstOffset < srcOffset)
8133  {
8134  return true;
8135  }
8136  return false;
8137 }
8138 
8140 // VmaRecorder
8141 
8142 #if VMA_RECORDING_ENABLED
8143 
8144 VmaRecorder::VmaRecorder() :
8145  m_UseMutex(true),
8146  m_Flags(0),
8147  m_File(VMA_NULL),
8148  m_Freq(INT64_MAX),
8149  m_StartCounter(INT64_MAX)
8150 {
8151 }
8152 
8153 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
8154 {
8155  m_UseMutex = useMutex;
8156  m_Flags = settings.flags;
8157 
8158  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
8159  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
8160 
8161  // Open file for writing.
8162  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
8163  if(err != 0)
8164  {
8165  return VK_ERROR_INITIALIZATION_FAILED;
8166  }
8167 
8168  // Write header.
8169  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
8170  fprintf(m_File, "%s\n", "1,3");
8171 
8172  return VK_SUCCESS;
8173 }
8174 
8175 VmaRecorder::~VmaRecorder()
8176 {
8177  if(m_File != VMA_NULL)
8178  {
8179  fclose(m_File);
8180  }
8181 }
8182 
8183 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
8184 {
8185  CallParams callParams;
8186  GetBasicParams(callParams);
8187 
8188  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8189  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
8190  Flush();
8191 }
8192 
8193 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
8194 {
8195  CallParams callParams;
8196  GetBasicParams(callParams);
8197 
8198  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8199  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
8200  Flush();
8201 }
8202 
8203 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
8204 {
8205  CallParams callParams;
8206  GetBasicParams(callParams);
8207 
8208  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8209  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
8210  createInfo.memoryTypeIndex,
8211  createInfo.flags,
8212  createInfo.blockSize,
8213  createInfo.minBlockCount,
8214  createInfo.maxBlockCount,
8215  createInfo.frameInUseCount,
8216  pool);
8217  Flush();
8218 }
8219 
8220 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
8221 {
8222  CallParams callParams;
8223  GetBasicParams(callParams);
8224 
8225  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8226  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
8227  pool);
8228  Flush();
8229 }
8230 
8231 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
8232  const VkMemoryRequirements& vkMemReq,
8233  const VmaAllocationCreateInfo& createInfo,
8234  VmaAllocation allocation)
8235 {
8236  CallParams callParams;
8237  GetBasicParams(callParams);
8238 
8239  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8240  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8241  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8242  vkMemReq.size,
8243  vkMemReq.alignment,
8244  vkMemReq.memoryTypeBits,
8245  createInfo.flags,
8246  createInfo.usage,
8247  createInfo.requiredFlags,
8248  createInfo.preferredFlags,
8249  createInfo.memoryTypeBits,
8250  createInfo.pool,
8251  allocation,
8252  userDataStr.GetString());
8253  Flush();
8254 }
8255 
8256 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
8257  const VkMemoryRequirements& vkMemReq,
8258  bool requiresDedicatedAllocation,
8259  bool prefersDedicatedAllocation,
8260  const VmaAllocationCreateInfo& createInfo,
8261  VmaAllocation allocation)
8262 {
8263  CallParams callParams;
8264  GetBasicParams(callParams);
8265 
8266  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8267  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8268  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8269  vkMemReq.size,
8270  vkMemReq.alignment,
8271  vkMemReq.memoryTypeBits,
8272  requiresDedicatedAllocation ? 1 : 0,
8273  prefersDedicatedAllocation ? 1 : 0,
8274  createInfo.flags,
8275  createInfo.usage,
8276  createInfo.requiredFlags,
8277  createInfo.preferredFlags,
8278  createInfo.memoryTypeBits,
8279  createInfo.pool,
8280  allocation,
8281  userDataStr.GetString());
8282  Flush();
8283 }
8284 
8285 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
8286  const VkMemoryRequirements& vkMemReq,
8287  bool requiresDedicatedAllocation,
8288  bool prefersDedicatedAllocation,
8289  const VmaAllocationCreateInfo& createInfo,
8290  VmaAllocation allocation)
8291 {
8292  CallParams callParams;
8293  GetBasicParams(callParams);
8294 
8295  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8296  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
8297  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8298  vkMemReq.size,
8299  vkMemReq.alignment,
8300  vkMemReq.memoryTypeBits,
8301  requiresDedicatedAllocation ? 1 : 0,
8302  prefersDedicatedAllocation ? 1 : 0,
8303  createInfo.flags,
8304  createInfo.usage,
8305  createInfo.requiredFlags,
8306  createInfo.preferredFlags,
8307  createInfo.memoryTypeBits,
8308  createInfo.pool,
8309  allocation,
8310  userDataStr.GetString());
8311  Flush();
8312 }
8313 
8314 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
8315  VmaAllocation allocation)
8316 {
8317  CallParams callParams;
8318  GetBasicParams(callParams);
8319 
8320  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8321  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8322  allocation);
8323  Flush();
8324 }
8325 
8326 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
8327  VmaAllocation allocation,
8328  const void* pUserData)
8329 {
8330  CallParams callParams;
8331  GetBasicParams(callParams);
8332 
8333  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8334  UserDataString userDataStr(
8335  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
8336  pUserData);
8337  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8338  allocation,
8339  userDataStr.GetString());
8340  Flush();
8341 }
8342 
8343 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
8344  VmaAllocation allocation)
8345 {
8346  CallParams callParams;
8347  GetBasicParams(callParams);
8348 
8349  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8350  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
8351  allocation);
8352  Flush();
8353 }
8354 
8355 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
8356  VmaAllocation allocation)
8357 {
8358  CallParams callParams;
8359  GetBasicParams(callParams);
8360 
8361  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8362  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8363  allocation);
8364  Flush();
8365 }
8366 
8367 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
8368  VmaAllocation allocation)
8369 {
8370  CallParams callParams;
8371  GetBasicParams(callParams);
8372 
8373  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8374  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
8375  allocation);
8376  Flush();
8377 }
8378 
8379 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
8380  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
8381 {
8382  CallParams callParams;
8383  GetBasicParams(callParams);
8384 
8385  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8386  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
8387  allocation,
8388  offset,
8389  size);
8390  Flush();
8391 }
8392 
8393 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
8394  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
8395 {
8396  CallParams callParams;
8397  GetBasicParams(callParams);
8398 
8399  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8400  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
8401  allocation,
8402  offset,
8403  size);
8404  Flush();
8405 }
8406 
8407 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
8408  const VkBufferCreateInfo& bufCreateInfo,
8409  const VmaAllocationCreateInfo& allocCreateInfo,
8410  VmaAllocation allocation)
8411 {
8412  CallParams callParams;
8413  GetBasicParams(callParams);
8414 
8415  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8416  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
8417  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8418  bufCreateInfo.flags,
8419  bufCreateInfo.size,
8420  bufCreateInfo.usage,
8421  bufCreateInfo.sharingMode,
8422  allocCreateInfo.flags,
8423  allocCreateInfo.usage,
8424  allocCreateInfo.requiredFlags,
8425  allocCreateInfo.preferredFlags,
8426  allocCreateInfo.memoryTypeBits,
8427  allocCreateInfo.pool,
8428  allocation,
8429  userDataStr.GetString());
8430  Flush();
8431 }
8432 
8433 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
8434  const VkImageCreateInfo& imageCreateInfo,
8435  const VmaAllocationCreateInfo& allocCreateInfo,
8436  VmaAllocation allocation)
8437 {
8438  CallParams callParams;
8439  GetBasicParams(callParams);
8440 
8441  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8442  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
8443  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
8444  imageCreateInfo.flags,
8445  imageCreateInfo.imageType,
8446  imageCreateInfo.format,
8447  imageCreateInfo.extent.width,
8448  imageCreateInfo.extent.height,
8449  imageCreateInfo.extent.depth,
8450  imageCreateInfo.mipLevels,
8451  imageCreateInfo.arrayLayers,
8452  imageCreateInfo.samples,
8453  imageCreateInfo.tiling,
8454  imageCreateInfo.usage,
8455  imageCreateInfo.sharingMode,
8456  imageCreateInfo.initialLayout,
8457  allocCreateInfo.flags,
8458  allocCreateInfo.usage,
8459  allocCreateInfo.requiredFlags,
8460  allocCreateInfo.preferredFlags,
8461  allocCreateInfo.memoryTypeBits,
8462  allocCreateInfo.pool,
8463  allocation,
8464  userDataStr.GetString());
8465  Flush();
8466 }
8467 
8468 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
8469  VmaAllocation allocation)
8470 {
8471  CallParams callParams;
8472  GetBasicParams(callParams);
8473 
8474  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8475  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
8476  allocation);
8477  Flush();
8478 }
8479 
8480 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
8481  VmaAllocation allocation)
8482 {
8483  CallParams callParams;
8484  GetBasicParams(callParams);
8485 
8486  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8487  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
8488  allocation);
8489  Flush();
8490 }
8491 
8492 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
8493  VmaAllocation allocation)
8494 {
8495  CallParams callParams;
8496  GetBasicParams(callParams);
8497 
8498  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8499  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
8500  allocation);
8501  Flush();
8502 }
8503 
8504 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
8505  VmaAllocation allocation)
8506 {
8507  CallParams callParams;
8508  GetBasicParams(callParams);
8509 
8510  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8511  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
8512  allocation);
8513  Flush();
8514 }
8515 
8516 void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex,
8517  VmaPool pool)
8518 {
8519  CallParams callParams;
8520  GetBasicParams(callParams);
8521 
8522  VmaMutexLock lock(m_FileMutex, m_UseMutex);
8523  fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex,
8524  pool);
8525  Flush();
8526 }
8527 
8528 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
8529 {
8530  if(pUserData != VMA_NULL)
8531  {
8532  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
8533  {
8534  m_Str = (const char*)pUserData;
8535  }
8536  else
8537  {
8538  sprintf_s(m_PtrStr, "%p", pUserData);
8539  m_Str = m_PtrStr;
8540  }
8541  }
8542  else
8543  {
8544  m_Str = "";
8545  }
8546 }
8547 
8548 void VmaRecorder::WriteConfiguration(
8549  const VkPhysicalDeviceProperties& devProps,
8550  const VkPhysicalDeviceMemoryProperties& memProps,
8551  bool dedicatedAllocationExtensionEnabled)
8552 {
8553  fprintf(m_File, "Config,Begin\n");
8554 
8555  fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion);
8556  fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion);
8557  fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID);
8558  fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID);
8559  fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType);
8560  fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName);
8561 
8562  fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount);
8563  fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity);
8564  fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize);
8565 
8566  fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount);
8567  for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i)
8568  {
8569  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size);
8570  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags);
8571  }
8572  fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount);
8573  for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i)
8574  {
8575  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex);
8576  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags);
8577  }
8578 
8579  fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0);
8580 
8581  fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0);
8582  fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT);
8583  fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN);
8584  fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0);
8585  fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0);
8586  fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0);
8587  fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY);
8588  fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE);
8589  fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
8590 
8591  fprintf(m_File, "Config,End\n");
8592 }
8593 
8594 void VmaRecorder::GetBasicParams(CallParams& outParams)
8595 {
8596  outParams.threadId = GetCurrentThreadId();
8597 
8598  LARGE_INTEGER counter;
8599  QueryPerformanceCounter(&counter);
8600  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
8601 }
8602 
8603 void VmaRecorder::Flush()
8604 {
8605  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
8606  {
8607  fflush(m_File);
8608  }
8609 }
8610 
8611 #endif // #if VMA_RECORDING_ENABLED
8612 
8614 // VmaAllocator_T
8615 
8616 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
8617  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
8618  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
8619  m_hDevice(pCreateInfo->device),
8620  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
8621  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
8622  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
8623  m_PreferredLargeHeapBlockSize(0),
8624  m_PhysicalDevice(pCreateInfo->physicalDevice),
8625  m_CurrentFrameIndex(0),
8626  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
8627  m_NextPoolId(0)
8628 #if VMA_RECORDING_ENABLED
8629  ,m_pRecorder(VMA_NULL)
8630 #endif
8631 {
8632  if(VMA_DEBUG_DETECT_CORRUPTION)
8633  {
8634  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
8635  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
8636  }
8637 
8638  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
8639 
8640 #if !(VMA_DEDICATED_ALLOCATION)
8641  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
8642  {
8643  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
8644  }
8645 #endif
8646 
8647  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
8648  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
8649  memset(&m_MemProps, 0, sizeof(m_MemProps));
8650 
8651  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
8652  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
8653 
8654  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
8655  {
8656  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
8657  }
8658 
8659  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
8660  {
8661  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
8662  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
8663  }
8664 
8665  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
8666 
8667  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
8668  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
8669 
8670  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
8671  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
8672 
8673  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
8674  {
8675  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
8676  {
8677  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
8678  if(limit != VK_WHOLE_SIZE)
8679  {
8680  m_HeapSizeLimit[heapIndex] = limit;
8681  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
8682  {
8683  m_MemProps.memoryHeaps[heapIndex].size = limit;
8684  }
8685  }
8686  }
8687  }
8688 
8689  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8690  {
8691  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
8692 
8693  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
8694  this,
8695  memTypeIndex,
8696  preferredBlockSize,
8697  0,
8698  SIZE_MAX,
8699  GetBufferImageGranularity(),
8700  pCreateInfo->frameInUseCount,
8701  false); // isCustomPool
8702  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
8703  // becase minBlockCount is 0.
8704  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
8705 
8706  }
8707 }
8708 
8709 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
8710 {
8711  VkResult res = VK_SUCCESS;
8712 
8713  if(pCreateInfo->pRecordSettings != VMA_NULL &&
8714  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
8715  {
8716 #if VMA_RECORDING_ENABLED
8717  m_pRecorder = vma_new(this, VmaRecorder)();
8718  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
8719  if(res != VK_SUCCESS)
8720  {
8721  return res;
8722  }
8723  m_pRecorder->WriteConfiguration(
8724  m_PhysicalDeviceProperties,
8725  m_MemProps,
8726  m_UseKhrDedicatedAllocation);
8727  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
8728 #else
8729  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
8730  return VK_ERROR_FEATURE_NOT_PRESENT;
8731 #endif
8732  }
8733 
8734  return res;
8735 }
8736 
8737 VmaAllocator_T::~VmaAllocator_T()
8738 {
8739 #if VMA_RECORDING_ENABLED
8740  if(m_pRecorder != VMA_NULL)
8741  {
8742  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
8743  vma_delete(this, m_pRecorder);
8744  }
8745 #endif
8746 
8747  VMA_ASSERT(m_Pools.empty());
8748 
8749  for(size_t i = GetMemoryTypeCount(); i--; )
8750  {
8751  vma_delete(this, m_pDedicatedAllocations[i]);
8752  vma_delete(this, m_pBlockVectors[i]);
8753  }
8754 }
8755 
8756 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
8757 {
8758 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
8759  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
8760  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
8761  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
8762  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
8763  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
8764  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
8765  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
8766  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
8767  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
8768  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
8769  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
8770  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
8771  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
8772  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
8773  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
8774  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
8775 #if VMA_DEDICATED_ALLOCATION
8776  if(m_UseKhrDedicatedAllocation)
8777  {
8778  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
8779  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
8780  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
8781  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
8782  }
8783 #endif // #if VMA_DEDICATED_ALLOCATION
8784 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
8785 
8786 #define VMA_COPY_IF_NOT_NULL(funcName) \
8787  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
8788 
8789  if(pVulkanFunctions != VMA_NULL)
8790  {
8791  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
8792  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
8793  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
8794  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
8795  VMA_COPY_IF_NOT_NULL(vkMapMemory);
8796  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
8797  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
8798  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
8799  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
8800  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
8801  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
8802  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
8803  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
8804  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
8805  VMA_COPY_IF_NOT_NULL(vkCreateImage);
8806  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
8807 #if VMA_DEDICATED_ALLOCATION
8808  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
8809  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
8810 #endif
8811  }
8812 
8813 #undef VMA_COPY_IF_NOT_NULL
8814 
8815  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
8816  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
8817  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
8818  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
8819  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
8820  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
8821  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
8822  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
8823  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
8824  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
8825  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
8826  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
8827  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
8828  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
8829  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
8830  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
8831  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
8832  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
8833 #if VMA_DEDICATED_ALLOCATION
8834  if(m_UseKhrDedicatedAllocation)
8835  {
8836  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
8837  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
8838  }
8839 #endif
8840 }
8841 
8842 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
8843 {
8844  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
8845  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
8846  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
8847  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
8848 }
8849 
8850 VkResult VmaAllocator_T::AllocateMemoryOfType(
8851  VkDeviceSize size,
8852  VkDeviceSize alignment,
8853  bool dedicatedAllocation,
8854  VkBuffer dedicatedBuffer,
8855  VkImage dedicatedImage,
8856  const VmaAllocationCreateInfo& createInfo,
8857  uint32_t memTypeIndex,
8858  VmaSuballocationType suballocType,
8859  VmaAllocation* pAllocation)
8860 {
8861  VMA_ASSERT(pAllocation != VMA_NULL);
8862  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
8863 
8864  VmaAllocationCreateInfo finalCreateInfo = createInfo;
8865 
8866  // If memory type is not HOST_VISIBLE, disable MAPPED.
8867  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
8868  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
8869  {
8870  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
8871  }
8872 
8873  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
8874  VMA_ASSERT(blockVector);
8875 
8876  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
8877  bool preferDedicatedMemory =
8878  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
8879  dedicatedAllocation ||
8880  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
8881  size > preferredBlockSize / 2;
8882 
8883  if(preferDedicatedMemory &&
8884  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
8885  finalCreateInfo.pool == VK_NULL_HANDLE)
8886  {
8887  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
8888  }
8889 
8890  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
8891  {
8892  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
8893  {
8894  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
8895  }
8896  else
8897  {
8898  return AllocateDedicatedMemory(
8899  size,
8900  suballocType,
8901  memTypeIndex,
8902  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
8903  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
8904  finalCreateInfo.pUserData,
8905  dedicatedBuffer,
8906  dedicatedImage,
8907  pAllocation);
8908  }
8909  }
8910  else
8911  {
8912  VkResult res = blockVector->Allocate(
8913  VK_NULL_HANDLE, // hCurrentPool
8914  m_CurrentFrameIndex.load(),
8915  size,
8916  alignment,
8917  finalCreateInfo,
8918  suballocType,
8919  pAllocation);
8920  if(res == VK_SUCCESS)
8921  {
8922  return res;
8923  }
8924 
8925  // 5. Try dedicated memory.
8926  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
8927  {
8928  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
8929  }
8930  else
8931  {
8932  res = AllocateDedicatedMemory(
8933  size,
8934  suballocType,
8935  memTypeIndex,
8936  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
8937  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
8938  finalCreateInfo.pUserData,
8939  dedicatedBuffer,
8940  dedicatedImage,
8941  pAllocation);
8942  if(res == VK_SUCCESS)
8943  {
8944  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
8945  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
8946  return VK_SUCCESS;
8947  }
8948  else
8949  {
8950  // Everything failed: Return error code.
8951  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
8952  return res;
8953  }
8954  }
8955  }
8956 }
8957 
8958 VkResult VmaAllocator_T::AllocateDedicatedMemory(
8959  VkDeviceSize size,
8960  VmaSuballocationType suballocType,
8961  uint32_t memTypeIndex,
8962  bool map,
8963  bool isUserDataString,
8964  void* pUserData,
8965  VkBuffer dedicatedBuffer,
8966  VkImage dedicatedImage,
8967  VmaAllocation* pAllocation)
8968 {
8969  VMA_ASSERT(pAllocation);
8970 
8971  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
8972  allocInfo.memoryTypeIndex = memTypeIndex;
8973  allocInfo.allocationSize = size;
8974 
8975 #if VMA_DEDICATED_ALLOCATION
8976  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
8977  if(m_UseKhrDedicatedAllocation)
8978  {
8979  if(dedicatedBuffer != VK_NULL_HANDLE)
8980  {
8981  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
8982  dedicatedAllocInfo.buffer = dedicatedBuffer;
8983  allocInfo.pNext = &dedicatedAllocInfo;
8984  }
8985  else if(dedicatedImage != VK_NULL_HANDLE)
8986  {
8987  dedicatedAllocInfo.image = dedicatedImage;
8988  allocInfo.pNext = &dedicatedAllocInfo;
8989  }
8990  }
8991 #endif // #if VMA_DEDICATED_ALLOCATION
8992 
8993  // Allocate VkDeviceMemory.
8994  VkDeviceMemory hMemory = VK_NULL_HANDLE;
8995  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
8996  if(res < 0)
8997  {
8998  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
8999  return res;
9000  }
9001 
9002  void* pMappedData = VMA_NULL;
9003  if(map)
9004  {
9005  res = (*m_VulkanFunctions.vkMapMemory)(
9006  m_hDevice,
9007  hMemory,
9008  0,
9009  VK_WHOLE_SIZE,
9010  0,
9011  &pMappedData);
9012  if(res < 0)
9013  {
9014  VMA_DEBUG_LOG(" vkMapMemory FAILED");
9015  FreeVulkanMemory(memTypeIndex, size, hMemory);
9016  return res;
9017  }
9018  }
9019 
9020  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
9021  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
9022  (*pAllocation)->SetUserData(this, pUserData);
9023  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9024  {
9025  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9026  }
9027 
9028  // Register it in m_pDedicatedAllocations.
9029  {
9030  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9031  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
9032  VMA_ASSERT(pDedicatedAllocations);
9033  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
9034  }
9035 
9036  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
9037 
9038  return VK_SUCCESS;
9039 }
9040 
9041 void VmaAllocator_T::GetBufferMemoryRequirements(
9042  VkBuffer hBuffer,
9043  VkMemoryRequirements& memReq,
9044  bool& requiresDedicatedAllocation,
9045  bool& prefersDedicatedAllocation) const
9046 {
9047 #if VMA_DEDICATED_ALLOCATION
9048  if(m_UseKhrDedicatedAllocation)
9049  {
9050  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
9051  memReqInfo.buffer = hBuffer;
9052 
9053  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
9054 
9055  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
9056  memReq2.pNext = &memDedicatedReq;
9057 
9058  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
9059 
9060  memReq = memReq2.memoryRequirements;
9061  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
9062  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
9063  }
9064  else
9065 #endif // #if VMA_DEDICATED_ALLOCATION
9066  {
9067  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
9068  requiresDedicatedAllocation = false;
9069  prefersDedicatedAllocation = false;
9070  }
9071 }
9072 
9073 void VmaAllocator_T::GetImageMemoryRequirements(
9074  VkImage hImage,
9075  VkMemoryRequirements& memReq,
9076  bool& requiresDedicatedAllocation,
9077  bool& prefersDedicatedAllocation) const
9078 {
9079 #if VMA_DEDICATED_ALLOCATION
9080  if(m_UseKhrDedicatedAllocation)
9081  {
9082  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
9083  memReqInfo.image = hImage;
9084 
9085  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
9086 
9087  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
9088  memReq2.pNext = &memDedicatedReq;
9089 
9090  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
9091 
9092  memReq = memReq2.memoryRequirements;
9093  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
9094  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
9095  }
9096  else
9097 #endif // #if VMA_DEDICATED_ALLOCATION
9098  {
9099  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
9100  requiresDedicatedAllocation = false;
9101  prefersDedicatedAllocation = false;
9102  }
9103 }
9104 
9105 VkResult VmaAllocator_T::AllocateMemory(
9106  const VkMemoryRequirements& vkMemReq,
9107  bool requiresDedicatedAllocation,
9108  bool prefersDedicatedAllocation,
9109  VkBuffer dedicatedBuffer,
9110  VkImage dedicatedImage,
9111  const VmaAllocationCreateInfo& createInfo,
9112  VmaSuballocationType suballocType,
9113  VmaAllocation* pAllocation)
9114 {
9115  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
9116  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
9117  {
9118  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
9119  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9120  }
9121  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
9122  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
9123  {
9124  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
9125  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9126  }
9127  if(requiresDedicatedAllocation)
9128  {
9129  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
9130  {
9131  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
9132  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9133  }
9134  if(createInfo.pool != VK_NULL_HANDLE)
9135  {
9136  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
9137  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9138  }
9139  }
9140  if((createInfo.pool != VK_NULL_HANDLE) &&
9141  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
9142  {
9143  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
9144  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9145  }
9146 
9147  if(createInfo.pool != VK_NULL_HANDLE)
9148  {
9149  const VkDeviceSize alignmentForPool = VMA_MAX(
9150  vkMemReq.alignment,
9151  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
9152  return createInfo.pool->m_BlockVector.Allocate(
9153  createInfo.pool,
9154  m_CurrentFrameIndex.load(),
9155  vkMemReq.size,
9156  alignmentForPool,
9157  createInfo,
9158  suballocType,
9159  pAllocation);
9160  }
9161  else
9162  {
9163  // Bit mask of memory Vulkan types acceptable for this allocation.
9164  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
9165  uint32_t memTypeIndex = UINT32_MAX;
9166  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
9167  if(res == VK_SUCCESS)
9168  {
9169  VkDeviceSize alignmentForMemType = VMA_MAX(
9170  vkMemReq.alignment,
9171  GetMemoryTypeMinAlignment(memTypeIndex));
9172 
9173  res = AllocateMemoryOfType(
9174  vkMemReq.size,
9175  alignmentForMemType,
9176  requiresDedicatedAllocation || prefersDedicatedAllocation,
9177  dedicatedBuffer,
9178  dedicatedImage,
9179  createInfo,
9180  memTypeIndex,
9181  suballocType,
9182  pAllocation);
9183  // Succeeded on first try.
9184  if(res == VK_SUCCESS)
9185  {
9186  return res;
9187  }
9188  // Allocation from this memory type failed. Try other compatible memory types.
9189  else
9190  {
9191  for(;;)
9192  {
9193  // Remove old memTypeIndex from list of possibilities.
9194  memoryTypeBits &= ~(1u << memTypeIndex);
9195  // Find alternative memTypeIndex.
9196  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
9197  if(res == VK_SUCCESS)
9198  {
9199  alignmentForMemType = VMA_MAX(
9200  vkMemReq.alignment,
9201  GetMemoryTypeMinAlignment(memTypeIndex));
9202 
9203  res = AllocateMemoryOfType(
9204  vkMemReq.size,
9205  alignmentForMemType,
9206  requiresDedicatedAllocation || prefersDedicatedAllocation,
9207  dedicatedBuffer,
9208  dedicatedImage,
9209  createInfo,
9210  memTypeIndex,
9211  suballocType,
9212  pAllocation);
9213  // Allocation from this alternative memory type succeeded.
9214  if(res == VK_SUCCESS)
9215  {
9216  return res;
9217  }
9218  // else: Allocation from this memory type failed. Try next one - next loop iteration.
9219  }
9220  // No other matching memory type index could be found.
9221  else
9222  {
9223  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
9224  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9225  }
9226  }
9227  }
9228  }
9229  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
9230  else
9231  return res;
9232  }
9233 }
9234 
9235 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
9236 {
9237  VMA_ASSERT(allocation);
9238 
9239  if(allocation->CanBecomeLost() == false ||
9240  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
9241  {
9242  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9243  {
9244  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
9245  }
9246 
9247  switch(allocation->GetType())
9248  {
9249  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9250  {
9251  VmaBlockVector* pBlockVector = VMA_NULL;
9252  VmaPool hPool = allocation->GetPool();
9253  if(hPool != VK_NULL_HANDLE)
9254  {
9255  pBlockVector = &hPool->m_BlockVector;
9256  }
9257  else
9258  {
9259  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
9260  pBlockVector = m_pBlockVectors[memTypeIndex];
9261  }
9262  pBlockVector->Free(allocation);
9263  }
9264  break;
9265  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9266  FreeDedicatedMemory(allocation);
9267  break;
9268  default:
9269  VMA_ASSERT(0);
9270  }
9271  }
9272 
9273  allocation->SetUserData(this, VMA_NULL);
9274  vma_delete(this, allocation);
9275 }
9276 
9277 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
9278 {
9279  // Initialize.
9280  InitStatInfo(pStats->total);
9281  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
9282  InitStatInfo(pStats->memoryType[i]);
9283  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
9284  InitStatInfo(pStats->memoryHeap[i]);
9285 
9286  // Process default pools.
9287  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9288  {
9289  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
9290  VMA_ASSERT(pBlockVector);
9291  pBlockVector->AddStats(pStats);
9292  }
9293 
9294  // Process custom pools.
9295  {
9296  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9297  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
9298  {
9299  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
9300  }
9301  }
9302 
9303  // Process dedicated allocations.
9304  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9305  {
9306  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
9307  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9308  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
9309  VMA_ASSERT(pDedicatedAllocVector);
9310  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
9311  {
9312  VmaStatInfo allocationStatInfo;
9313  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
9314  VmaAddStatInfo(pStats->total, allocationStatInfo);
9315  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
9316  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
9317  }
9318  }
9319 
9320  // Postprocess.
9321  VmaPostprocessCalcStatInfo(pStats->total);
9322  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
9323  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
9324  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
9325  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
9326 }
9327 
9328 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
9329 
9330 VkResult VmaAllocator_T::Defragment(
9331  VmaAllocation* pAllocations,
9332  size_t allocationCount,
9333  VkBool32* pAllocationsChanged,
9334  const VmaDefragmentationInfo* pDefragmentationInfo,
9335  VmaDefragmentationStats* pDefragmentationStats)
9336 {
9337  if(pAllocationsChanged != VMA_NULL)
9338  {
9339  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
9340  }
9341  if(pDefragmentationStats != VMA_NULL)
9342  {
9343  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
9344  }
9345 
9346  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
9347 
9348  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
9349 
9350  const size_t poolCount = m_Pools.size();
9351 
9352  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
9353  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
9354  {
9355  VmaAllocation hAlloc = pAllocations[allocIndex];
9356  VMA_ASSERT(hAlloc);
9357  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
9358  // DedicatedAlloc cannot be defragmented.
9359  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
9360  // Only HOST_VISIBLE memory types can be defragmented.
9361  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
9362  // Lost allocation cannot be defragmented.
9363  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
9364  {
9365  VmaBlockVector* pAllocBlockVector = VMA_NULL;
9366 
9367  const VmaPool hAllocPool = hAlloc->GetPool();
9368  // This allocation belongs to custom pool.
9369  if(hAllocPool != VK_NULL_HANDLE)
9370  {
9371  pAllocBlockVector = &hAllocPool->GetBlockVector();
9372  }
9373  // This allocation belongs to general pool.
9374  else
9375  {
9376  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
9377  }
9378 
9379  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
9380 
9381  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
9382  &pAllocationsChanged[allocIndex] : VMA_NULL;
9383  pDefragmentator->AddAllocation(hAlloc, pChanged);
9384  }
9385  }
9386 
9387  VkResult result = VK_SUCCESS;
9388 
9389  // ======== Main processing.
9390 
9391  VkDeviceSize maxBytesToMove = SIZE_MAX;
9392  uint32_t maxAllocationsToMove = UINT32_MAX;
9393  if(pDefragmentationInfo != VMA_NULL)
9394  {
9395  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
9396  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
9397  }
9398 
9399  // Process standard memory.
9400  for(uint32_t memTypeIndex = 0;
9401  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
9402  ++memTypeIndex)
9403  {
9404  // Only HOST_VISIBLE memory types can be defragmented.
9405  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9406  {
9407  result = m_pBlockVectors[memTypeIndex]->Defragment(
9408  pDefragmentationStats,
9409  maxBytesToMove,
9410  maxAllocationsToMove);
9411  }
9412  }
9413 
9414  // Process custom pools.
9415  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
9416  {
9417  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
9418  pDefragmentationStats,
9419  maxBytesToMove,
9420  maxAllocationsToMove);
9421  }
9422 
9423  // ======== Destroy defragmentators.
9424 
9425  // Process custom pools.
9426  for(size_t poolIndex = poolCount; poolIndex--; )
9427  {
9428  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
9429  }
9430 
9431  // Process standard memory.
9432  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
9433  {
9434  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9435  {
9436  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
9437  }
9438  }
9439 
9440  return result;
9441 }
9442 
9443 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
9444 {
9445  if(hAllocation->CanBecomeLost())
9446  {
9447  /*
9448  Warning: This is a carefully designed algorithm.
9449  Do not modify unless you really know what you're doing :)
9450  */
9451  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9452  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9453  for(;;)
9454  {
9455  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
9456  {
9457  pAllocationInfo->memoryType = UINT32_MAX;
9458  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
9459  pAllocationInfo->offset = 0;
9460  pAllocationInfo->size = hAllocation->GetSize();
9461  pAllocationInfo->pMappedData = VMA_NULL;
9462  pAllocationInfo->pUserData = hAllocation->GetUserData();
9463  return;
9464  }
9465  else if(localLastUseFrameIndex == localCurrFrameIndex)
9466  {
9467  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
9468  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
9469  pAllocationInfo->offset = hAllocation->GetOffset();
9470  pAllocationInfo->size = hAllocation->GetSize();
9471  pAllocationInfo->pMappedData = VMA_NULL;
9472  pAllocationInfo->pUserData = hAllocation->GetUserData();
9473  return;
9474  }
9475  else // Last use time earlier than current time.
9476  {
9477  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9478  {
9479  localLastUseFrameIndex = localCurrFrameIndex;
9480  }
9481  }
9482  }
9483  }
9484  else
9485  {
9486 #if VMA_STATS_STRING_ENABLED
9487  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9488  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9489  for(;;)
9490  {
9491  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
9492  if(localLastUseFrameIndex == localCurrFrameIndex)
9493  {
9494  break;
9495  }
9496  else // Last use time earlier than current time.
9497  {
9498  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9499  {
9500  localLastUseFrameIndex = localCurrFrameIndex;
9501  }
9502  }
9503  }
9504 #endif
9505 
9506  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
9507  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
9508  pAllocationInfo->offset = hAllocation->GetOffset();
9509  pAllocationInfo->size = hAllocation->GetSize();
9510  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
9511  pAllocationInfo->pUserData = hAllocation->GetUserData();
9512  }
9513 }
9514 
9515 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
9516 {
9517  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
9518  if(hAllocation->CanBecomeLost())
9519  {
9520  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9521  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9522  for(;;)
9523  {
9524  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
9525  {
9526  return false;
9527  }
9528  else if(localLastUseFrameIndex == localCurrFrameIndex)
9529  {
9530  return true;
9531  }
9532  else // Last use time earlier than current time.
9533  {
9534  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9535  {
9536  localLastUseFrameIndex = localCurrFrameIndex;
9537  }
9538  }
9539  }
9540  }
9541  else
9542  {
9543 #if VMA_STATS_STRING_ENABLED
9544  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
9545  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
9546  for(;;)
9547  {
9548  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
9549  if(localLastUseFrameIndex == localCurrFrameIndex)
9550  {
9551  break;
9552  }
9553  else // Last use time earlier than current time.
9554  {
9555  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
9556  {
9557  localLastUseFrameIndex = localCurrFrameIndex;
9558  }
9559  }
9560  }
9561 #endif
9562 
9563  return true;
9564  }
9565 }
9566 
9567 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
9568 {
9569  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
9570 
9571  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
9572 
9573  if(newCreateInfo.maxBlockCount == 0)
9574  {
9575  newCreateInfo.maxBlockCount = SIZE_MAX;
9576  }
9577  if(newCreateInfo.blockSize == 0)
9578  {
9579  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
9580  }
9581 
9582  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
9583 
9584  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
9585  if(res != VK_SUCCESS)
9586  {
9587  vma_delete(this, *pPool);
9588  *pPool = VMA_NULL;
9589  return res;
9590  }
9591 
9592  // Add to m_Pools.
9593  {
9594  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9595  (*pPool)->SetId(m_NextPoolId++);
9596  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
9597  }
9598 
9599  return VK_SUCCESS;
9600 }
9601 
9602 void VmaAllocator_T::DestroyPool(VmaPool pool)
9603 {
9604  // Remove from m_Pools.
9605  {
9606  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9607  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
9608  VMA_ASSERT(success && "Pool not found in Allocator.");
9609  }
9610 
9611  vma_delete(this, pool);
9612 }
9613 
9614 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
9615 {
9616  pool->m_BlockVector.GetPoolStats(pPoolStats);
9617 }
9618 
9619 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
9620 {
9621  m_CurrentFrameIndex.store(frameIndex);
9622 }
9623 
9624 void VmaAllocator_T::MakePoolAllocationsLost(
9625  VmaPool hPool,
9626  size_t* pLostAllocationCount)
9627 {
9628  hPool->m_BlockVector.MakePoolAllocationsLost(
9629  m_CurrentFrameIndex.load(),
9630  pLostAllocationCount);
9631 }
9632 
9633 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
9634 {
9635  return hPool->m_BlockVector.CheckCorruption();
9636 }
9637 
9638 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
9639 {
9640  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
9641 
9642  // Process default pools.
9643  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9644  {
9645  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
9646  {
9647  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
9648  VMA_ASSERT(pBlockVector);
9649  VkResult localRes = pBlockVector->CheckCorruption();
9650  switch(localRes)
9651  {
9652  case VK_ERROR_FEATURE_NOT_PRESENT:
9653  break;
9654  case VK_SUCCESS:
9655  finalRes = VK_SUCCESS;
9656  break;
9657  default:
9658  return localRes;
9659  }
9660  }
9661  }
9662 
9663  // Process custom pools.
9664  {
9665  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
9666  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
9667  {
9668  if(((1u << m_Pools[poolIndex]->GetBlockVector().GetMemoryTypeIndex()) & memoryTypeBits) != 0)
9669  {
9670  VkResult localRes = m_Pools[poolIndex]->GetBlockVector().CheckCorruption();
9671  switch(localRes)
9672  {
9673  case VK_ERROR_FEATURE_NOT_PRESENT:
9674  break;
9675  case VK_SUCCESS:
9676  finalRes = VK_SUCCESS;
9677  break;
9678  default:
9679  return localRes;
9680  }
9681  }
9682  }
9683  }
9684 
9685  return finalRes;
9686 }
9687 
9688 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
9689 {
9690  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
9691  (*pAllocation)->InitLost();
9692 }
9693 
9694 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
9695 {
9696  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
9697 
9698  VkResult res;
9699  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
9700  {
9701  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
9702  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
9703  {
9704  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
9705  if(res == VK_SUCCESS)
9706  {
9707  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
9708  }
9709  }
9710  else
9711  {
9712  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
9713  }
9714  }
9715  else
9716  {
9717  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
9718  }
9719 
9720  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
9721  {
9722  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
9723  }
9724 
9725  return res;
9726 }
9727 
9728 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
9729 {
9730  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
9731  {
9732  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
9733  }
9734 
9735  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
9736 
9737  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
9738  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
9739  {
9740  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
9741  m_HeapSizeLimit[heapIndex] += size;
9742  }
9743 }
9744 
9745 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
9746 {
9747  if(hAllocation->CanBecomeLost())
9748  {
9749  return VK_ERROR_MEMORY_MAP_FAILED;
9750  }
9751 
9752  switch(hAllocation->GetType())
9753  {
9754  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9755  {
9756  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
9757  char *pBytes = VMA_NULL;
9758  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
9759  if(res == VK_SUCCESS)
9760  {
9761  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
9762  hAllocation->BlockAllocMap();
9763  }
9764  return res;
9765  }
9766  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9767  return hAllocation->DedicatedAllocMap(this, ppData);
9768  default:
9769  VMA_ASSERT(0);
9770  return VK_ERROR_MEMORY_MAP_FAILED;
9771  }
9772 }
9773 
9774 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
9775 {
9776  switch(hAllocation->GetType())
9777  {
9778  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9779  {
9780  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
9781  hAllocation->BlockAllocUnmap();
9782  pBlock->Unmap(this, 1);
9783  }
9784  break;
9785  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9786  hAllocation->DedicatedAllocUnmap(this);
9787  break;
9788  default:
9789  VMA_ASSERT(0);
9790  }
9791 }
9792 
9793 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
9794 {
9795  VkResult res = VK_SUCCESS;
9796  switch(hAllocation->GetType())
9797  {
9798  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9799  res = GetVulkanFunctions().vkBindBufferMemory(
9800  m_hDevice,
9801  hBuffer,
9802  hAllocation->GetMemory(),
9803  0); //memoryOffset
9804  break;
9805  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9806  {
9807  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9808  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
9809  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
9810  break;
9811  }
9812  default:
9813  VMA_ASSERT(0);
9814  }
9815  return res;
9816 }
9817 
9818 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
9819 {
9820  VkResult res = VK_SUCCESS;
9821  switch(hAllocation->GetType())
9822  {
9823  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9824  res = GetVulkanFunctions().vkBindImageMemory(
9825  m_hDevice,
9826  hImage,
9827  hAllocation->GetMemory(),
9828  0); //memoryOffset
9829  break;
9830  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9831  {
9832  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9833  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
9834  res = pBlock->BindImageMemory(this, hAllocation, hImage);
9835  break;
9836  }
9837  default:
9838  VMA_ASSERT(0);
9839  }
9840  return res;
9841 }
9842 
9843 void VmaAllocator_T::FlushOrInvalidateAllocation(
9844  VmaAllocation hAllocation,
9845  VkDeviceSize offset, VkDeviceSize size,
9846  VMA_CACHE_OPERATION op)
9847 {
9848  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
9849  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
9850  {
9851  const VkDeviceSize allocationSize = hAllocation->GetSize();
9852  VMA_ASSERT(offset <= allocationSize);
9853 
9854  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
9855 
9856  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
9857  memRange.memory = hAllocation->GetMemory();
9858 
9859  switch(hAllocation->GetType())
9860  {
9861  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
9862  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
9863  if(size == VK_WHOLE_SIZE)
9864  {
9865  memRange.size = allocationSize - memRange.offset;
9866  }
9867  else
9868  {
9869  VMA_ASSERT(offset + size <= allocationSize);
9870  memRange.size = VMA_MIN(
9871  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
9872  allocationSize - memRange.offset);
9873  }
9874  break;
9875 
9876  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
9877  {
9878  // 1. Still within this allocation.
9879  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
9880  if(size == VK_WHOLE_SIZE)
9881  {
9882  size = allocationSize - offset;
9883  }
9884  else
9885  {
9886  VMA_ASSERT(offset + size <= allocationSize);
9887  }
9888  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
9889 
9890  // 2. Adjust to whole block.
9891  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
9892  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
9893  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_Metadata.GetSize();
9894  memRange.offset += allocationOffset;
9895  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
9896 
9897  break;
9898  }
9899 
9900  default:
9901  VMA_ASSERT(0);
9902  }
9903 
9904  switch(op)
9905  {
9906  case VMA_CACHE_FLUSH:
9907  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
9908  break;
9909  case VMA_CACHE_INVALIDATE:
9910  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
9911  break;
9912  default:
9913  VMA_ASSERT(0);
9914  }
9915  }
9916  // else: Just ignore this call.
9917 }
9918 
9919 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
9920 {
9921  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
9922 
9923  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
9924  {
9925  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9926  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
9927  VMA_ASSERT(pDedicatedAllocations);
9928  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
9929  VMA_ASSERT(success);
9930  }
9931 
9932  VkDeviceMemory hMemory = allocation->GetMemory();
9933 
9934  if(allocation->GetMappedData() != VMA_NULL)
9935  {
9936  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
9937  }
9938 
9939  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
9940 
9941  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
9942 }
9943 
9944 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
9945 {
9946  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
9947  !hAllocation->CanBecomeLost() &&
9948  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
9949  {
9950  void* pData = VMA_NULL;
9951  VkResult res = Map(hAllocation, &pData);
9952  if(res == VK_SUCCESS)
9953  {
9954  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
9955  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
9956  Unmap(hAllocation);
9957  }
9958  else
9959  {
9960  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
9961  }
9962  }
9963 }
9964 
9965 #if VMA_STATS_STRING_ENABLED
9966 
9967 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
9968 {
9969  bool dedicatedAllocationsStarted = false;
9970  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
9971  {
9972  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
9973  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
9974  VMA_ASSERT(pDedicatedAllocVector);
9975  if(pDedicatedAllocVector->empty() == false)
9976  {
9977  if(dedicatedAllocationsStarted == false)
9978  {
9979  dedicatedAllocationsStarted = true;
9980  json.WriteString("DedicatedAllocations");
9981  json.BeginObject();
9982  }
9983 
9984  json.BeginString("Type ");
9985  json.ContinueString(memTypeIndex);
9986  json.EndString();
9987 
9988  json.BeginArray();
9989 
9990  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
9991  {
9992  json.BeginObject(true);
9993  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
9994  hAlloc->PrintParameters(json);
9995  json.EndObject();
9996  }
9997 
9998  json.EndArray();
9999  }
10000  }
10001  if(dedicatedAllocationsStarted)
10002  {
10003  json.EndObject();
10004  }
10005 
10006  {
10007  bool allocationsStarted = false;
10008  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
10009  {
10010  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
10011  {
10012  if(allocationsStarted == false)
10013  {
10014  allocationsStarted = true;
10015  json.WriteString("DefaultPools");
10016  json.BeginObject();
10017  }
10018 
10019  json.BeginString("Type ");
10020  json.ContinueString(memTypeIndex);
10021  json.EndString();
10022 
10023  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
10024  }
10025  }
10026  if(allocationsStarted)
10027  {
10028  json.EndObject();
10029  }
10030  }
10031 
10032  {
10033  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
10034  const size_t poolCount = m_Pools.size();
10035  if(poolCount > 0)
10036  {
10037  json.WriteString("Pools");
10038  json.BeginObject();
10039  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
10040  {
10041  json.BeginString();
10042  json.ContinueString(m_Pools[poolIndex]->GetId());
10043  json.EndString();
10044 
10045  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
10046  }
10047  json.EndObject();
10048  }
10049  }
10050 }
10051 
10052 #endif // #if VMA_STATS_STRING_ENABLED
10053 
10055 // Public interface
10056 
10057 VkResult vmaCreateAllocator(
10058  const VmaAllocatorCreateInfo* pCreateInfo,
10059  VmaAllocator* pAllocator)
10060 {
10061  VMA_ASSERT(pCreateInfo && pAllocator);
10062  VMA_DEBUG_LOG("vmaCreateAllocator");
10063  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
10064  return (*pAllocator)->Init(pCreateInfo);
10065 }
10066 
10067 void vmaDestroyAllocator(
10068  VmaAllocator allocator)
10069 {
10070  if(allocator != VK_NULL_HANDLE)
10071  {
10072  VMA_DEBUG_LOG("vmaDestroyAllocator");
10073  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
10074  vma_delete(&allocationCallbacks, allocator);
10075  }
10076 }
10077 
10078 void vmaGetPhysicalDeviceProperties(
10079  VmaAllocator allocator,
10080  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
10081 {
10082  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
10083  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
10084 }
10085 
10086 void vmaGetMemoryProperties(
10087  VmaAllocator allocator,
10088  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
10089 {
10090  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
10091  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
10092 }
10093 
10094 void vmaGetMemoryTypeProperties(
10095  VmaAllocator allocator,
10096  uint32_t memoryTypeIndex,
10097  VkMemoryPropertyFlags* pFlags)
10098 {
10099  VMA_ASSERT(allocator && pFlags);
10100  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
10101  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
10102 }
10103 
10104 void vmaSetCurrentFrameIndex(
10105  VmaAllocator allocator,
10106  uint32_t frameIndex)
10107 {
10108  VMA_ASSERT(allocator);
10109  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
10110 
10111  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10112 
10113  allocator->SetCurrentFrameIndex(frameIndex);
10114 }
10115 
10116 void vmaCalculateStats(
10117  VmaAllocator allocator,
10118  VmaStats* pStats)
10119 {
10120  VMA_ASSERT(allocator && pStats);
10121  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10122  allocator->CalculateStats(pStats);
10123 }
10124 
10125 #if VMA_STATS_STRING_ENABLED
10126 
10127 void vmaBuildStatsString(
10128  VmaAllocator allocator,
10129  char** ppStatsString,
10130  VkBool32 detailedMap)
10131 {
10132  VMA_ASSERT(allocator && ppStatsString);
10133  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10134 
10135  VmaStringBuilder sb(allocator);
10136  {
10137  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
10138  json.BeginObject();
10139 
10140  VmaStats stats;
10141  allocator->CalculateStats(&stats);
10142 
10143  json.WriteString("Total");
10144  VmaPrintStatInfo(json, stats.total);
10145 
10146  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
10147  {
10148  json.BeginString("Heap ");
10149  json.ContinueString(heapIndex);
10150  json.EndString();
10151  json.BeginObject();
10152 
10153  json.WriteString("Size");
10154  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
10155 
10156  json.WriteString("Flags");
10157  json.BeginArray(true);
10158  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
10159  {
10160  json.WriteString("DEVICE_LOCAL");
10161  }
10162  json.EndArray();
10163 
10164  if(stats.memoryHeap[heapIndex].blockCount > 0)
10165  {
10166  json.WriteString("Stats");
10167  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
10168  }
10169 
10170  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
10171  {
10172  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
10173  {
10174  json.BeginString("Type ");
10175  json.ContinueString(typeIndex);
10176  json.EndString();
10177 
10178  json.BeginObject();
10179 
10180  json.WriteString("Flags");
10181  json.BeginArray(true);
10182  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
10183  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
10184  {
10185  json.WriteString("DEVICE_LOCAL");
10186  }
10187  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
10188  {
10189  json.WriteString("HOST_VISIBLE");
10190  }
10191  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
10192  {
10193  json.WriteString("HOST_COHERENT");
10194  }
10195  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
10196  {
10197  json.WriteString("HOST_CACHED");
10198  }
10199  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
10200  {
10201  json.WriteString("LAZILY_ALLOCATED");
10202  }
10203  json.EndArray();
10204 
10205  if(stats.memoryType[typeIndex].blockCount > 0)
10206  {
10207  json.WriteString("Stats");
10208  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
10209  }
10210 
10211  json.EndObject();
10212  }
10213  }
10214 
10215  json.EndObject();
10216  }
10217  if(detailedMap == VK_TRUE)
10218  {
10219  allocator->PrintDetailedMap(json);
10220  }
10221 
10222  json.EndObject();
10223  }
10224 
10225  const size_t len = sb.GetLength();
10226  char* const pChars = vma_new_array(allocator, char, len + 1);
10227  if(len > 0)
10228  {
10229  memcpy(pChars, sb.GetData(), len);
10230  }
10231  pChars[len] = '\0';
10232  *ppStatsString = pChars;
10233 }
10234 
10235 void vmaFreeStatsString(
10236  VmaAllocator allocator,
10237  char* pStatsString)
10238 {
10239  if(pStatsString != VMA_NULL)
10240  {
10241  VMA_ASSERT(allocator);
10242  size_t len = strlen(pStatsString);
10243  vma_delete_array(allocator, pStatsString, len + 1);
10244  }
10245 }
10246 
10247 #endif // #if VMA_STATS_STRING_ENABLED
10248 
10249 /*
10250 This function is not protected by any mutex because it just reads immutable data.
10251 */
10252 VkResult vmaFindMemoryTypeIndex(
10253  VmaAllocator allocator,
10254  uint32_t memoryTypeBits,
10255  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10256  uint32_t* pMemoryTypeIndex)
10257 {
10258  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10259  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10260  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10261 
10262  if(pAllocationCreateInfo->memoryTypeBits != 0)
10263  {
10264  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
10265  }
10266 
10267  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
10268  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
10269 
10270  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
10271  if(mapped)
10272  {
10273  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10274  }
10275 
10276  // Convert usage to requiredFlags and preferredFlags.
10277  switch(pAllocationCreateInfo->usage)
10278  {
10279  case VMA_MEMORY_USAGE_UNKNOWN:
10280  break;
10281  case VMA_MEMORY_USAGE_GPU_ONLY:
10282  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
10283  {
10284  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
10285  }
10286  break;
10287  case VMA_MEMORY_USAGE_CPU_ONLY:
10288  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
10289  break;
10290  case VMA_MEMORY_USAGE_CPU_TO_GPU:
10291  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10292  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
10293  {
10294  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
10295  }
10296  break;
10297  case VMA_MEMORY_USAGE_GPU_TO_CPU:
10298  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
10299  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
10300  break;
10301  default:
10302  break;
10303  }
10304 
10305  *pMemoryTypeIndex = UINT32_MAX;
10306  uint32_t minCost = UINT32_MAX;
10307  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
10308  memTypeIndex < allocator->GetMemoryTypeCount();
10309  ++memTypeIndex, memTypeBit <<= 1)
10310  {
10311  // This memory type is acceptable according to memoryTypeBits bitmask.
10312  if((memTypeBit & memoryTypeBits) != 0)
10313  {
10314  const VkMemoryPropertyFlags currFlags =
10315  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
10316  // This memory type contains requiredFlags.
10317  if((requiredFlags & ~currFlags) == 0)
10318  {
10319  // Calculate cost as number of bits from preferredFlags not present in this memory type.
10320  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
10321  // Remember memory type with lowest cost.
10322  if(currCost < minCost)
10323  {
10324  *pMemoryTypeIndex = memTypeIndex;
10325  if(currCost == 0)
10326  {
10327  return VK_SUCCESS;
10328  }
10329  minCost = currCost;
10330  }
10331  }
10332  }
10333  }
10334  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
10335 }
10336 
10337 VkResult vmaFindMemoryTypeIndexForBufferInfo(
10338  VmaAllocator allocator,
10339  const VkBufferCreateInfo* pBufferCreateInfo,
10340  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10341  uint32_t* pMemoryTypeIndex)
10342 {
10343  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10344  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
10345  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10346  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10347 
10348  const VkDevice hDev = allocator->m_hDevice;
10349  VkBuffer hBuffer = VK_NULL_HANDLE;
10350  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
10351  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
10352  if(res == VK_SUCCESS)
10353  {
10354  VkMemoryRequirements memReq = {};
10355  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
10356  hDev, hBuffer, &memReq);
10357 
10358  res = vmaFindMemoryTypeIndex(
10359  allocator,
10360  memReq.memoryTypeBits,
10361  pAllocationCreateInfo,
10362  pMemoryTypeIndex);
10363 
10364  allocator->GetVulkanFunctions().vkDestroyBuffer(
10365  hDev, hBuffer, allocator->GetAllocationCallbacks());
10366  }
10367  return res;
10368 }
10369 
10370 VkResult vmaFindMemoryTypeIndexForImageInfo(
10371  VmaAllocator allocator,
10372  const VkImageCreateInfo* pImageCreateInfo,
10373  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10374  uint32_t* pMemoryTypeIndex)
10375 {
10376  VMA_ASSERT(allocator != VK_NULL_HANDLE);
10377  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
10378  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
10379  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
10380 
10381  const VkDevice hDev = allocator->m_hDevice;
10382  VkImage hImage = VK_NULL_HANDLE;
10383  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
10384  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
10385  if(res == VK_SUCCESS)
10386  {
10387  VkMemoryRequirements memReq = {};
10388  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
10389  hDev, hImage, &memReq);
10390 
10391  res = vmaFindMemoryTypeIndex(
10392  allocator,
10393  memReq.memoryTypeBits,
10394  pAllocationCreateInfo,
10395  pMemoryTypeIndex);
10396 
10397  allocator->GetVulkanFunctions().vkDestroyImage(
10398  hDev, hImage, allocator->GetAllocationCallbacks());
10399  }
10400  return res;
10401 }
10402 
10403 VkResult vmaCreatePool(
10404  VmaAllocator allocator,
10405  const VmaPoolCreateInfo* pCreateInfo,
10406  VmaPool* pPool)
10407 {
10408  VMA_ASSERT(allocator && pCreateInfo && pPool);
10409 
10410  VMA_DEBUG_LOG("vmaCreatePool");
10411 
10412  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10413 
10414  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
10415 
10416 #if VMA_RECORDING_ENABLED
10417  if(allocator->GetRecorder() != VMA_NULL)
10418  {
10419  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
10420  }
10421 #endif
10422 
10423  return res;
10424 }
10425 
10426 void vmaDestroyPool(
10427  VmaAllocator allocator,
10428  VmaPool pool)
10429 {
10430  VMA_ASSERT(allocator);
10431 
10432  if(pool == VK_NULL_HANDLE)
10433  {
10434  return;
10435  }
10436 
10437  VMA_DEBUG_LOG("vmaDestroyPool");
10438 
10439  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10440 
10441 #if VMA_RECORDING_ENABLED
10442  if(allocator->GetRecorder() != VMA_NULL)
10443  {
10444  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
10445  }
10446 #endif
10447 
10448  allocator->DestroyPool(pool);
10449 }
10450 
10451 void vmaGetPoolStats(
10452  VmaAllocator allocator,
10453  VmaPool pool,
10454  VmaPoolStats* pPoolStats)
10455 {
10456  VMA_ASSERT(allocator && pool && pPoolStats);
10457 
10458  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10459 
10460  allocator->GetPoolStats(pool, pPoolStats);
10461 }
10462 
10463 void vmaMakePoolAllocationsLost(
10464  VmaAllocator allocator,
10465  VmaPool pool,
10466  size_t* pLostAllocationCount)
10467 {
10468  VMA_ASSERT(allocator && pool);
10469 
10470  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10471 
10472 #if VMA_RECORDING_ENABLED
10473  if(allocator->GetRecorder() != VMA_NULL)
10474  {
10475  allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool);
10476  }
10477 #endif
10478 
10479  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
10480 }
10481 
10482 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
10483 {
10484  VMA_ASSERT(allocator && pool);
10485 
10486  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10487 
10488  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
10489 
10490  return allocator->CheckPoolCorruption(pool);
10491 }
10492 
10493 VkResult vmaAllocateMemory(
10494  VmaAllocator allocator,
10495  const VkMemoryRequirements* pVkMemoryRequirements,
10496  const VmaAllocationCreateInfo* pCreateInfo,
10497  VmaAllocation* pAllocation,
10498  VmaAllocationInfo* pAllocationInfo)
10499 {
10500  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
10501 
10502  VMA_DEBUG_LOG("vmaAllocateMemory");
10503 
10504  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10505 
10506  VkResult result = allocator->AllocateMemory(
10507  *pVkMemoryRequirements,
10508  false, // requiresDedicatedAllocation
10509  false, // prefersDedicatedAllocation
10510  VK_NULL_HANDLE, // dedicatedBuffer
10511  VK_NULL_HANDLE, // dedicatedImage
10512  *pCreateInfo,
10513  VMA_SUBALLOCATION_TYPE_UNKNOWN,
10514  pAllocation);
10515 
10516 #if VMA_RECORDING_ENABLED
10517  if(allocator->GetRecorder() != VMA_NULL)
10518  {
10519  allocator->GetRecorder()->RecordAllocateMemory(
10520  allocator->GetCurrentFrameIndex(),
10521  *pVkMemoryRequirements,
10522  *pCreateInfo,
10523  *pAllocation);
10524  }
10525 #endif
10526 
10527  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
10528  {
10529  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10530  }
10531 
10532  return result;
10533 }
10534 
10535 VkResult vmaAllocateMemoryForBuffer(
10536  VmaAllocator allocator,
10537  VkBuffer buffer,
10538  const VmaAllocationCreateInfo* pCreateInfo,
10539  VmaAllocation* pAllocation,
10540  VmaAllocationInfo* pAllocationInfo)
10541 {
10542  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
10543 
10544  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
10545 
10546  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10547 
10548  VkMemoryRequirements vkMemReq = {};
10549  bool requiresDedicatedAllocation = false;
10550  bool prefersDedicatedAllocation = false;
10551  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
10552  requiresDedicatedAllocation,
10553  prefersDedicatedAllocation);
10554 
10555  VkResult result = allocator->AllocateMemory(
10556  vkMemReq,
10557  requiresDedicatedAllocation,
10558  prefersDedicatedAllocation,
10559  buffer, // dedicatedBuffer
10560  VK_NULL_HANDLE, // dedicatedImage
10561  *pCreateInfo,
10562  VMA_SUBALLOCATION_TYPE_BUFFER,
10563  pAllocation);
10564 
10565 #if VMA_RECORDING_ENABLED
10566  if(allocator->GetRecorder() != VMA_NULL)
10567  {
10568  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
10569  allocator->GetCurrentFrameIndex(),
10570  vkMemReq,
10571  requiresDedicatedAllocation,
10572  prefersDedicatedAllocation,
10573  *pCreateInfo,
10574  *pAllocation);
10575  }
10576 #endif
10577 
10578  if(pAllocationInfo && result == VK_SUCCESS)
10579  {
10580  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10581  }
10582 
10583  return result;
10584 }
10585 
10586 VkResult vmaAllocateMemoryForImage(
10587  VmaAllocator allocator,
10588  VkImage image,
10589  const VmaAllocationCreateInfo* pCreateInfo,
10590  VmaAllocation* pAllocation,
10591  VmaAllocationInfo* pAllocationInfo)
10592 {
10593  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
10594 
10595  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
10596 
10597  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10598 
10599  VkMemoryRequirements vkMemReq = {};
10600  bool requiresDedicatedAllocation = false;
10601  bool prefersDedicatedAllocation = false;
10602  allocator->GetImageMemoryRequirements(image, vkMemReq,
10603  requiresDedicatedAllocation, prefersDedicatedAllocation);
10604 
10605  VkResult result = allocator->AllocateMemory(
10606  vkMemReq,
10607  requiresDedicatedAllocation,
10608  prefersDedicatedAllocation,
10609  VK_NULL_HANDLE, // dedicatedBuffer
10610  image, // dedicatedImage
10611  *pCreateInfo,
10612  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
10613  pAllocation);
10614 
10615 #if VMA_RECORDING_ENABLED
10616  if(allocator->GetRecorder() != VMA_NULL)
10617  {
10618  allocator->GetRecorder()->RecordAllocateMemoryForImage(
10619  allocator->GetCurrentFrameIndex(),
10620  vkMemReq,
10621  requiresDedicatedAllocation,
10622  prefersDedicatedAllocation,
10623  *pCreateInfo,
10624  *pAllocation);
10625  }
10626 #endif
10627 
10628  if(pAllocationInfo && result == VK_SUCCESS)
10629  {
10630  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10631  }
10632 
10633  return result;
10634 }
10635 
10636 void vmaFreeMemory(
10637  VmaAllocator allocator,
10638  VmaAllocation allocation)
10639 {
10640  VMA_ASSERT(allocator);
10641 
10642  if(allocation == VK_NULL_HANDLE)
10643  {
10644  return;
10645  }
10646 
10647  VMA_DEBUG_LOG("vmaFreeMemory");
10648 
10649  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10650 
10651 #if VMA_RECORDING_ENABLED
10652  if(allocator->GetRecorder() != VMA_NULL)
10653  {
10654  allocator->GetRecorder()->RecordFreeMemory(
10655  allocator->GetCurrentFrameIndex(),
10656  allocation);
10657  }
10658 #endif
10659 
10660  allocator->FreeMemory(allocation);
10661 }
10662 
10663 void vmaGetAllocationInfo(
10664  VmaAllocator allocator,
10665  VmaAllocation allocation,
10666  VmaAllocationInfo* pAllocationInfo)
10667 {
10668  VMA_ASSERT(allocator && allocation && pAllocationInfo);
10669 
10670  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10671 
10672 #if VMA_RECORDING_ENABLED
10673  if(allocator->GetRecorder() != VMA_NULL)
10674  {
10675  allocator->GetRecorder()->RecordGetAllocationInfo(
10676  allocator->GetCurrentFrameIndex(),
10677  allocation);
10678  }
10679 #endif
10680 
10681  allocator->GetAllocationInfo(allocation, pAllocationInfo);
10682 }
10683 
10684 VkBool32 vmaTouchAllocation(
10685  VmaAllocator allocator,
10686  VmaAllocation allocation)
10687 {
10688  VMA_ASSERT(allocator && allocation);
10689 
10690  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10691 
10692 #if VMA_RECORDING_ENABLED
10693  if(allocator->GetRecorder() != VMA_NULL)
10694  {
10695  allocator->GetRecorder()->RecordTouchAllocation(
10696  allocator->GetCurrentFrameIndex(),
10697  allocation);
10698  }
10699 #endif
10700 
10701  return allocator->TouchAllocation(allocation);
10702 }
10703 
10704 void vmaSetAllocationUserData(
10705  VmaAllocator allocator,
10706  VmaAllocation allocation,
10707  void* pUserData)
10708 {
10709  VMA_ASSERT(allocator && allocation);
10710 
10711  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10712 
10713  allocation->SetUserData(allocator, pUserData);
10714 
10715 #if VMA_RECORDING_ENABLED
10716  if(allocator->GetRecorder() != VMA_NULL)
10717  {
10718  allocator->GetRecorder()->RecordSetAllocationUserData(
10719  allocator->GetCurrentFrameIndex(),
10720  allocation,
10721  pUserData);
10722  }
10723 #endif
10724 }
10725 
10726 void vmaCreateLostAllocation(
10727  VmaAllocator allocator,
10728  VmaAllocation* pAllocation)
10729 {
10730  VMA_ASSERT(allocator && pAllocation);
10731 
10732  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
10733 
10734  allocator->CreateLostAllocation(pAllocation);
10735 
10736 #if VMA_RECORDING_ENABLED
10737  if(allocator->GetRecorder() != VMA_NULL)
10738  {
10739  allocator->GetRecorder()->RecordCreateLostAllocation(
10740  allocator->GetCurrentFrameIndex(),
10741  *pAllocation);
10742  }
10743 #endif
10744 }
10745 
10746 VkResult vmaMapMemory(
10747  VmaAllocator allocator,
10748  VmaAllocation allocation,
10749  void** ppData)
10750 {
10751  VMA_ASSERT(allocator && allocation && ppData);
10752 
10753  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10754 
10755  VkResult res = allocator->Map(allocation, ppData);
10756 
10757 #if VMA_RECORDING_ENABLED
10758  if(allocator->GetRecorder() != VMA_NULL)
10759  {
10760  allocator->GetRecorder()->RecordMapMemory(
10761  allocator->GetCurrentFrameIndex(),
10762  allocation);
10763  }
10764 #endif
10765 
10766  return res;
10767 }
10768 
10769 void vmaUnmapMemory(
10770  VmaAllocator allocator,
10771  VmaAllocation allocation)
10772 {
10773  VMA_ASSERT(allocator && allocation);
10774 
10775  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10776 
10777 #if VMA_RECORDING_ENABLED
10778  if(allocator->GetRecorder() != VMA_NULL)
10779  {
10780  allocator->GetRecorder()->RecordUnmapMemory(
10781  allocator->GetCurrentFrameIndex(),
10782  allocation);
10783  }
10784 #endif
10785 
10786  allocator->Unmap(allocation);
10787 }
10788 
10789 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10790 {
10791  VMA_ASSERT(allocator && allocation);
10792 
10793  VMA_DEBUG_LOG("vmaFlushAllocation");
10794 
10795  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10796 
10797  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
10798 
10799 #if VMA_RECORDING_ENABLED
10800  if(allocator->GetRecorder() != VMA_NULL)
10801  {
10802  allocator->GetRecorder()->RecordFlushAllocation(
10803  allocator->GetCurrentFrameIndex(),
10804  allocation, offset, size);
10805  }
10806 #endif
10807 }
10808 
10809 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10810 {
10811  VMA_ASSERT(allocator && allocation);
10812 
10813  VMA_DEBUG_LOG("vmaInvalidateAllocation");
10814 
10815  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10816 
10817  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
10818 
10819 #if VMA_RECORDING_ENABLED
10820  if(allocator->GetRecorder() != VMA_NULL)
10821  {
10822  allocator->GetRecorder()->RecordInvalidateAllocation(
10823  allocator->GetCurrentFrameIndex(),
10824  allocation, offset, size);
10825  }
10826 #endif
10827 }
10828 
10829 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
10830 {
10831  VMA_ASSERT(allocator);
10832 
10833  VMA_DEBUG_LOG("vmaCheckCorruption");
10834 
10835  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10836 
10837  return allocator->CheckCorruption(memoryTypeBits);
10838 }
10839 
10840 VkResult vmaDefragment(
10841  VmaAllocator allocator,
10842  VmaAllocation* pAllocations,
10843  size_t allocationCount,
10844  VkBool32* pAllocationsChanged,
10845  const VmaDefragmentationInfo *pDefragmentationInfo,
10846  VmaDefragmentationStats* pDefragmentationStats)
10847 {
10848  VMA_ASSERT(allocator && pAllocations);
10849 
10850  VMA_DEBUG_LOG("vmaDefragment");
10851 
10852  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10853 
10854  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
10855 }
10856 
10857 VkResult vmaBindBufferMemory(
10858  VmaAllocator allocator,
10859  VmaAllocation allocation,
10860  VkBuffer buffer)
10861 {
10862  VMA_ASSERT(allocator && allocation && buffer);
10863 
10864  VMA_DEBUG_LOG("vmaBindBufferMemory");
10865 
10866  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10867 
10868  return allocator->BindBufferMemory(allocation, buffer);
10869 }
10870 
10871 VkResult vmaBindImageMemory(
10872  VmaAllocator allocator,
10873  VmaAllocation allocation,
10874  VkImage image)
10875 {
10876  VMA_ASSERT(allocator && allocation && image);
10877 
10878  VMA_DEBUG_LOG("vmaBindImageMemory");
10879 
10880  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10881 
10882  return allocator->BindImageMemory(allocation, image);
10883 }
10884 
10885 VkResult vmaCreateBuffer(
10886  VmaAllocator allocator,
10887  const VkBufferCreateInfo* pBufferCreateInfo,
10888  const VmaAllocationCreateInfo* pAllocationCreateInfo,
10889  VkBuffer* pBuffer,
10890  VmaAllocation* pAllocation,
10891  VmaAllocationInfo* pAllocationInfo)
10892 {
10893  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
10894 
10895  VMA_DEBUG_LOG("vmaCreateBuffer");
10896 
10897  VMA_DEBUG_GLOBAL_MUTEX_LOCK
10898 
10899  *pBuffer = VK_NULL_HANDLE;
10900  *pAllocation = VK_NULL_HANDLE;
10901 
10902  // 1. Create VkBuffer.
10903  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
10904  allocator->m_hDevice,
10905  pBufferCreateInfo,
10906  allocator->GetAllocationCallbacks(),
10907  pBuffer);
10908  if(res >= 0)
10909  {
10910  // 2. vkGetBufferMemoryRequirements.
10911  VkMemoryRequirements vkMemReq = {};
10912  bool requiresDedicatedAllocation = false;
10913  bool prefersDedicatedAllocation = false;
10914  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
10915  requiresDedicatedAllocation, prefersDedicatedAllocation);
10916 
10917  // Make sure alignment requirements for specific buffer usages reported
10918  // in Physical Device Properties are included in alignment reported by memory requirements.
10919  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
10920  {
10921  VMA_ASSERT(vkMemReq.alignment %
10922  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
10923  }
10924  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
10925  {
10926  VMA_ASSERT(vkMemReq.alignment %
10927  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
10928  }
10929  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
10930  {
10931  VMA_ASSERT(vkMemReq.alignment %
10932  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
10933  }
10934 
10935  // 3. Allocate memory using allocator.
10936  res = allocator->AllocateMemory(
10937  vkMemReq,
10938  requiresDedicatedAllocation,
10939  prefersDedicatedAllocation,
10940  *pBuffer, // dedicatedBuffer
10941  VK_NULL_HANDLE, // dedicatedImage
10942  *pAllocationCreateInfo,
10943  VMA_SUBALLOCATION_TYPE_BUFFER,
10944  pAllocation);
10945 
10946 #if VMA_RECORDING_ENABLED
10947  if(allocator->GetRecorder() != VMA_NULL)
10948  {
10949  allocator->GetRecorder()->RecordCreateBuffer(
10950  allocator->GetCurrentFrameIndex(),
10951  *pBufferCreateInfo,
10952  *pAllocationCreateInfo,
10953  *pAllocation);
10954  }
10955 #endif
10956 
10957  if(res >= 0)
10958  {
10959  // 3. Bind buffer with memory.
10960  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
10961  if(res >= 0)
10962  {
10963  // All steps succeeded.
10964  #if VMA_STATS_STRING_ENABLED
10965  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
10966  #endif
10967  if(pAllocationInfo != VMA_NULL)
10968  {
10969  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
10970  }
10971 
10972  return VK_SUCCESS;
10973  }
10974  allocator->FreeMemory(*pAllocation);
10975  *pAllocation = VK_NULL_HANDLE;
10976  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
10977  *pBuffer = VK_NULL_HANDLE;
10978  return res;
10979  }
10980  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
10981  *pBuffer = VK_NULL_HANDLE;
10982  return res;
10983  }
10984  return res;
10985 }
10986 
10987 void vmaDestroyBuffer(
10988  VmaAllocator allocator,
10989  VkBuffer buffer,
10990  VmaAllocation allocation)
10991 {
10992  VMA_ASSERT(allocator);
10993 
10994  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
10995  {
10996  return;
10997  }
10998 
10999  VMA_DEBUG_LOG("vmaDestroyBuffer");
11000 
11001  VMA_DEBUG_GLOBAL_MUTEX_LOCK
11002 
11003 #if VMA_RECORDING_ENABLED
11004  if(allocator->GetRecorder() != VMA_NULL)
11005  {
11006  allocator->GetRecorder()->RecordDestroyBuffer(
11007  allocator->GetCurrentFrameIndex(),
11008  allocation);
11009  }
11010 #endif
11011 
11012  if(buffer != VK_NULL_HANDLE)
11013  {
11014  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
11015  }
11016 
11017  if(allocation != VK_NULL_HANDLE)
11018  {
11019  allocator->FreeMemory(allocation);
11020  }
11021 }
11022 
11023 VkResult vmaCreateImage(
11024  VmaAllocator allocator,
11025  const VkImageCreateInfo* pImageCreateInfo,
11026  const VmaAllocationCreateInfo* pAllocationCreateInfo,
11027  VkImage* pImage,
11028  VmaAllocation* pAllocation,
11029  VmaAllocationInfo* pAllocationInfo)
11030 {
11031  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
11032 
11033  VMA_DEBUG_LOG("vmaCreateImage");
11034 
11035  VMA_DEBUG_GLOBAL_MUTEX_LOCK
11036 
11037  *pImage = VK_NULL_HANDLE;
11038  *pAllocation = VK_NULL_HANDLE;
11039 
11040  // 1. Create VkImage.
11041  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
11042  allocator->m_hDevice,
11043  pImageCreateInfo,
11044  allocator->GetAllocationCallbacks(),
11045  pImage);
11046  if(res >= 0)
11047  {
11048  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
11049  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
11050  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
11051 
11052  // 2. Allocate memory using allocator.
11053  VkMemoryRequirements vkMemReq = {};
11054  bool requiresDedicatedAllocation = false;
11055  bool prefersDedicatedAllocation = false;
11056  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
11057  requiresDedicatedAllocation, prefersDedicatedAllocation);
11058 
11059  res = allocator->AllocateMemory(
11060  vkMemReq,
11061  requiresDedicatedAllocation,
11062  prefersDedicatedAllocation,
11063  VK_NULL_HANDLE, // dedicatedBuffer
11064  *pImage, // dedicatedImage
11065  *pAllocationCreateInfo,
11066  suballocType,
11067  pAllocation);
11068 
11069 #if VMA_RECORDING_ENABLED
11070  if(allocator->GetRecorder() != VMA_NULL)
11071  {
11072  allocator->GetRecorder()->RecordCreateImage(
11073  allocator->GetCurrentFrameIndex(),
11074  *pImageCreateInfo,
11075  *pAllocationCreateInfo,
11076  *pAllocation);
11077  }
11078 #endif
11079 
11080  if(res >= 0)
11081  {
11082  // 3. Bind image with memory.
11083  res = allocator->BindImageMemory(*pAllocation, *pImage);
11084  if(res >= 0)
11085  {
11086  // All steps succeeded.
11087  #if VMA_STATS_STRING_ENABLED
11088  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
11089  #endif
11090  if(pAllocationInfo != VMA_NULL)
11091  {
11092  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
11093  }
11094 
11095  return VK_SUCCESS;
11096  }
11097  allocator->FreeMemory(*pAllocation);
11098  *pAllocation = VK_NULL_HANDLE;
11099  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
11100  *pImage = VK_NULL_HANDLE;
11101  return res;
11102  }
11103  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
11104  *pImage = VK_NULL_HANDLE;
11105  return res;
11106  }
11107  return res;
11108 }
11109 
11110 void vmaDestroyImage(
11111  VmaAllocator allocator,
11112  VkImage image,
11113  VmaAllocation allocation)
11114 {
11115  VMA_ASSERT(allocator);
11116 
11117  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
11118  {
11119  return;
11120  }
11121 
11122  VMA_DEBUG_LOG("vmaDestroyImage");
11123 
11124  VMA_DEBUG_GLOBAL_MUTEX_LOCK
11125 
11126 #if VMA_RECORDING_ENABLED
11127  if(allocator->GetRecorder() != VMA_NULL)
11128  {
11129  allocator->GetRecorder()->RecordDestroyImage(
11130  allocator->GetCurrentFrameIndex(),
11131  allocation);
11132  }
11133 #endif
11134 
11135  if(image != VK_NULL_HANDLE)
11136  {
11137  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
11138  }
11139  if(allocation != VK_NULL_HANDLE)
11140  {
11141  allocator->FreeMemory(allocation);
11142  }
11143 }
11144 
11145 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1345
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1658
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:1414
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index a908c90..5b8fcc0 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -29,7 +29,7 @@ extern "C" { /** \mainpage Vulkan Memory Allocator -Version 2.1.0-alpha.3 (2018-06-14) +Version 2.1.0-alpha.4 (2018-08-22) Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT