diff --git a/docs/html/memory_mapping.html b/docs/html/memory_mapping.html index 58da591..f4e37a0 100644 --- a/docs/html/memory_mapping.html +++ b/docs/html/memory_mapping.html @@ -86,7 +86,8 @@ Cache control

Finding out if memory is mappable

It may happen that your allocation ends up in memory that is HOST_VISIBLE (available for mapping) despite it wasn't explicitly requested. For example, application may work on integrated graphics with unified memory (like Intel) or allocation from video memory might have failed, so the library chose system memory as fallback.

-

You can detect this case and map such allocation to access its memory on CPU directly, instead of launching a transfer operation. You can even use VMA_ALLOCATION_CREATE_MAPPED_BIT flag while creating allocations that are not necessarily HOST_VISIBLE (e.g. using VMA_MEMORY_USAGE_GPU_ONLY). If the allocation ends up in memory type that is HOST_VISIBLE, it will be persistently mapped and you can use it directly. If not, the flag is just ignored. Example:

+

You can detect this case and map such allocation to access its memory on CPU directly, instead of launching a transfer operation. In order to do that: inspect allocInfo.memoryType, call vmaGetMemoryTypeProperties(), and look for VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT flag in properties of that memory type.

+

You can even use VMA_ALLOCATION_CREATE_MAPPED_BIT flag while creating allocations that are not necessarily HOST_VISIBLE (e.g. using VMA_MEMORY_USAGE_GPU_ONLY). If the allocation ends up in memory type that is HOST_VISIBLE, it will be persistently mapped and you can use it directly. If not, the flag is just ignored. Example:

VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufCreateInfo.size = sizeof(ConstantBuffer);
bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VmaAllocationCreateInfo allocCreateInfo = {};
allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
VkBuffer buf;
VmaAllocation alloc;
VmaAllocationInfo allocInfo;
vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
if(allocInfo.pUserData != nullptr)
{
// Allocation ended up in mappable memory.
// It's persistently mapped. You can access it directly.
memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
}
else
{
// Allocation ended up in non-mappable memory.
// You need to create CPU-side copy in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
}
diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 959443f..5a9e8b1 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,154 +62,154 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
823 #include <vulkan/vulkan.h>
824 
825 VK_DEFINE_HANDLE(VmaAllocator)
826 
827 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
829  VmaAllocator allocator,
830  uint32_t memoryType,
831  VkDeviceMemory memory,
832  VkDeviceSize size);
834 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
835  VmaAllocator allocator,
836  uint32_t memoryType,
837  VkDeviceMemory memory,
838  VkDeviceSize size);
839 
847 typedef struct VmaDeviceMemoryCallbacks {
849  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
851  PFN_vmaFreeDeviceMemoryFunction pfnFree;
852 } VmaDeviceMemoryCallbacks;
853 
855 typedef enum VmaAllocatorCreateFlagBits {
860  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
882  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
883 
884  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
885 } VmaAllocatorCreateFlagBits;
886 typedef VkFlags VmaAllocatorCreateFlags;
887 
892 typedef struct VmaVulkanFunctions {
893  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
894  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
895  PFN_vkAllocateMemory vkAllocateMemory;
896  PFN_vkFreeMemory vkFreeMemory;
897  PFN_vkMapMemory vkMapMemory;
898  PFN_vkUnmapMemory vkUnmapMemory;
899  PFN_vkBindBufferMemory vkBindBufferMemory;
900  PFN_vkBindImageMemory vkBindImageMemory;
901  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
902  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
903  PFN_vkCreateBuffer vkCreateBuffer;
904  PFN_vkDestroyBuffer vkDestroyBuffer;
905  PFN_vkCreateImage vkCreateImage;
906  PFN_vkDestroyImage vkDestroyImage;
907  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
908  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
909 } VmaVulkanFunctions;
910 
912 typedef struct VmaAllocatorCreateInfo
913 {
915  VmaAllocatorCreateFlags flags;
917 
918  VkPhysicalDevice physicalDevice;
920 
921  VkDevice device;
923 
924  VkDeviceSize preferredLargeHeapBlockSize;
926 
927  const VkAllocationCallbacks* pAllocationCallbacks;
929 
930  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
944  uint32_t frameInUseCount;
968  const VkDeviceSize* pHeapSizeLimit;
980  const VmaVulkanFunctions* pVulkanFunctions;
981 } VmaAllocatorCreateInfo;
982 
984 VkResult vmaCreateAllocator(
985  const VmaAllocatorCreateInfo* pCreateInfo,
986  VmaAllocator* pAllocator);
987 
989 void vmaDestroyAllocator(
990  VmaAllocator allocator);
991 
996 void vmaGetPhysicalDeviceProperties(
997  VmaAllocator allocator,
998  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
999 
1004 void vmaGetMemoryProperties(
1005  VmaAllocator allocator,
1006  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1007 
1014 void vmaGetMemoryTypeProperties(
1015  VmaAllocator allocator,
1016  uint32_t memoryTypeIndex,
1017  VkMemoryPropertyFlags* pFlags);
1018 
1027 void vmaSetCurrentFrameIndex(
1028  VmaAllocator allocator,
1029  uint32_t frameIndex);
1030 
1033 typedef struct VmaStatInfo
1034 {
1036  uint32_t blockCount;
1038  uint32_t allocationCount;
1040  uint32_t unusedRangeCount;
1042  VkDeviceSize usedBytes;
1044  VkDeviceSize unusedBytes;
1045  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1046  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1047 } VmaStatInfo;
1048 
1050 typedef struct VmaStats
1051 {
1052  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1053  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1054  VmaStatInfo total;
1055 } VmaStats;
1056 
1058 void vmaCalculateStats(
1059  VmaAllocator allocator,
1060  VmaStats* pStats);
1061 
1062 #define VMA_STATS_STRING_ENABLED 1
1063 
1064 #if VMA_STATS_STRING_ENABLED
1065 
1067 
1069 void vmaBuildStatsString(
1070  VmaAllocator allocator,
1071  char** ppStatsString,
1072  VkBool32 detailedMap);
1073 
1074 void vmaFreeStatsString(
1075  VmaAllocator allocator,
1076  char* pStatsString);
1077 
1078 #endif // #if VMA_STATS_STRING_ENABLED
1079 
1080 VK_DEFINE_HANDLE(VmaPool)
1081 
1082 typedef enum VmaMemoryUsage
1083 {
1087  VMA_MEMORY_USAGE_UNKNOWN = 0,
1104  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1114  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1121  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1130  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1131  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1132 } VmaMemoryUsage;
1133 
1135 typedef enum VmaAllocationCreateFlagBits {
1147  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1148 
1158  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1171  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1184  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1191  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1197  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1198 
1199  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1200 } VmaAllocationCreateFlagBits;
1201 typedef VkFlags VmaAllocationCreateFlags;
1202 
1203 typedef struct VmaAllocationCreateInfo
1204 {
1206  VmaAllocationCreateFlags flags;
1212  VmaMemoryUsage usage;
1217  VkMemoryPropertyFlags requiredFlags;
1222  VkMemoryPropertyFlags preferredFlags;
1230  uint32_t memoryTypeBits;
1236  VmaPool pool;
1243  void* pUserData;
1244 } VmaAllocationCreateInfo;
1245 
1260 VkResult vmaFindMemoryTypeIndex(
1261  VmaAllocator allocator,
1262  uint32_t memoryTypeBits,
1263  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1264  uint32_t* pMemoryTypeIndex);
1265 
1267 typedef enum VmaPoolCreateFlagBits {
1285  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1286 
1287  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1288 } VmaPoolCreateFlagBits;
1289 typedef VkFlags VmaPoolCreateFlags;
1290 
1293 typedef struct VmaPoolCreateInfo {
1296  uint32_t memoryTypeIndex;
1299  VmaPoolCreateFlags flags;
1304  VkDeviceSize blockSize;
1309  size_t minBlockCount;
1317  size_t maxBlockCount;
1331  uint32_t frameInUseCount;
1332 } VmaPoolCreateInfo;
1333 
1336 typedef struct VmaPoolStats {
1339  VkDeviceSize size;
1342  VkDeviceSize unusedSize;
1345  size_t allocationCount;
1348  size_t unusedRangeCount;
1355  VkDeviceSize unusedRangeSizeMax;
1356 } VmaPoolStats;
1357 
1364 VkResult vmaCreatePool(
1365  VmaAllocator allocator,
1366  const VmaPoolCreateInfo* pCreateInfo,
1367  VmaPool* pPool);
1368 
1371 void vmaDestroyPool(
1372  VmaAllocator allocator,
1373  VmaPool pool);
1374 
1381 void vmaGetPoolStats(
1382  VmaAllocator allocator,
1383  VmaPool pool,
1384  VmaPoolStats* pPoolStats);
1385 
1392 void vmaMakePoolAllocationsLost(
1393  VmaAllocator allocator,
1394  VmaPool pool,
1395  size_t* pLostAllocationCount);
1396 
1397 VK_DEFINE_HANDLE(VmaAllocation)
1398 
1399 
1401 typedef struct VmaAllocationInfo {
1406  uint32_t memoryType;
1415  VkDeviceMemory deviceMemory;
1420  VkDeviceSize offset;
1425  VkDeviceSize size;
1434  void* pMappedData;
1439  void* pUserData;
1440 } VmaAllocationInfo;
1441 
1452 VkResult vmaAllocateMemory(
1453  VmaAllocator allocator,
1454  const VkMemoryRequirements* pVkMemoryRequirements,
1455  const VmaAllocationCreateInfo* pCreateInfo,
1456  VmaAllocation* pAllocation,
1457  VmaAllocationInfo* pAllocationInfo);
1458 
1465 VkResult vmaAllocateMemoryForBuffer(
1466  VmaAllocator allocator,
1467  VkBuffer buffer,
1468  const VmaAllocationCreateInfo* pCreateInfo,
1469  VmaAllocation* pAllocation,
1470  VmaAllocationInfo* pAllocationInfo);
1471 
1473 VkResult vmaAllocateMemoryForImage(
1474  VmaAllocator allocator,
1475  VkImage image,
1476  const VmaAllocationCreateInfo* pCreateInfo,
1477  VmaAllocation* pAllocation,
1478  VmaAllocationInfo* pAllocationInfo);
1479 
1481 void vmaFreeMemory(
1482  VmaAllocator allocator,
1483  VmaAllocation allocation);
1484 
1486 void vmaGetAllocationInfo(
1487  VmaAllocator allocator,
1488  VmaAllocation allocation,
1489  VmaAllocationInfo* pAllocationInfo);
1490 
1504 void vmaSetAllocationUserData(
1505  VmaAllocator allocator,
1506  VmaAllocation allocation,
1507  void* pUserData);
1508 
1519 void vmaCreateLostAllocation(
1520  VmaAllocator allocator,
1521  VmaAllocation* pAllocation);
1522 
1557 VkResult vmaMapMemory(
1558  VmaAllocator allocator,
1559  VmaAllocation allocation,
1560  void** ppData);
1561 
1566 void vmaUnmapMemory(
1567  VmaAllocator allocator,
1568  VmaAllocation allocation);
1569 
1571 typedef struct VmaDefragmentationInfo {
1576  VkDeviceSize maxBytesToMove;
1581  uint32_t maxAllocationsToMove;
1582 } VmaDefragmentationInfo;
1583 
1585 typedef struct VmaDefragmentationStats {
1587  VkDeviceSize bytesMoved;
1589  VkDeviceSize bytesFreed;
1591  uint32_t allocationsMoved;
1593  uint32_t deviceMemoryBlocksFreed;
1594 } VmaDefragmentationStats;
1595 
1678 VkResult vmaDefragment(
1679  VmaAllocator allocator,
1680  VmaAllocation* pAllocations,
1681  size_t allocationCount,
1682  VkBool32* pAllocationsChanged,
1683  const VmaDefragmentationInfo *pDefragmentationInfo,
1684  VmaDefragmentationStats* pDefragmentationStats);
1685 
1712 VkResult vmaCreateBuffer(
1713  VmaAllocator allocator,
1714  const VkBufferCreateInfo* pBufferCreateInfo,
1715  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1716  VkBuffer* pBuffer,
1717  VmaAllocation* pAllocation,
1718  VmaAllocationInfo* pAllocationInfo);
1719 
1731 void vmaDestroyBuffer(
1732  VmaAllocator allocator,
1733  VkBuffer buffer,
1734  VmaAllocation allocation);
1735 
1737 VkResult vmaCreateImage(
1738  VmaAllocator allocator,
1739  const VkImageCreateInfo* pImageCreateInfo,
1740  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1741  VkImage* pImage,
1742  VmaAllocation* pAllocation,
1743  VmaAllocationInfo* pAllocationInfo);
1744 
1756 void vmaDestroyImage(
1757  VmaAllocator allocator,
1758  VkImage image,
1759  VmaAllocation allocation);
1760 
1761 #ifdef __cplusplus
1762 }
1763 #endif
1764 
1765 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1766 
1767 // For Visual Studio IntelliSense.
1768 #ifdef __INTELLISENSE__
1769 #define VMA_IMPLEMENTATION
1770 #endif
1771 
1772 #ifdef VMA_IMPLEMENTATION
1773 #undef VMA_IMPLEMENTATION
1774 
1775 #include <cstdint>
1776 #include <cstdlib>
1777 #include <cstring>
1778 
1779 /*******************************************************************************
1780 CONFIGURATION SECTION
1781 
1782 Define some of these macros before each #include of this header or change them
1783 here if you need other then default behavior depending on your environment.
1784 */
1785 
1786 /*
1787 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1788 internally, like:
1789 
1790  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1791 
1792 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1793 VmaAllocatorCreateInfo::pVulkanFunctions.
1794 */
1795 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1796 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1797 #endif
1798 
1799 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1800 //#define VMA_USE_STL_CONTAINERS 1
1801 
1802 /* Set this macro to 1 to make the library including and using STL containers:
1803 std::pair, std::vector, std::list, std::unordered_map.
1804 
1805 Set it to 0 or undefined to make the library using its own implementation of
1806 the containers.
1807 */
1808 #if VMA_USE_STL_CONTAINERS
1809  #define VMA_USE_STL_VECTOR 1
1810  #define VMA_USE_STL_UNORDERED_MAP 1
1811  #define VMA_USE_STL_LIST 1
1812 #endif
1813 
1814 #if VMA_USE_STL_VECTOR
1815  #include <vector>
1816 #endif
1817 
1818 #if VMA_USE_STL_UNORDERED_MAP
1819  #include <unordered_map>
1820 #endif
1821 
1822 #if VMA_USE_STL_LIST
1823  #include <list>
1824 #endif
1825 
1826 /*
1827 Following headers are used in this CONFIGURATION section only, so feel free to
1828 remove them if not needed.
1829 */
1830 #include <cassert> // for assert
1831 #include <algorithm> // for min, max
1832 #include <mutex> // for std::mutex
1833 #include <atomic> // for std::atomic
1834 
1835 #if !defined(_WIN32) && !defined(__APPLE__)
1836  #include <malloc.h> // for aligned_alloc()
1837 #endif
1838 
1839 #if defined(__APPLE__)
1840 #include <cstdlib>
1841 void *aligned_alloc(size_t alignment, size_t size)
1842 {
1843  // alignment must be >= sizeof(void*)
1844  if(alignment < sizeof(void*))
1845  {
1846  alignment = sizeof(void*);
1847  }
1848 
1849  void *pointer;
1850  if(posix_memalign(&pointer, alignment, size) == 0)
1851  return pointer;
1852  return VMA_NULL;
1853 }
1854 #endif
1855 
1856 // Normal assert to check for programmer's errors, especially in Debug configuration.
1857 #ifndef VMA_ASSERT
1858  #ifdef _DEBUG
1859  #define VMA_ASSERT(expr) assert(expr)
1860  #else
1861  #define VMA_ASSERT(expr)
1862  #endif
1863 #endif
1864 
1865 // Assert that will be called very often, like inside data structures e.g. operator[].
1866 // Making it non-empty can make program slow.
1867 #ifndef VMA_HEAVY_ASSERT
1868  #ifdef _DEBUG
1869  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1870  #else
1871  #define VMA_HEAVY_ASSERT(expr)
1872  #endif
1873 #endif
1874 
1875 #ifndef VMA_NULL
1876  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1877  #define VMA_NULL nullptr
1878 #endif
1879 
1880 #ifndef VMA_ALIGN_OF
1881  #define VMA_ALIGN_OF(type) (__alignof(type))
1882 #endif
1883 
1884 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1885  #if defined(_WIN32)
1886  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1887  #else
1888  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1889  #endif
1890 #endif
1891 
1892 #ifndef VMA_SYSTEM_FREE
1893  #if defined(_WIN32)
1894  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1895  #else
1896  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1897  #endif
1898 #endif
1899 
1900 #ifndef VMA_MIN
1901  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1902 #endif
1903 
1904 #ifndef VMA_MAX
1905  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1906 #endif
1907 
1908 #ifndef VMA_SWAP
1909  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1910 #endif
1911 
1912 #ifndef VMA_SORT
1913  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1914 #endif
1915 
1916 #ifndef VMA_DEBUG_LOG
1917  #define VMA_DEBUG_LOG(format, ...)
1918  /*
1919  #define VMA_DEBUG_LOG(format, ...) do { \
1920  printf(format, __VA_ARGS__); \
1921  printf("\n"); \
1922  } while(false)
1923  */
1924 #endif
1925 
1926 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1927 #if VMA_STATS_STRING_ENABLED
1928  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1929  {
1930  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1931  }
1932  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1933  {
1934  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1935  }
1936  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1937  {
1938  snprintf(outStr, strLen, "%p", ptr);
1939  }
1940 #endif
1941 
1942 #ifndef VMA_MUTEX
1943  class VmaMutex
1944  {
1945  public:
1946  VmaMutex() { }
1947  ~VmaMutex() { }
1948  void Lock() { m_Mutex.lock(); }
1949  void Unlock() { m_Mutex.unlock(); }
1950  private:
1951  std::mutex m_Mutex;
1952  };
1953  #define VMA_MUTEX VmaMutex
1954 #endif
1955 
1956 /*
1957 If providing your own implementation, you need to implement a subset of std::atomic:
1958 
1959 - Constructor(uint32_t desired)
1960 - uint32_t load() const
1961 - void store(uint32_t desired)
1962 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1963 */
1964 #ifndef VMA_ATOMIC_UINT32
1965  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1966 #endif
1967 
1968 #ifndef VMA_BEST_FIT
1969 
1981  #define VMA_BEST_FIT (1)
1982 #endif
1983 
1984 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
1985 
1989  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
1990 #endif
1991 
1992 #ifndef VMA_DEBUG_ALIGNMENT
1993 
1997  #define VMA_DEBUG_ALIGNMENT (1)
1998 #endif
1999 
2000 #ifndef VMA_DEBUG_MARGIN
2001 
2005  #define VMA_DEBUG_MARGIN (0)
2006 #endif
2007 
2008 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2009 
2013  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2014 #endif
2015 
2016 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2017 
2021  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2022 #endif
2023 
2024 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2025  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2027 #endif
2028 
2029 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2030  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2032 #endif
2033 
2034 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2035 
2036 /*******************************************************************************
2037 END OF CONFIGURATION
2038 */
2039 
2040 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2041  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2042 
2043 // Returns number of bits set to 1 in (v).
2044 static inline uint32_t VmaCountBitsSet(uint32_t v)
2045 {
2046  uint32_t c = v - ((v >> 1) & 0x55555555);
2047  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2048  c = ((c >> 4) + c) & 0x0F0F0F0F;
2049  c = ((c >> 8) + c) & 0x00FF00FF;
2050  c = ((c >> 16) + c) & 0x0000FFFF;
2051  return c;
2052 }
2053 
2054 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2055 // Use types like uint32_t, uint64_t as T.
2056 template <typename T>
2057 static inline T VmaAlignUp(T val, T align)
2058 {
2059  return (val + align - 1) / align * align;
2060 }
2061 
2062 // Division with mathematical rounding to nearest number.
2063 template <typename T>
2064 inline T VmaRoundDiv(T x, T y)
2065 {
2066  return (x + (y / (T)2)) / y;
2067 }
2068 
2069 #ifndef VMA_SORT
2070 
2071 template<typename Iterator, typename Compare>
2072 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2073 {
2074  Iterator centerValue = end; --centerValue;
2075  Iterator insertIndex = beg;
2076  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2077  {
2078  if(cmp(*memTypeIndex, *centerValue))
2079  {
2080  if(insertIndex != memTypeIndex)
2081  {
2082  VMA_SWAP(*memTypeIndex, *insertIndex);
2083  }
2084  ++insertIndex;
2085  }
2086  }
2087  if(insertIndex != centerValue)
2088  {
2089  VMA_SWAP(*insertIndex, *centerValue);
2090  }
2091  return insertIndex;
2092 }
2093 
2094 template<typename Iterator, typename Compare>
2095 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2096 {
2097  if(beg < end)
2098  {
2099  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2100  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2101  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2102  }
2103 }
2104 
2105 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2106 
2107 #endif // #ifndef VMA_SORT
2108 
2109 /*
2110 Returns true if two memory blocks occupy overlapping pages.
2111 ResourceA must be in less memory offset than ResourceB.
2112 
2113 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2114 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2115 */
2116 static inline bool VmaBlocksOnSamePage(
2117  VkDeviceSize resourceAOffset,
2118  VkDeviceSize resourceASize,
2119  VkDeviceSize resourceBOffset,
2120  VkDeviceSize pageSize)
2121 {
2122  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2123  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2124  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2125  VkDeviceSize resourceBStart = resourceBOffset;
2126  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2127  return resourceAEndPage == resourceBStartPage;
2128 }
2129 
2130 enum VmaSuballocationType
2131 {
2132  VMA_SUBALLOCATION_TYPE_FREE = 0,
2133  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2134  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2135  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2136  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2137  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2138  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2139 };
2140 
2141 /*
2142 Returns true if given suballocation types could conflict and must respect
2143 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2144 or linear image and another one is optimal image. If type is unknown, behave
2145 conservatively.
2146 */
2147 static inline bool VmaIsBufferImageGranularityConflict(
2148  VmaSuballocationType suballocType1,
2149  VmaSuballocationType suballocType2)
2150 {
2151  if(suballocType1 > suballocType2)
2152  {
2153  VMA_SWAP(suballocType1, suballocType2);
2154  }
2155 
2156  switch(suballocType1)
2157  {
2158  case VMA_SUBALLOCATION_TYPE_FREE:
2159  return false;
2160  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2161  return true;
2162  case VMA_SUBALLOCATION_TYPE_BUFFER:
2163  return
2164  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2165  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2166  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2167  return
2168  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2169  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2170  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2171  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2172  return
2173  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2174  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2175  return false;
2176  default:
2177  VMA_ASSERT(0);
2178  return true;
2179  }
2180 }
2181 
2182 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2183 struct VmaMutexLock
2184 {
2185 public:
2186  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2187  m_pMutex(useMutex ? &mutex : VMA_NULL)
2188  {
2189  if(m_pMutex)
2190  {
2191  m_pMutex->Lock();
2192  }
2193  }
2194 
2195  ~VmaMutexLock()
2196  {
2197  if(m_pMutex)
2198  {
2199  m_pMutex->Unlock();
2200  }
2201  }
2202 
2203 private:
2204  VMA_MUTEX* m_pMutex;
2205 };
2206 
2207 #if VMA_DEBUG_GLOBAL_MUTEX
2208  static VMA_MUTEX gDebugGlobalMutex;
2209  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2210 #else
2211  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2212 #endif
2213 
2214 // Minimum size of a free suballocation to register it in the free suballocation collection.
2215 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2216 
2217 /*
2218 Performs binary search and returns iterator to first element that is greater or
2219 equal to (key), according to comparison (cmp).
2220 
2221 Cmp should return true if first argument is less than second argument.
2222 
2223 Returned value is the found element, if present in the collection or place where
2224 new element with value (key) should be inserted.
2225 */
2226 template <typename IterT, typename KeyT, typename CmpT>
2227 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2228 {
2229  size_t down = 0, up = (end - beg);
2230  while(down < up)
2231  {
2232  const size_t mid = (down + up) / 2;
2233  if(cmp(*(beg+mid), key))
2234  {
2235  down = mid + 1;
2236  }
2237  else
2238  {
2239  up = mid;
2240  }
2241  }
2242  return beg + down;
2243 }
2244 
2246 // Memory allocation
2247 
2248 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2249 {
2250  if((pAllocationCallbacks != VMA_NULL) &&
2251  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2252  {
2253  return (*pAllocationCallbacks->pfnAllocation)(
2254  pAllocationCallbacks->pUserData,
2255  size,
2256  alignment,
2257  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2258  }
2259  else
2260  {
2261  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2262  }
2263 }
2264 
2265 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2266 {
2267  if((pAllocationCallbacks != VMA_NULL) &&
2268  (pAllocationCallbacks->pfnFree != VMA_NULL))
2269  {
2270  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2271  }
2272  else
2273  {
2274  VMA_SYSTEM_FREE(ptr);
2275  }
2276 }
2277 
2278 template<typename T>
2279 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2280 {
2281  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2282 }
2283 
2284 template<typename T>
2285 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2286 {
2287  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2288 }
2289 
2290 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2291 
2292 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2293 
2294 template<typename T>
2295 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2296 {
2297  ptr->~T();
2298  VmaFree(pAllocationCallbacks, ptr);
2299 }
2300 
2301 template<typename T>
2302 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2303 {
2304  if(ptr != VMA_NULL)
2305  {
2306  for(size_t i = count; i--; )
2307  {
2308  ptr[i].~T();
2309  }
2310  VmaFree(pAllocationCallbacks, ptr);
2311  }
2312 }
2313 
2314 // STL-compatible allocator.
2315 template<typename T>
2316 class VmaStlAllocator
2317 {
2318 public:
2319  const VkAllocationCallbacks* const m_pCallbacks;
2320  typedef T value_type;
2321 
2322  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2323  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2324 
2325  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2326  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2327 
2328  template<typename U>
2329  bool operator==(const VmaStlAllocator<U>& rhs) const
2330  {
2331  return m_pCallbacks == rhs.m_pCallbacks;
2332  }
2333  template<typename U>
2334  bool operator!=(const VmaStlAllocator<U>& rhs) const
2335  {
2336  return m_pCallbacks != rhs.m_pCallbacks;
2337  }
2338 
2339  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2340 };
2341 
2342 #if VMA_USE_STL_VECTOR
2343 
2344 #define VmaVector std::vector
2345 
2346 template<typename T, typename allocatorT>
2347 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2348 {
2349  vec.insert(vec.begin() + index, item);
2350 }
2351 
2352 template<typename T, typename allocatorT>
2353 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2354 {
2355  vec.erase(vec.begin() + index);
2356 }
2357 
2358 #else // #if VMA_USE_STL_VECTOR
2359 
2360 /* Class with interface compatible with subset of std::vector.
2361 T must be POD because constructors and destructors are not called and memcpy is
2362 used for these objects. */
2363 template<typename T, typename AllocatorT>
2364 class VmaVector
2365 {
2366 public:
2367  typedef T value_type;
2368 
2369  VmaVector(const AllocatorT& allocator) :
2370  m_Allocator(allocator),
2371  m_pArray(VMA_NULL),
2372  m_Count(0),
2373  m_Capacity(0)
2374  {
2375  }
2376 
2377  VmaVector(size_t count, const AllocatorT& allocator) :
2378  m_Allocator(allocator),
2379  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2380  m_Count(count),
2381  m_Capacity(count)
2382  {
2383  }
2384 
2385  VmaVector(const VmaVector<T, AllocatorT>& src) :
2386  m_Allocator(src.m_Allocator),
2387  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2388  m_Count(src.m_Count),
2389  m_Capacity(src.m_Count)
2390  {
2391  if(m_Count != 0)
2392  {
2393  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2394  }
2395  }
2396 
2397  ~VmaVector()
2398  {
2399  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2400  }
2401 
2402  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2403  {
2404  if(&rhs != this)
2405  {
2406  resize(rhs.m_Count);
2407  if(m_Count != 0)
2408  {
2409  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2410  }
2411  }
2412  return *this;
2413  }
2414 
2415  bool empty() const { return m_Count == 0; }
2416  size_t size() const { return m_Count; }
2417  T* data() { return m_pArray; }
2418  const T* data() const { return m_pArray; }
2419 
2420  T& operator[](size_t index)
2421  {
2422  VMA_HEAVY_ASSERT(index < m_Count);
2423  return m_pArray[index];
2424  }
2425  const T& operator[](size_t index) const
2426  {
2427  VMA_HEAVY_ASSERT(index < m_Count);
2428  return m_pArray[index];
2429  }
2430 
2431  T& front()
2432  {
2433  VMA_HEAVY_ASSERT(m_Count > 0);
2434  return m_pArray[0];
2435  }
2436  const T& front() const
2437  {
2438  VMA_HEAVY_ASSERT(m_Count > 0);
2439  return m_pArray[0];
2440  }
2441  T& back()
2442  {
2443  VMA_HEAVY_ASSERT(m_Count > 0);
2444  return m_pArray[m_Count - 1];
2445  }
2446  const T& back() const
2447  {
2448  VMA_HEAVY_ASSERT(m_Count > 0);
2449  return m_pArray[m_Count - 1];
2450  }
2451 
2452  void reserve(size_t newCapacity, bool freeMemory = false)
2453  {
2454  newCapacity = VMA_MAX(newCapacity, m_Count);
2455 
2456  if((newCapacity < m_Capacity) && !freeMemory)
2457  {
2458  newCapacity = m_Capacity;
2459  }
2460 
2461  if(newCapacity != m_Capacity)
2462  {
2463  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2464  if(m_Count != 0)
2465  {
2466  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2467  }
2468  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2469  m_Capacity = newCapacity;
2470  m_pArray = newArray;
2471  }
2472  }
2473 
2474  void resize(size_t newCount, bool freeMemory = false)
2475  {
2476  size_t newCapacity = m_Capacity;
2477  if(newCount > m_Capacity)
2478  {
2479  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2480  }
2481  else if(freeMemory)
2482  {
2483  newCapacity = newCount;
2484  }
2485 
2486  if(newCapacity != m_Capacity)
2487  {
2488  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2489  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2490  if(elementsToCopy != 0)
2491  {
2492  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2493  }
2494  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2495  m_Capacity = newCapacity;
2496  m_pArray = newArray;
2497  }
2498 
2499  m_Count = newCount;
2500  }
2501 
2502  void clear(bool freeMemory = false)
2503  {
2504  resize(0, freeMemory);
2505  }
2506 
2507  void insert(size_t index, const T& src)
2508  {
2509  VMA_HEAVY_ASSERT(index <= m_Count);
2510  const size_t oldCount = size();
2511  resize(oldCount + 1);
2512  if(index < oldCount)
2513  {
2514  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2515  }
2516  m_pArray[index] = src;
2517  }
2518 
2519  void remove(size_t index)
2520  {
2521  VMA_HEAVY_ASSERT(index < m_Count);
2522  const size_t oldCount = size();
2523  if(index < oldCount - 1)
2524  {
2525  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2526  }
2527  resize(oldCount - 1);
2528  }
2529 
2530  void push_back(const T& src)
2531  {
2532  const size_t newIndex = size();
2533  resize(newIndex + 1);
2534  m_pArray[newIndex] = src;
2535  }
2536 
2537  void pop_back()
2538  {
2539  VMA_HEAVY_ASSERT(m_Count > 0);
2540  resize(size() - 1);
2541  }
2542 
2543  void push_front(const T& src)
2544  {
2545  insert(0, src);
2546  }
2547 
2548  void pop_front()
2549  {
2550  VMA_HEAVY_ASSERT(m_Count > 0);
2551  remove(0);
2552  }
2553 
2554  typedef T* iterator;
2555 
2556  iterator begin() { return m_pArray; }
2557  iterator end() { return m_pArray + m_Count; }
2558 
2559 private:
2560  AllocatorT m_Allocator;
2561  T* m_pArray;
2562  size_t m_Count;
2563  size_t m_Capacity;
2564 };
2565 
2566 template<typename T, typename allocatorT>
2567 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2568 {
2569  vec.insert(index, item);
2570 }
2571 
2572 template<typename T, typename allocatorT>
2573 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2574 {
2575  vec.remove(index);
2576 }
2577 
2578 #endif // #if VMA_USE_STL_VECTOR
2579 
2580 template<typename CmpLess, typename VectorT>
2581 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2582 {
2583  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2584  vector.data(),
2585  vector.data() + vector.size(),
2586  value,
2587  CmpLess()) - vector.data();
2588  VmaVectorInsert(vector, indexToInsert, value);
2589  return indexToInsert;
2590 }
2591 
2592 template<typename CmpLess, typename VectorT>
2593 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2594 {
2595  CmpLess comparator;
2596  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2597  vector.begin(),
2598  vector.end(),
2599  value,
2600  comparator);
2601  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2602  {
2603  size_t indexToRemove = it - vector.begin();
2604  VmaVectorRemove(vector, indexToRemove);
2605  return true;
2606  }
2607  return false;
2608 }
2609 
2610 template<typename CmpLess, typename VectorT>
2611 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2612 {
2613  CmpLess comparator;
2614  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2615  vector.data(),
2616  vector.data() + vector.size(),
2617  value,
2618  comparator);
2619  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2620  {
2621  return it - vector.begin();
2622  }
2623  else
2624  {
2625  return vector.size();
2626  }
2627 }
2628 
2630 // class VmaPoolAllocator
2631 
2632 /*
2633 Allocator for objects of type T using a list of arrays (pools) to speed up
2634 allocation. Number of elements that can be allocated is not bounded because
2635 allocator can create multiple blocks.
2636 */
2637 template<typename T>
2638 class VmaPoolAllocator
2639 {
2640 public:
2641  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2642  ~VmaPoolAllocator();
2643  void Clear();
2644  T* Alloc();
2645  void Free(T* ptr);
2646 
2647 private:
2648  union Item
2649  {
2650  uint32_t NextFreeIndex;
2651  T Value;
2652  };
2653 
2654  struct ItemBlock
2655  {
2656  Item* pItems;
2657  uint32_t FirstFreeIndex;
2658  };
2659 
2660  const VkAllocationCallbacks* m_pAllocationCallbacks;
2661  size_t m_ItemsPerBlock;
2662  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2663 
2664  ItemBlock& CreateNewBlock();
2665 };
2666 
2667 template<typename T>
2668 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2669  m_pAllocationCallbacks(pAllocationCallbacks),
2670  m_ItemsPerBlock(itemsPerBlock),
2671  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2672 {
2673  VMA_ASSERT(itemsPerBlock > 0);
2674 }
2675 
2676 template<typename T>
2677 VmaPoolAllocator<T>::~VmaPoolAllocator()
2678 {
2679  Clear();
2680 }
2681 
2682 template<typename T>
2683 void VmaPoolAllocator<T>::Clear()
2684 {
2685  for(size_t i = m_ItemBlocks.size(); i--; )
2686  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2687  m_ItemBlocks.clear();
2688 }
2689 
2690 template<typename T>
2691 T* VmaPoolAllocator<T>::Alloc()
2692 {
2693  for(size_t i = m_ItemBlocks.size(); i--; )
2694  {
2695  ItemBlock& block = m_ItemBlocks[i];
2696  // This block has some free items: Use first one.
2697  if(block.FirstFreeIndex != UINT32_MAX)
2698  {
2699  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2700  block.FirstFreeIndex = pItem->NextFreeIndex;
2701  return &pItem->Value;
2702  }
2703  }
2704 
2705  // No block has free item: Create new one and use it.
2706  ItemBlock& newBlock = CreateNewBlock();
2707  Item* const pItem = &newBlock.pItems[0];
2708  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2709  return &pItem->Value;
2710 }
2711 
2712 template<typename T>
2713 void VmaPoolAllocator<T>::Free(T* ptr)
2714 {
2715  // Search all memory blocks to find ptr.
2716  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2717  {
2718  ItemBlock& block = m_ItemBlocks[i];
2719 
2720  // Casting to union.
2721  Item* pItemPtr;
2722  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2723 
2724  // Check if pItemPtr is in address range of this block.
2725  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2726  {
2727  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2728  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2729  block.FirstFreeIndex = index;
2730  return;
2731  }
2732  }
2733  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2734 }
2735 
2736 template<typename T>
2737 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2738 {
2739  ItemBlock newBlock = {
2740  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2741 
2742  m_ItemBlocks.push_back(newBlock);
2743 
2744  // Setup singly-linked list of all free items in this block.
2745  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2746  newBlock.pItems[i].NextFreeIndex = i + 1;
2747  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2748  return m_ItemBlocks.back();
2749 }
2750 
2752 // class VmaRawList, VmaList
2753 
2754 #if VMA_USE_STL_LIST
2755 
2756 #define VmaList std::list
2757 
2758 #else // #if VMA_USE_STL_LIST
2759 
2760 template<typename T>
2761 struct VmaListItem
2762 {
2763  VmaListItem* pPrev;
2764  VmaListItem* pNext;
2765  T Value;
2766 };
2767 
2768 // Doubly linked list.
2769 template<typename T>
2770 class VmaRawList
2771 {
2772 public:
2773  typedef VmaListItem<T> ItemType;
2774 
2775  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2776  ~VmaRawList();
2777  void Clear();
2778 
2779  size_t GetCount() const { return m_Count; }
2780  bool IsEmpty() const { return m_Count == 0; }
2781 
2782  ItemType* Front() { return m_pFront; }
2783  const ItemType* Front() const { return m_pFront; }
2784  ItemType* Back() { return m_pBack; }
2785  const ItemType* Back() const { return m_pBack; }
2786 
2787  ItemType* PushBack();
2788  ItemType* PushFront();
2789  ItemType* PushBack(const T& value);
2790  ItemType* PushFront(const T& value);
2791  void PopBack();
2792  void PopFront();
2793 
2794  // Item can be null - it means PushBack.
2795  ItemType* InsertBefore(ItemType* pItem);
2796  // Item can be null - it means PushFront.
2797  ItemType* InsertAfter(ItemType* pItem);
2798 
2799  ItemType* InsertBefore(ItemType* pItem, const T& value);
2800  ItemType* InsertAfter(ItemType* pItem, const T& value);
2801 
2802  void Remove(ItemType* pItem);
2803 
2804 private:
2805  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2806  VmaPoolAllocator<ItemType> m_ItemAllocator;
2807  ItemType* m_pFront;
2808  ItemType* m_pBack;
2809  size_t m_Count;
2810 
2811  // Declared not defined, to block copy constructor and assignment operator.
2812  VmaRawList(const VmaRawList<T>& src);
2813  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2814 };
2815 
2816 template<typename T>
2817 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2818  m_pAllocationCallbacks(pAllocationCallbacks),
2819  m_ItemAllocator(pAllocationCallbacks, 128),
2820  m_pFront(VMA_NULL),
2821  m_pBack(VMA_NULL),
2822  m_Count(0)
2823 {
2824 }
2825 
2826 template<typename T>
2827 VmaRawList<T>::~VmaRawList()
2828 {
2829  // Intentionally not calling Clear, because that would be unnecessary
2830  // computations to return all items to m_ItemAllocator as free.
2831 }
2832 
2833 template<typename T>
2834 void VmaRawList<T>::Clear()
2835 {
2836  if(IsEmpty() == false)
2837  {
2838  ItemType* pItem = m_pBack;
2839  while(pItem != VMA_NULL)
2840  {
2841  ItemType* const pPrevItem = pItem->pPrev;
2842  m_ItemAllocator.Free(pItem);
2843  pItem = pPrevItem;
2844  }
2845  m_pFront = VMA_NULL;
2846  m_pBack = VMA_NULL;
2847  m_Count = 0;
2848  }
2849 }
2850 
2851 template<typename T>
2852 VmaListItem<T>* VmaRawList<T>::PushBack()
2853 {
2854  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2855  pNewItem->pNext = VMA_NULL;
2856  if(IsEmpty())
2857  {
2858  pNewItem->pPrev = VMA_NULL;
2859  m_pFront = pNewItem;
2860  m_pBack = pNewItem;
2861  m_Count = 1;
2862  }
2863  else
2864  {
2865  pNewItem->pPrev = m_pBack;
2866  m_pBack->pNext = pNewItem;
2867  m_pBack = pNewItem;
2868  ++m_Count;
2869  }
2870  return pNewItem;
2871 }
2872 
2873 template<typename T>
2874 VmaListItem<T>* VmaRawList<T>::PushFront()
2875 {
2876  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2877  pNewItem->pPrev = VMA_NULL;
2878  if(IsEmpty())
2879  {
2880  pNewItem->pNext = VMA_NULL;
2881  m_pFront = pNewItem;
2882  m_pBack = pNewItem;
2883  m_Count = 1;
2884  }
2885  else
2886  {
2887  pNewItem->pNext = m_pFront;
2888  m_pFront->pPrev = pNewItem;
2889  m_pFront = pNewItem;
2890  ++m_Count;
2891  }
2892  return pNewItem;
2893 }
2894 
2895 template<typename T>
2896 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2897 {
2898  ItemType* const pNewItem = PushBack();
2899  pNewItem->Value = value;
2900  return pNewItem;
2901 }
2902 
2903 template<typename T>
2904 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2905 {
2906  ItemType* const pNewItem = PushFront();
2907  pNewItem->Value = value;
2908  return pNewItem;
2909 }
2910 
2911 template<typename T>
2912 void VmaRawList<T>::PopBack()
2913 {
2914  VMA_HEAVY_ASSERT(m_Count > 0);
2915  ItemType* const pBackItem = m_pBack;
2916  ItemType* const pPrevItem = pBackItem->pPrev;
2917  if(pPrevItem != VMA_NULL)
2918  {
2919  pPrevItem->pNext = VMA_NULL;
2920  }
2921  m_pBack = pPrevItem;
2922  m_ItemAllocator.Free(pBackItem);
2923  --m_Count;
2924 }
2925 
2926 template<typename T>
2927 void VmaRawList<T>::PopFront()
2928 {
2929  VMA_HEAVY_ASSERT(m_Count > 0);
2930  ItemType* const pFrontItem = m_pFront;
2931  ItemType* const pNextItem = pFrontItem->pNext;
2932  if(pNextItem != VMA_NULL)
2933  {
2934  pNextItem->pPrev = VMA_NULL;
2935  }
2936  m_pFront = pNextItem;
2937  m_ItemAllocator.Free(pFrontItem);
2938  --m_Count;
2939 }
2940 
2941 template<typename T>
2942 void VmaRawList<T>::Remove(ItemType* pItem)
2943 {
2944  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2945  VMA_HEAVY_ASSERT(m_Count > 0);
2946 
2947  if(pItem->pPrev != VMA_NULL)
2948  {
2949  pItem->pPrev->pNext = pItem->pNext;
2950  }
2951  else
2952  {
2953  VMA_HEAVY_ASSERT(m_pFront == pItem);
2954  m_pFront = pItem->pNext;
2955  }
2956 
2957  if(pItem->pNext != VMA_NULL)
2958  {
2959  pItem->pNext->pPrev = pItem->pPrev;
2960  }
2961  else
2962  {
2963  VMA_HEAVY_ASSERT(m_pBack == pItem);
2964  m_pBack = pItem->pPrev;
2965  }
2966 
2967  m_ItemAllocator.Free(pItem);
2968  --m_Count;
2969 }
2970 
2971 template<typename T>
2972 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2973 {
2974  if(pItem != VMA_NULL)
2975  {
2976  ItemType* const prevItem = pItem->pPrev;
2977  ItemType* const newItem = m_ItemAllocator.Alloc();
2978  newItem->pPrev = prevItem;
2979  newItem->pNext = pItem;
2980  pItem->pPrev = newItem;
2981  if(prevItem != VMA_NULL)
2982  {
2983  prevItem->pNext = newItem;
2984  }
2985  else
2986  {
2987  VMA_HEAVY_ASSERT(m_pFront == pItem);
2988  m_pFront = newItem;
2989  }
2990  ++m_Count;
2991  return newItem;
2992  }
2993  else
2994  return PushBack();
2995 }
2996 
2997 template<typename T>
2998 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2999 {
3000  if(pItem != VMA_NULL)
3001  {
3002  ItemType* const nextItem = pItem->pNext;
3003  ItemType* const newItem = m_ItemAllocator.Alloc();
3004  newItem->pNext = nextItem;
3005  newItem->pPrev = pItem;
3006  pItem->pNext = newItem;
3007  if(nextItem != VMA_NULL)
3008  {
3009  nextItem->pPrev = newItem;
3010  }
3011  else
3012  {
3013  VMA_HEAVY_ASSERT(m_pBack == pItem);
3014  m_pBack = newItem;
3015  }
3016  ++m_Count;
3017  return newItem;
3018  }
3019  else
3020  return PushFront();
3021 }
3022 
3023 template<typename T>
3024 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3025 {
3026  ItemType* const newItem = InsertBefore(pItem);
3027  newItem->Value = value;
3028  return newItem;
3029 }
3030 
3031 template<typename T>
3032 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3033 {
3034  ItemType* const newItem = InsertAfter(pItem);
3035  newItem->Value = value;
3036  return newItem;
3037 }
3038 
3039 template<typename T, typename AllocatorT>
3040 class VmaList
3041 {
3042 public:
3043  class iterator
3044  {
3045  public:
3046  iterator() :
3047  m_pList(VMA_NULL),
3048  m_pItem(VMA_NULL)
3049  {
3050  }
3051 
3052  T& operator*() const
3053  {
3054  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3055  return m_pItem->Value;
3056  }
3057  T* operator->() const
3058  {
3059  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3060  return &m_pItem->Value;
3061  }
3062 
3063  iterator& operator++()
3064  {
3065  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3066  m_pItem = m_pItem->pNext;
3067  return *this;
3068  }
3069  iterator& operator--()
3070  {
3071  if(m_pItem != VMA_NULL)
3072  {
3073  m_pItem = m_pItem->pPrev;
3074  }
3075  else
3076  {
3077  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3078  m_pItem = m_pList->Back();
3079  }
3080  return *this;
3081  }
3082 
3083  iterator operator++(int)
3084  {
3085  iterator result = *this;
3086  ++*this;
3087  return result;
3088  }
3089  iterator operator--(int)
3090  {
3091  iterator result = *this;
3092  --*this;
3093  return result;
3094  }
3095 
3096  bool operator==(const iterator& rhs) const
3097  {
3098  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3099  return m_pItem == rhs.m_pItem;
3100  }
3101  bool operator!=(const iterator& rhs) const
3102  {
3103  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3104  return m_pItem != rhs.m_pItem;
3105  }
3106 
3107  private:
3108  VmaRawList<T>* m_pList;
3109  VmaListItem<T>* m_pItem;
3110 
3111  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3112  m_pList(pList),
3113  m_pItem(pItem)
3114  {
3115  }
3116 
3117  friend class VmaList<T, AllocatorT>;
3118  };
3119 
3120  class const_iterator
3121  {
3122  public:
3123  const_iterator() :
3124  m_pList(VMA_NULL),
3125  m_pItem(VMA_NULL)
3126  {
3127  }
3128 
3129  const_iterator(const iterator& src) :
3130  m_pList(src.m_pList),
3131  m_pItem(src.m_pItem)
3132  {
3133  }
3134 
3135  const T& operator*() const
3136  {
3137  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3138  return m_pItem->Value;
3139  }
3140  const T* operator->() const
3141  {
3142  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3143  return &m_pItem->Value;
3144  }
3145 
3146  const_iterator& operator++()
3147  {
3148  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3149  m_pItem = m_pItem->pNext;
3150  return *this;
3151  }
3152  const_iterator& operator--()
3153  {
3154  if(m_pItem != VMA_NULL)
3155  {
3156  m_pItem = m_pItem->pPrev;
3157  }
3158  else
3159  {
3160  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3161  m_pItem = m_pList->Back();
3162  }
3163  return *this;
3164  }
3165 
3166  const_iterator operator++(int)
3167  {
3168  const_iterator result = *this;
3169  ++*this;
3170  return result;
3171  }
3172  const_iterator operator--(int)
3173  {
3174  const_iterator result = *this;
3175  --*this;
3176  return result;
3177  }
3178 
3179  bool operator==(const const_iterator& rhs) const
3180  {
3181  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3182  return m_pItem == rhs.m_pItem;
3183  }
3184  bool operator!=(const const_iterator& rhs) const
3185  {
3186  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3187  return m_pItem != rhs.m_pItem;
3188  }
3189 
3190  private:
3191  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3192  m_pList(pList),
3193  m_pItem(pItem)
3194  {
3195  }
3196 
3197  const VmaRawList<T>* m_pList;
3198  const VmaListItem<T>* m_pItem;
3199 
3200  friend class VmaList<T, AllocatorT>;
3201  };
3202 
3203  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3204 
3205  bool empty() const { return m_RawList.IsEmpty(); }
3206  size_t size() const { return m_RawList.GetCount(); }
3207 
3208  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3209  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3210 
3211  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3212  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3213 
3214  void clear() { m_RawList.Clear(); }
3215  void push_back(const T& value) { m_RawList.PushBack(value); }
3216  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3217  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3218 
3219 private:
3220  VmaRawList<T> m_RawList;
3221 };
3222 
3223 #endif // #if VMA_USE_STL_LIST
3224 
3226 // class VmaMap
3227 
3228 // Unused in this version.
3229 #if 0
3230 
3231 #if VMA_USE_STL_UNORDERED_MAP
3232 
3233 #define VmaPair std::pair
3234 
3235 #define VMA_MAP_TYPE(KeyT, ValueT) \
3236  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3237 
3238 #else // #if VMA_USE_STL_UNORDERED_MAP
3239 
3240 template<typename T1, typename T2>
3241 struct VmaPair
3242 {
3243  T1 first;
3244  T2 second;
3245 
3246  VmaPair() : first(), second() { }
3247  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3248 };
3249 
3250 /* Class compatible with subset of interface of std::unordered_map.
3251 KeyT, ValueT must be POD because they will be stored in VmaVector.
3252 */
3253 template<typename KeyT, typename ValueT>
3254 class VmaMap
3255 {
3256 public:
3257  typedef VmaPair<KeyT, ValueT> PairType;
3258  typedef PairType* iterator;
3259 
3260  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3261 
3262  iterator begin() { return m_Vector.begin(); }
3263  iterator end() { return m_Vector.end(); }
3264 
3265  void insert(const PairType& pair);
3266  iterator find(const KeyT& key);
3267  void erase(iterator it);
3268 
3269 private:
3270  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3271 };
3272 
3273 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3274 
3275 template<typename FirstT, typename SecondT>
3276 struct VmaPairFirstLess
3277 {
3278  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3279  {
3280  return lhs.first < rhs.first;
3281  }
3282  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3283  {
3284  return lhs.first < rhsFirst;
3285  }
3286 };
3287 
3288 template<typename KeyT, typename ValueT>
3289 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3290 {
3291  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3292  m_Vector.data(),
3293  m_Vector.data() + m_Vector.size(),
3294  pair,
3295  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3296  VmaVectorInsert(m_Vector, indexToInsert, pair);
3297 }
3298 
3299 template<typename KeyT, typename ValueT>
3300 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3301 {
3302  PairType* it = VmaBinaryFindFirstNotLess(
3303  m_Vector.data(),
3304  m_Vector.data() + m_Vector.size(),
3305  key,
3306  VmaPairFirstLess<KeyT, ValueT>());
3307  if((it != m_Vector.end()) && (it->first == key))
3308  {
3309  return it;
3310  }
3311  else
3312  {
3313  return m_Vector.end();
3314  }
3315 }
3316 
3317 template<typename KeyT, typename ValueT>
3318 void VmaMap<KeyT, ValueT>::erase(iterator it)
3319 {
3320  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3321 }
3322 
3323 #endif // #if VMA_USE_STL_UNORDERED_MAP
3324 
3325 #endif // #if 0
3326 
3328 
3329 class VmaDeviceMemoryBlock;
3330 
3331 struct VmaAllocation_T
3332 {
3333 private:
3334  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3335 
3336  enum FLAGS
3337  {
3338  FLAG_USER_DATA_STRING = 0x01,
3339  };
3340 
3341 public:
3342  enum ALLOCATION_TYPE
3343  {
3344  ALLOCATION_TYPE_NONE,
3345  ALLOCATION_TYPE_BLOCK,
3346  ALLOCATION_TYPE_DEDICATED,
3347  };
3348 
3349  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3350  m_Alignment(1),
3351  m_Size(0),
3352  m_pUserData(VMA_NULL),
3353  m_LastUseFrameIndex(currentFrameIndex),
3354  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3355  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3356  m_MapCount(0),
3357  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3358  {
3359  }
3360 
3361  ~VmaAllocation_T()
3362  {
3363  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3364 
3365  // Check if owned string was freed.
3366  VMA_ASSERT(m_pUserData == VMA_NULL);
3367  }
3368 
3369  void InitBlockAllocation(
3370  VmaPool hPool,
3371  VmaDeviceMemoryBlock* block,
3372  VkDeviceSize offset,
3373  VkDeviceSize alignment,
3374  VkDeviceSize size,
3375  VmaSuballocationType suballocationType,
3376  bool mapped,
3377  bool canBecomeLost)
3378  {
3379  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3380  VMA_ASSERT(block != VMA_NULL);
3381  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3382  m_Alignment = alignment;
3383  m_Size = size;
3384  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3385  m_SuballocationType = (uint8_t)suballocationType;
3386  m_BlockAllocation.m_hPool = hPool;
3387  m_BlockAllocation.m_Block = block;
3388  m_BlockAllocation.m_Offset = offset;
3389  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3390  }
3391 
3392  void InitLost()
3393  {
3394  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3395  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3396  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3397  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3398  m_BlockAllocation.m_Block = VMA_NULL;
3399  m_BlockAllocation.m_Offset = 0;
3400  m_BlockAllocation.m_CanBecomeLost = true;
3401  }
3402 
3403  void ChangeBlockAllocation(
3404  VmaAllocator hAllocator,
3405  VmaDeviceMemoryBlock* block,
3406  VkDeviceSize offset);
3407 
3408  // pMappedData not null means allocation is created with MAPPED flag.
3409  void InitDedicatedAllocation(
3410  uint32_t memoryTypeIndex,
3411  VkDeviceMemory hMemory,
3412  VmaSuballocationType suballocationType,
3413  void* pMappedData,
3414  VkDeviceSize size)
3415  {
3416  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3417  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3418  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3419  m_Alignment = 0;
3420  m_Size = size;
3421  m_SuballocationType = (uint8_t)suballocationType;
3422  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3423  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3424  m_DedicatedAllocation.m_hMemory = hMemory;
3425  m_DedicatedAllocation.m_pMappedData = pMappedData;
3426  }
3427 
3428  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3429  VkDeviceSize GetAlignment() const { return m_Alignment; }
3430  VkDeviceSize GetSize() const { return m_Size; }
3431  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3432  void* GetUserData() const { return m_pUserData; }
3433  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3434  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3435 
3436  VmaDeviceMemoryBlock* GetBlock() const
3437  {
3438  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3439  return m_BlockAllocation.m_Block;
3440  }
3441  VkDeviceSize GetOffset() const;
3442  VkDeviceMemory GetMemory() const;
3443  uint32_t GetMemoryTypeIndex() const;
3444  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3445  void* GetMappedData() const;
3446  bool CanBecomeLost() const;
3447  VmaPool GetPool() const;
3448 
3449  uint32_t GetLastUseFrameIndex() const
3450  {
3451  return m_LastUseFrameIndex.load();
3452  }
3453  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3454  {
3455  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3456  }
3457  /*
3458  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3459  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3460  - Else, returns false.
3461 
3462  If hAllocation is already lost, assert - you should not call it then.
3463  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3464  */
3465  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3466 
3467  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3468  {
3469  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3470  outInfo.blockCount = 1;
3471  outInfo.allocationCount = 1;
3472  outInfo.unusedRangeCount = 0;
3473  outInfo.usedBytes = m_Size;
3474  outInfo.unusedBytes = 0;
3475  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3476  outInfo.unusedRangeSizeMin = UINT64_MAX;
3477  outInfo.unusedRangeSizeMax = 0;
3478  }
3479 
3480  void BlockAllocMap();
3481  void BlockAllocUnmap();
3482  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3483  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3484 
3485 private:
3486  VkDeviceSize m_Alignment;
3487  VkDeviceSize m_Size;
3488  void* m_pUserData;
3489  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3490  uint8_t m_Type; // ALLOCATION_TYPE
3491  uint8_t m_SuballocationType; // VmaSuballocationType
3492  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3493  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3494  uint8_t m_MapCount;
3495  uint8_t m_Flags; // enum FLAGS
3496 
3497  // Allocation out of VmaDeviceMemoryBlock.
3498  struct BlockAllocation
3499  {
3500  VmaPool m_hPool; // Null if belongs to general memory.
3501  VmaDeviceMemoryBlock* m_Block;
3502  VkDeviceSize m_Offset;
3503  bool m_CanBecomeLost;
3504  };
3505 
3506  // Allocation for an object that has its own private VkDeviceMemory.
3507  struct DedicatedAllocation
3508  {
3509  uint32_t m_MemoryTypeIndex;
3510  VkDeviceMemory m_hMemory;
3511  void* m_pMappedData; // Not null means memory is mapped.
3512  };
3513 
3514  union
3515  {
3516  // Allocation out of VmaDeviceMemoryBlock.
3517  BlockAllocation m_BlockAllocation;
3518  // Allocation for an object that has its own private VkDeviceMemory.
3519  DedicatedAllocation m_DedicatedAllocation;
3520  };
3521 
3522  void FreeUserDataString(VmaAllocator hAllocator);
3523 };
3524 
3525 /*
3526 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3527 allocated memory block or free.
3528 */
3529 struct VmaSuballocation
3530 {
3531  VkDeviceSize offset;
3532  VkDeviceSize size;
3533  VmaAllocation hAllocation;
3534  VmaSuballocationType type;
3535 };
3536 
3537 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3538 
3539 // Cost of one additional allocation lost, as equivalent in bytes.
3540 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3541 
3542 /*
3543 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3544 
3545 If canMakeOtherLost was false:
3546 - item points to a FREE suballocation.
3547 - itemsToMakeLostCount is 0.
3548 
3549 If canMakeOtherLost was true:
3550 - item points to first of sequence of suballocations, which are either FREE,
3551  or point to VmaAllocations that can become lost.
3552 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3553  the requested allocation to succeed.
3554 */
3555 struct VmaAllocationRequest
3556 {
3557  VkDeviceSize offset;
3558  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3559  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3560  VmaSuballocationList::iterator item;
3561  size_t itemsToMakeLostCount;
3562 
3563  VkDeviceSize CalcCost() const
3564  {
3565  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3566  }
3567 };
3568 
3569 /*
3570 Data structure used for bookkeeping of allocations and unused ranges of memory
3571 in a single VkDeviceMemory block.
3572 */
3573 class VmaBlockMetadata
3574 {
3575 public:
3576  VmaBlockMetadata(VmaAllocator hAllocator);
3577  ~VmaBlockMetadata();
3578  void Init(VkDeviceSize size);
3579 
3580  // Validates all data structures inside this object. If not valid, returns false.
3581  bool Validate() const;
3582  VkDeviceSize GetSize() const { return m_Size; }
3583  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3584  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3585  VkDeviceSize GetUnusedRangeSizeMax() const;
3586  // Returns true if this block is empty - contains only single free suballocation.
3587  bool IsEmpty() const;
3588 
3589  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3590  void AddPoolStats(VmaPoolStats& inoutStats) const;
3591 
3592 #if VMA_STATS_STRING_ENABLED
3593  void PrintDetailedMap(class VmaJsonWriter& json) const;
3594 #endif
3595 
3596  // Creates trivial request for case when block is empty.
3597  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3598 
3599  // Tries to find a place for suballocation with given parameters inside this block.
3600  // If succeeded, fills pAllocationRequest and returns true.
3601  // If failed, returns false.
3602  bool CreateAllocationRequest(
3603  uint32_t currentFrameIndex,
3604  uint32_t frameInUseCount,
3605  VkDeviceSize bufferImageGranularity,
3606  VkDeviceSize allocSize,
3607  VkDeviceSize allocAlignment,
3608  VmaSuballocationType allocType,
3609  bool canMakeOtherLost,
3610  VmaAllocationRequest* pAllocationRequest);
3611 
3612  bool MakeRequestedAllocationsLost(
3613  uint32_t currentFrameIndex,
3614  uint32_t frameInUseCount,
3615  VmaAllocationRequest* pAllocationRequest);
3616 
3617  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3618 
3619  // Makes actual allocation based on request. Request must already be checked and valid.
3620  void Alloc(
3621  const VmaAllocationRequest& request,
3622  VmaSuballocationType type,
3623  VkDeviceSize allocSize,
3624  VmaAllocation hAllocation);
3625 
3626  // Frees suballocation assigned to given memory region.
3627  void Free(const VmaAllocation allocation);
3628  void FreeAtOffset(VkDeviceSize offset);
3629 
3630 private:
3631  VkDeviceSize m_Size;
3632  uint32_t m_FreeCount;
3633  VkDeviceSize m_SumFreeSize;
3634  VmaSuballocationList m_Suballocations;
3635  // Suballocations that are free and have size greater than certain threshold.
3636  // Sorted by size, ascending.
3637  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3638 
3639  bool ValidateFreeSuballocationList() const;
3640 
3641  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3642  // If yes, fills pOffset and returns true. If no, returns false.
3643  bool CheckAllocation(
3644  uint32_t currentFrameIndex,
3645  uint32_t frameInUseCount,
3646  VkDeviceSize bufferImageGranularity,
3647  VkDeviceSize allocSize,
3648  VkDeviceSize allocAlignment,
3649  VmaSuballocationType allocType,
3650  VmaSuballocationList::const_iterator suballocItem,
3651  bool canMakeOtherLost,
3652  VkDeviceSize* pOffset,
3653  size_t* itemsToMakeLostCount,
3654  VkDeviceSize* pSumFreeSize,
3655  VkDeviceSize* pSumItemSize) const;
3656  // Given free suballocation, it merges it with following one, which must also be free.
3657  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3658  // Releases given suballocation, making it free.
3659  // Merges it with adjacent free suballocations if applicable.
3660  // Returns iterator to new free suballocation at this place.
3661  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3662  // Given free suballocation, it inserts it into sorted list of
3663  // m_FreeSuballocationsBySize if it's suitable.
3664  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3665  // Given free suballocation, it removes it from sorted list of
3666  // m_FreeSuballocationsBySize if it's suitable.
3667  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3668 };
3669 
3670 // Helper class that represents mapped memory. Synchronized internally.
3671 class VmaDeviceMemoryMapping
3672 {
3673 public:
3674  VmaDeviceMemoryMapping();
3675  ~VmaDeviceMemoryMapping();
3676 
3677  void* GetMappedData() const { return m_pMappedData; }
3678 
3679  // ppData can be null.
3680  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3681  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3682 
3683 private:
3684  VMA_MUTEX m_Mutex;
3685  uint32_t m_MapCount;
3686  void* m_pMappedData;
3687 };
3688 
3689 /*
3690 Represents a single block of device memory (`VkDeviceMemory`) with all the
3691 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3692 
3693 Thread-safety: This class must be externally synchronized.
3694 */
3695 class VmaDeviceMemoryBlock
3696 {
3697 public:
3698  uint32_t m_MemoryTypeIndex;
3699  VkDeviceMemory m_hMemory;
3700  VmaDeviceMemoryMapping m_Mapping;
3701  VmaBlockMetadata m_Metadata;
3702 
3703  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3704 
3705  ~VmaDeviceMemoryBlock()
3706  {
3707  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3708  }
3709 
3710  // Always call after construction.
3711  void Init(
3712  uint32_t newMemoryTypeIndex,
3713  VkDeviceMemory newMemory,
3714  VkDeviceSize newSize);
3715  // Always call before destruction.
3716  void Destroy(VmaAllocator allocator);
3717 
3718  // Validates all data structures inside this object. If not valid, returns false.
3719  bool Validate() const;
3720 
3721  // ppData can be null.
3722  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3723  void Unmap(VmaAllocator hAllocator, uint32_t count);
3724 };
3725 
3726 struct VmaPointerLess
3727 {
3728  bool operator()(const void* lhs, const void* rhs) const
3729  {
3730  return lhs < rhs;
3731  }
3732 };
3733 
3734 class VmaDefragmentator;
3735 
3736 /*
3737 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3738 Vulkan memory type.
3739 
3740 Synchronized internally with a mutex.
3741 */
3742 struct VmaBlockVector
3743 {
3744  VmaBlockVector(
3745  VmaAllocator hAllocator,
3746  uint32_t memoryTypeIndex,
3747  VkDeviceSize preferredBlockSize,
3748  size_t minBlockCount,
3749  size_t maxBlockCount,
3750  VkDeviceSize bufferImageGranularity,
3751  uint32_t frameInUseCount,
3752  bool isCustomPool);
3753  ~VmaBlockVector();
3754 
3755  VkResult CreateMinBlocks();
3756 
3757  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3758  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3759  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3760  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3761 
3762  void GetPoolStats(VmaPoolStats* pStats);
3763 
3764  bool IsEmpty() const { return m_Blocks.empty(); }
3765 
3766  VkResult Allocate(
3767  VmaPool hCurrentPool,
3768  uint32_t currentFrameIndex,
3769  const VkMemoryRequirements& vkMemReq,
3770  const VmaAllocationCreateInfo& createInfo,
3771  VmaSuballocationType suballocType,
3772  VmaAllocation* pAllocation);
3773 
3774  void Free(
3775  VmaAllocation hAllocation);
3776 
3777  // Adds statistics of this BlockVector to pStats.
3778  void AddStats(VmaStats* pStats);
3779 
3780 #if VMA_STATS_STRING_ENABLED
3781  void PrintDetailedMap(class VmaJsonWriter& json);
3782 #endif
3783 
3784  void MakePoolAllocationsLost(
3785  uint32_t currentFrameIndex,
3786  size_t* pLostAllocationCount);
3787 
3788  VmaDefragmentator* EnsureDefragmentator(
3789  VmaAllocator hAllocator,
3790  uint32_t currentFrameIndex);
3791 
3792  VkResult Defragment(
3793  VmaDefragmentationStats* pDefragmentationStats,
3794  VkDeviceSize& maxBytesToMove,
3795  uint32_t& maxAllocationsToMove);
3796 
3797  void DestroyDefragmentator();
3798 
3799 private:
3800  friend class VmaDefragmentator;
3801 
3802  const VmaAllocator m_hAllocator;
3803  const uint32_t m_MemoryTypeIndex;
3804  const VkDeviceSize m_PreferredBlockSize;
3805  const size_t m_MinBlockCount;
3806  const size_t m_MaxBlockCount;
3807  const VkDeviceSize m_BufferImageGranularity;
3808  const uint32_t m_FrameInUseCount;
3809  const bool m_IsCustomPool;
3810  VMA_MUTEX m_Mutex;
3811  // Incrementally sorted by sumFreeSize, ascending.
3812  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3813  /* There can be at most one allocation that is completely empty - a
3814  hysteresis to avoid pessimistic case of alternating creation and destruction
3815  of a VkDeviceMemory. */
3816  bool m_HasEmptyBlock;
3817  VmaDefragmentator* m_pDefragmentator;
3818 
3819  size_t CalcMaxBlockSize() const;
3820 
3821  // Finds and removes given block from vector.
3822  void Remove(VmaDeviceMemoryBlock* pBlock);
3823 
3824  // Performs single step in sorting m_Blocks. They may not be fully sorted
3825  // after this call.
3826  void IncrementallySortBlocks();
3827 
3828  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3829 };
3830 
3831 struct VmaPool_T
3832 {
3833 public:
3834  VmaBlockVector m_BlockVector;
3835 
3836  // Takes ownership.
3837  VmaPool_T(
3838  VmaAllocator hAllocator,
3839  const VmaPoolCreateInfo& createInfo);
3840  ~VmaPool_T();
3841 
3842  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3843 
3844 #if VMA_STATS_STRING_ENABLED
3845  //void PrintDetailedMap(class VmaStringBuilder& sb);
3846 #endif
3847 };
3848 
3849 class VmaDefragmentator
3850 {
3851  const VmaAllocator m_hAllocator;
3852  VmaBlockVector* const m_pBlockVector;
3853  uint32_t m_CurrentFrameIndex;
3854  VkDeviceSize m_BytesMoved;
3855  uint32_t m_AllocationsMoved;
3856 
3857  struct AllocationInfo
3858  {
3859  VmaAllocation m_hAllocation;
3860  VkBool32* m_pChanged;
3861 
3862  AllocationInfo() :
3863  m_hAllocation(VK_NULL_HANDLE),
3864  m_pChanged(VMA_NULL)
3865  {
3866  }
3867  };
3868 
3869  struct AllocationInfoSizeGreater
3870  {
3871  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3872  {
3873  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3874  }
3875  };
3876 
3877  // Used between AddAllocation and Defragment.
3878  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3879 
3880  struct BlockInfo
3881  {
3882  VmaDeviceMemoryBlock* m_pBlock;
3883  bool m_HasNonMovableAllocations;
3884  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3885 
3886  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3887  m_pBlock(VMA_NULL),
3888  m_HasNonMovableAllocations(true),
3889  m_Allocations(pAllocationCallbacks),
3890  m_pMappedDataForDefragmentation(VMA_NULL)
3891  {
3892  }
3893 
3894  void CalcHasNonMovableAllocations()
3895  {
3896  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3897  const size_t defragmentAllocCount = m_Allocations.size();
3898  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3899  }
3900 
3901  void SortAllocationsBySizeDescecnding()
3902  {
3903  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3904  }
3905 
3906  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3907  void Unmap(VmaAllocator hAllocator);
3908 
3909  private:
3910  // Not null if mapped for defragmentation only, not originally mapped.
3911  void* m_pMappedDataForDefragmentation;
3912  };
3913 
3914  struct BlockPointerLess
3915  {
3916  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3917  {
3918  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3919  }
3920  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3921  {
3922  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3923  }
3924  };
3925 
3926  // 1. Blocks with some non-movable allocations go first.
3927  // 2. Blocks with smaller sumFreeSize go first.
3928  struct BlockInfoCompareMoveDestination
3929  {
3930  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3931  {
3932  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3933  {
3934  return true;
3935  }
3936  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3937  {
3938  return false;
3939  }
3940  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3941  {
3942  return true;
3943  }
3944  return false;
3945  }
3946  };
3947 
3948  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3949  BlockInfoVector m_Blocks;
3950 
3951  VkResult DefragmentRound(
3952  VkDeviceSize maxBytesToMove,
3953  uint32_t maxAllocationsToMove);
3954 
3955  static bool MoveMakesSense(
3956  size_t dstBlockIndex, VkDeviceSize dstOffset,
3957  size_t srcBlockIndex, VkDeviceSize srcOffset);
3958 
3959 public:
3960  VmaDefragmentator(
3961  VmaAllocator hAllocator,
3962  VmaBlockVector* pBlockVector,
3963  uint32_t currentFrameIndex);
3964 
3965  ~VmaDefragmentator();
3966 
3967  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3968  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3969 
3970  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3971 
3972  VkResult Defragment(
3973  VkDeviceSize maxBytesToMove,
3974  uint32_t maxAllocationsToMove);
3975 };
3976 
3977 // Main allocator object.
3978 struct VmaAllocator_T
3979 {
3980  bool m_UseMutex;
3981  bool m_UseKhrDedicatedAllocation;
3982  VkDevice m_hDevice;
3983  bool m_AllocationCallbacksSpecified;
3984  VkAllocationCallbacks m_AllocationCallbacks;
3985  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3986 
3987  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3988  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3989  VMA_MUTEX m_HeapSizeLimitMutex;
3990 
3991  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3992  VkPhysicalDeviceMemoryProperties m_MemProps;
3993 
3994  // Default pools.
3995  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
3996 
3997  // Each vector is sorted by memory (handle value).
3998  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3999  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4000  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4001 
4002  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4003  ~VmaAllocator_T();
4004 
4005  const VkAllocationCallbacks* GetAllocationCallbacks() const
4006  {
4007  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4008  }
4009  const VmaVulkanFunctions& GetVulkanFunctions() const
4010  {
4011  return m_VulkanFunctions;
4012  }
4013 
4014  VkDeviceSize GetBufferImageGranularity() const
4015  {
4016  return VMA_MAX(
4017  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4018  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4019  }
4020 
4021  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4022  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4023 
4024  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4025  {
4026  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4027  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4028  }
4029 
4030  void GetBufferMemoryRequirements(
4031  VkBuffer hBuffer,
4032  VkMemoryRequirements& memReq,
4033  bool& requiresDedicatedAllocation,
4034  bool& prefersDedicatedAllocation) const;
4035  void GetImageMemoryRequirements(
4036  VkImage hImage,
4037  VkMemoryRequirements& memReq,
4038  bool& requiresDedicatedAllocation,
4039  bool& prefersDedicatedAllocation) const;
4040 
4041  // Main allocation function.
4042  VkResult AllocateMemory(
4043  const VkMemoryRequirements& vkMemReq,
4044  bool requiresDedicatedAllocation,
4045  bool prefersDedicatedAllocation,
4046  VkBuffer dedicatedBuffer,
4047  VkImage dedicatedImage,
4048  const VmaAllocationCreateInfo& createInfo,
4049  VmaSuballocationType suballocType,
4050  VmaAllocation* pAllocation);
4051 
4052  // Main deallocation function.
4053  void FreeMemory(const VmaAllocation allocation);
4054 
4055  void CalculateStats(VmaStats* pStats);
4056 
4057 #if VMA_STATS_STRING_ENABLED
4058  void PrintDetailedMap(class VmaJsonWriter& json);
4059 #endif
4060 
4061  VkResult Defragment(
4062  VmaAllocation* pAllocations,
4063  size_t allocationCount,
4064  VkBool32* pAllocationsChanged,
4065  const VmaDefragmentationInfo* pDefragmentationInfo,
4066  VmaDefragmentationStats* pDefragmentationStats);
4067 
4068  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4069 
4070  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4071  void DestroyPool(VmaPool pool);
4072  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4073 
4074  void SetCurrentFrameIndex(uint32_t frameIndex);
4075 
4076  void MakePoolAllocationsLost(
4077  VmaPool hPool,
4078  size_t* pLostAllocationCount);
4079 
4080  void CreateLostAllocation(VmaAllocation* pAllocation);
4081 
4082  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4083  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4084 
4085  VkResult Map(VmaAllocation hAllocation, void** ppData);
4086  void Unmap(VmaAllocation hAllocation);
4087 
4088 private:
4089  VkDeviceSize m_PreferredLargeHeapBlockSize;
4090 
4091  VkPhysicalDevice m_PhysicalDevice;
4092  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4093 
4094  VMA_MUTEX m_PoolsMutex;
4095  // Protected by m_PoolsMutex. Sorted by pointer value.
4096  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4097 
4098  VmaVulkanFunctions m_VulkanFunctions;
4099 
4100  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4101 
4102  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4103 
4104  VkResult AllocateMemoryOfType(
4105  const VkMemoryRequirements& vkMemReq,
4106  bool dedicatedAllocation,
4107  VkBuffer dedicatedBuffer,
4108  VkImage dedicatedImage,
4109  const VmaAllocationCreateInfo& createInfo,
4110  uint32_t memTypeIndex,
4111  VmaSuballocationType suballocType,
4112  VmaAllocation* pAllocation);
4113 
4114  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4115  VkResult AllocateDedicatedMemory(
4116  VkDeviceSize size,
4117  VmaSuballocationType suballocType,
4118  uint32_t memTypeIndex,
4119  bool map,
4120  bool isUserDataString,
4121  void* pUserData,
4122  VkBuffer dedicatedBuffer,
4123  VkImage dedicatedImage,
4124  VmaAllocation* pAllocation);
4125 
4126  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4127  void FreeDedicatedMemory(VmaAllocation allocation);
4128 };
4129 
4131 // Memory allocation #2 after VmaAllocator_T definition
4132 
4133 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4134 {
4135  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4136 }
4137 
4138 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4139 {
4140  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4141 }
4142 
4143 template<typename T>
4144 static T* VmaAllocate(VmaAllocator hAllocator)
4145 {
4146  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4147 }
4148 
4149 template<typename T>
4150 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4151 {
4152  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4153 }
4154 
4155 template<typename T>
4156 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4157 {
4158  if(ptr != VMA_NULL)
4159  {
4160  ptr->~T();
4161  VmaFree(hAllocator, ptr);
4162  }
4163 }
4164 
4165 template<typename T>
4166 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4167 {
4168  if(ptr != VMA_NULL)
4169  {
4170  for(size_t i = count; i--; )
4171  ptr[i].~T();
4172  VmaFree(hAllocator, ptr);
4173  }
4174 }
4175 
4177 // VmaStringBuilder
4178 
4179 #if VMA_STATS_STRING_ENABLED
4180 
4181 class VmaStringBuilder
4182 {
4183 public:
4184  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4185  size_t GetLength() const { return m_Data.size(); }
4186  const char* GetData() const { return m_Data.data(); }
4187 
4188  void Add(char ch) { m_Data.push_back(ch); }
4189  void Add(const char* pStr);
4190  void AddNewLine() { Add('\n'); }
4191  void AddNumber(uint32_t num);
4192  void AddNumber(uint64_t num);
4193  void AddPointer(const void* ptr);
4194 
4195 private:
4196  VmaVector< char, VmaStlAllocator<char> > m_Data;
4197 };
4198 
4199 void VmaStringBuilder::Add(const char* pStr)
4200 {
4201  const size_t strLen = strlen(pStr);
4202  if(strLen > 0)
4203  {
4204  const size_t oldCount = m_Data.size();
4205  m_Data.resize(oldCount + strLen);
4206  memcpy(m_Data.data() + oldCount, pStr, strLen);
4207  }
4208 }
4209 
4210 void VmaStringBuilder::AddNumber(uint32_t num)
4211 {
4212  char buf[11];
4213  VmaUint32ToStr(buf, sizeof(buf), num);
4214  Add(buf);
4215 }
4216 
4217 void VmaStringBuilder::AddNumber(uint64_t num)
4218 {
4219  char buf[21];
4220  VmaUint64ToStr(buf, sizeof(buf), num);
4221  Add(buf);
4222 }
4223 
4224 void VmaStringBuilder::AddPointer(const void* ptr)
4225 {
4226  char buf[21];
4227  VmaPtrToStr(buf, sizeof(buf), ptr);
4228  Add(buf);
4229 }
4230 
4231 #endif // #if VMA_STATS_STRING_ENABLED
4232 
4234 // VmaJsonWriter
4235 
4236 #if VMA_STATS_STRING_ENABLED
4237 
4238 class VmaJsonWriter
4239 {
4240 public:
4241  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4242  ~VmaJsonWriter();
4243 
4244  void BeginObject(bool singleLine = false);
4245  void EndObject();
4246 
4247  void BeginArray(bool singleLine = false);
4248  void EndArray();
4249 
4250  void WriteString(const char* pStr);
4251  void BeginString(const char* pStr = VMA_NULL);
4252  void ContinueString(const char* pStr);
4253  void ContinueString(uint32_t n);
4254  void ContinueString(uint64_t n);
4255  void ContinueString_Pointer(const void* ptr);
4256  void EndString(const char* pStr = VMA_NULL);
4257 
4258  void WriteNumber(uint32_t n);
4259  void WriteNumber(uint64_t n);
4260  void WriteBool(bool b);
4261  void WriteNull();
4262 
4263 private:
4264  static const char* const INDENT;
4265 
4266  enum COLLECTION_TYPE
4267  {
4268  COLLECTION_TYPE_OBJECT,
4269  COLLECTION_TYPE_ARRAY,
4270  };
4271  struct StackItem
4272  {
4273  COLLECTION_TYPE type;
4274  uint32_t valueCount;
4275  bool singleLineMode;
4276  };
4277 
4278  VmaStringBuilder& m_SB;
4279  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4280  bool m_InsideString;
4281 
4282  void BeginValue(bool isString);
4283  void WriteIndent(bool oneLess = false);
4284 };
4285 
4286 const char* const VmaJsonWriter::INDENT = " ";
4287 
4288 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4289  m_SB(sb),
4290  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4291  m_InsideString(false)
4292 {
4293 }
4294 
4295 VmaJsonWriter::~VmaJsonWriter()
4296 {
4297  VMA_ASSERT(!m_InsideString);
4298  VMA_ASSERT(m_Stack.empty());
4299 }
4300 
4301 void VmaJsonWriter::BeginObject(bool singleLine)
4302 {
4303  VMA_ASSERT(!m_InsideString);
4304 
4305  BeginValue(false);
4306  m_SB.Add('{');
4307 
4308  StackItem item;
4309  item.type = COLLECTION_TYPE_OBJECT;
4310  item.valueCount = 0;
4311  item.singleLineMode = singleLine;
4312  m_Stack.push_back(item);
4313 }
4314 
4315 void VmaJsonWriter::EndObject()
4316 {
4317  VMA_ASSERT(!m_InsideString);
4318 
4319  WriteIndent(true);
4320  m_SB.Add('}');
4321 
4322  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4323  m_Stack.pop_back();
4324 }
4325 
4326 void VmaJsonWriter::BeginArray(bool singleLine)
4327 {
4328  VMA_ASSERT(!m_InsideString);
4329 
4330  BeginValue(false);
4331  m_SB.Add('[');
4332 
4333  StackItem item;
4334  item.type = COLLECTION_TYPE_ARRAY;
4335  item.valueCount = 0;
4336  item.singleLineMode = singleLine;
4337  m_Stack.push_back(item);
4338 }
4339 
4340 void VmaJsonWriter::EndArray()
4341 {
4342  VMA_ASSERT(!m_InsideString);
4343 
4344  WriteIndent(true);
4345  m_SB.Add(']');
4346 
4347  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4348  m_Stack.pop_back();
4349 }
4350 
4351 void VmaJsonWriter::WriteString(const char* pStr)
4352 {
4353  BeginString(pStr);
4354  EndString();
4355 }
4356 
4357 void VmaJsonWriter::BeginString(const char* pStr)
4358 {
4359  VMA_ASSERT(!m_InsideString);
4360 
4361  BeginValue(true);
4362  m_SB.Add('"');
4363  m_InsideString = true;
4364  if(pStr != VMA_NULL && pStr[0] != '\0')
4365  {
4366  ContinueString(pStr);
4367  }
4368 }
4369 
4370 void VmaJsonWriter::ContinueString(const char* pStr)
4371 {
4372  VMA_ASSERT(m_InsideString);
4373 
4374  const size_t strLen = strlen(pStr);
4375  for(size_t i = 0; i < strLen; ++i)
4376  {
4377  char ch = pStr[i];
4378  if(ch == '\'')
4379  {
4380  m_SB.Add("\\\\");
4381  }
4382  else if(ch == '"')
4383  {
4384  m_SB.Add("\\\"");
4385  }
4386  else if(ch >= 32)
4387  {
4388  m_SB.Add(ch);
4389  }
4390  else switch(ch)
4391  {
4392  case '\b':
4393  m_SB.Add("\\b");
4394  break;
4395  case '\f':
4396  m_SB.Add("\\f");
4397  break;
4398  case '\n':
4399  m_SB.Add("\\n");
4400  break;
4401  case '\r':
4402  m_SB.Add("\\r");
4403  break;
4404  case '\t':
4405  m_SB.Add("\\t");
4406  break;
4407  default:
4408  VMA_ASSERT(0 && "Character not currently supported.");
4409  break;
4410  }
4411  }
4412 }
4413 
4414 void VmaJsonWriter::ContinueString(uint32_t n)
4415 {
4416  VMA_ASSERT(m_InsideString);
4417  m_SB.AddNumber(n);
4418 }
4419 
4420 void VmaJsonWriter::ContinueString(uint64_t n)
4421 {
4422  VMA_ASSERT(m_InsideString);
4423  m_SB.AddNumber(n);
4424 }
4425 
4426 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4427 {
4428  VMA_ASSERT(m_InsideString);
4429  m_SB.AddPointer(ptr);
4430 }
4431 
4432 void VmaJsonWriter::EndString(const char* pStr)
4433 {
4434  VMA_ASSERT(m_InsideString);
4435  if(pStr != VMA_NULL && pStr[0] != '\0')
4436  {
4437  ContinueString(pStr);
4438  }
4439  m_SB.Add('"');
4440  m_InsideString = false;
4441 }
4442 
4443 void VmaJsonWriter::WriteNumber(uint32_t n)
4444 {
4445  VMA_ASSERT(!m_InsideString);
4446  BeginValue(false);
4447  m_SB.AddNumber(n);
4448 }
4449 
4450 void VmaJsonWriter::WriteNumber(uint64_t n)
4451 {
4452  VMA_ASSERT(!m_InsideString);
4453  BeginValue(false);
4454  m_SB.AddNumber(n);
4455 }
4456 
4457 void VmaJsonWriter::WriteBool(bool b)
4458 {
4459  VMA_ASSERT(!m_InsideString);
4460  BeginValue(false);
4461  m_SB.Add(b ? "true" : "false");
4462 }
4463 
4464 void VmaJsonWriter::WriteNull()
4465 {
4466  VMA_ASSERT(!m_InsideString);
4467  BeginValue(false);
4468  m_SB.Add("null");
4469 }
4470 
4471 void VmaJsonWriter::BeginValue(bool isString)
4472 {
4473  if(!m_Stack.empty())
4474  {
4475  StackItem& currItem = m_Stack.back();
4476  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4477  currItem.valueCount % 2 == 0)
4478  {
4479  VMA_ASSERT(isString);
4480  }
4481 
4482  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4483  currItem.valueCount % 2 != 0)
4484  {
4485  m_SB.Add(": ");
4486  }
4487  else if(currItem.valueCount > 0)
4488  {
4489  m_SB.Add(", ");
4490  WriteIndent();
4491  }
4492  else
4493  {
4494  WriteIndent();
4495  }
4496  ++currItem.valueCount;
4497  }
4498 }
4499 
4500 void VmaJsonWriter::WriteIndent(bool oneLess)
4501 {
4502  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4503  {
4504  m_SB.AddNewLine();
4505 
4506  size_t count = m_Stack.size();
4507  if(count > 0 && oneLess)
4508  {
4509  --count;
4510  }
4511  for(size_t i = 0; i < count; ++i)
4512  {
4513  m_SB.Add(INDENT);
4514  }
4515  }
4516 }
4517 
4518 #endif // #if VMA_STATS_STRING_ENABLED
4519 
4521 
4522 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4523 {
4524  if(IsUserDataString())
4525  {
4526  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4527 
4528  FreeUserDataString(hAllocator);
4529 
4530  if(pUserData != VMA_NULL)
4531  {
4532  const char* const newStrSrc = (char*)pUserData;
4533  const size_t newStrLen = strlen(newStrSrc);
4534  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4535  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4536  m_pUserData = newStrDst;
4537  }
4538  }
4539  else
4540  {
4541  m_pUserData = pUserData;
4542  }
4543 }
4544 
4545 void VmaAllocation_T::ChangeBlockAllocation(
4546  VmaAllocator hAllocator,
4547  VmaDeviceMemoryBlock* block,
4548  VkDeviceSize offset)
4549 {
4550  VMA_ASSERT(block != VMA_NULL);
4551  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4552 
4553  // Move mapping reference counter from old block to new block.
4554  if(block != m_BlockAllocation.m_Block)
4555  {
4556  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4557  if(IsPersistentMap())
4558  ++mapRefCount;
4559  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4560  block->Map(hAllocator, mapRefCount, VMA_NULL);
4561  }
4562 
4563  m_BlockAllocation.m_Block = block;
4564  m_BlockAllocation.m_Offset = offset;
4565 }
4566 
4567 VkDeviceSize VmaAllocation_T::GetOffset() const
4568 {
4569  switch(m_Type)
4570  {
4571  case ALLOCATION_TYPE_BLOCK:
4572  return m_BlockAllocation.m_Offset;
4573  case ALLOCATION_TYPE_DEDICATED:
4574  return 0;
4575  default:
4576  VMA_ASSERT(0);
4577  return 0;
4578  }
4579 }
4580 
4581 VkDeviceMemory VmaAllocation_T::GetMemory() const
4582 {
4583  switch(m_Type)
4584  {
4585  case ALLOCATION_TYPE_BLOCK:
4586  return m_BlockAllocation.m_Block->m_hMemory;
4587  case ALLOCATION_TYPE_DEDICATED:
4588  return m_DedicatedAllocation.m_hMemory;
4589  default:
4590  VMA_ASSERT(0);
4591  return VK_NULL_HANDLE;
4592  }
4593 }
4594 
4595 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4596 {
4597  switch(m_Type)
4598  {
4599  case ALLOCATION_TYPE_BLOCK:
4600  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4601  case ALLOCATION_TYPE_DEDICATED:
4602  return m_DedicatedAllocation.m_MemoryTypeIndex;
4603  default:
4604  VMA_ASSERT(0);
4605  return UINT32_MAX;
4606  }
4607 }
4608 
4609 void* VmaAllocation_T::GetMappedData() const
4610 {
4611  switch(m_Type)
4612  {
4613  case ALLOCATION_TYPE_BLOCK:
4614  if(m_MapCount != 0)
4615  {
4616  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4617  VMA_ASSERT(pBlockData != VMA_NULL);
4618  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4619  }
4620  else
4621  {
4622  return VMA_NULL;
4623  }
4624  break;
4625  case ALLOCATION_TYPE_DEDICATED:
4626  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4627  return m_DedicatedAllocation.m_pMappedData;
4628  default:
4629  VMA_ASSERT(0);
4630  return VMA_NULL;
4631  }
4632 }
4633 
4634 bool VmaAllocation_T::CanBecomeLost() const
4635 {
4636  switch(m_Type)
4637  {
4638  case ALLOCATION_TYPE_BLOCK:
4639  return m_BlockAllocation.m_CanBecomeLost;
4640  case ALLOCATION_TYPE_DEDICATED:
4641  return false;
4642  default:
4643  VMA_ASSERT(0);
4644  return false;
4645  }
4646 }
4647 
4648 VmaPool VmaAllocation_T::GetPool() const
4649 {
4650  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4651  return m_BlockAllocation.m_hPool;
4652 }
4653 
4654 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4655 {
4656  VMA_ASSERT(CanBecomeLost());
4657 
4658  /*
4659  Warning: This is a carefully designed algorithm.
4660  Do not modify unless you really know what you're doing :)
4661  */
4662  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4663  for(;;)
4664  {
4665  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4666  {
4667  VMA_ASSERT(0);
4668  return false;
4669  }
4670  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4671  {
4672  return false;
4673  }
4674  else // Last use time earlier than current time.
4675  {
4676  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4677  {
4678  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4679  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4680  return true;
4681  }
4682  }
4683  }
4684 }
4685 
4686 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4687 {
4688  VMA_ASSERT(IsUserDataString());
4689  if(m_pUserData != VMA_NULL)
4690  {
4691  char* const oldStr = (char*)m_pUserData;
4692  const size_t oldStrLen = strlen(oldStr);
4693  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4694  m_pUserData = VMA_NULL;
4695  }
4696 }
4697 
4698 void VmaAllocation_T::BlockAllocMap()
4699 {
4700  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4701 
4702  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4703  {
4704  ++m_MapCount;
4705  }
4706  else
4707  {
4708  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4709  }
4710 }
4711 
4712 void VmaAllocation_T::BlockAllocUnmap()
4713 {
4714  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4715 
4716  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4717  {
4718  --m_MapCount;
4719  }
4720  else
4721  {
4722  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4723  }
4724 }
4725 
4726 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4727 {
4728  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4729 
4730  if(m_MapCount != 0)
4731  {
4732  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4733  {
4734  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4735  *ppData = m_DedicatedAllocation.m_pMappedData;
4736  ++m_MapCount;
4737  return VK_SUCCESS;
4738  }
4739  else
4740  {
4741  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4742  return VK_ERROR_MEMORY_MAP_FAILED;
4743  }
4744  }
4745  else
4746  {
4747  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4748  hAllocator->m_hDevice,
4749  m_DedicatedAllocation.m_hMemory,
4750  0, // offset
4751  VK_WHOLE_SIZE,
4752  0, // flags
4753  ppData);
4754  if(result == VK_SUCCESS)
4755  {
4756  m_DedicatedAllocation.m_pMappedData = *ppData;
4757  m_MapCount = 1;
4758  }
4759  return result;
4760  }
4761 }
4762 
4763 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4764 {
4765  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4766 
4767  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4768  {
4769  --m_MapCount;
4770  if(m_MapCount == 0)
4771  {
4772  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4773  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4774  hAllocator->m_hDevice,
4775  m_DedicatedAllocation.m_hMemory);
4776  }
4777  }
4778  else
4779  {
4780  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4781  }
4782 }
4783 
4784 #if VMA_STATS_STRING_ENABLED
4785 
4786 // Correspond to values of enum VmaSuballocationType.
4787 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4788  "FREE",
4789  "UNKNOWN",
4790  "BUFFER",
4791  "IMAGE_UNKNOWN",
4792  "IMAGE_LINEAR",
4793  "IMAGE_OPTIMAL",
4794 };
4795 
4796 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4797 {
4798  json.BeginObject();
4799 
4800  json.WriteString("Blocks");
4801  json.WriteNumber(stat.blockCount);
4802 
4803  json.WriteString("Allocations");
4804  json.WriteNumber(stat.allocationCount);
4805 
4806  json.WriteString("UnusedRanges");
4807  json.WriteNumber(stat.unusedRangeCount);
4808 
4809  json.WriteString("UsedBytes");
4810  json.WriteNumber(stat.usedBytes);
4811 
4812  json.WriteString("UnusedBytes");
4813  json.WriteNumber(stat.unusedBytes);
4814 
4815  if(stat.allocationCount > 1)
4816  {
4817  json.WriteString("AllocationSize");
4818  json.BeginObject(true);
4819  json.WriteString("Min");
4820  json.WriteNumber(stat.allocationSizeMin);
4821  json.WriteString("Avg");
4822  json.WriteNumber(stat.allocationSizeAvg);
4823  json.WriteString("Max");
4824  json.WriteNumber(stat.allocationSizeMax);
4825  json.EndObject();
4826  }
4827 
4828  if(stat.unusedRangeCount > 1)
4829  {
4830  json.WriteString("UnusedRangeSize");
4831  json.BeginObject(true);
4832  json.WriteString("Min");
4833  json.WriteNumber(stat.unusedRangeSizeMin);
4834  json.WriteString("Avg");
4835  json.WriteNumber(stat.unusedRangeSizeAvg);
4836  json.WriteString("Max");
4837  json.WriteNumber(stat.unusedRangeSizeMax);
4838  json.EndObject();
4839  }
4840 
4841  json.EndObject();
4842 }
4843 
4844 #endif // #if VMA_STATS_STRING_ENABLED
4845 
4846 struct VmaSuballocationItemSizeLess
4847 {
4848  bool operator()(
4849  const VmaSuballocationList::iterator lhs,
4850  const VmaSuballocationList::iterator rhs) const
4851  {
4852  return lhs->size < rhs->size;
4853  }
4854  bool operator()(
4855  const VmaSuballocationList::iterator lhs,
4856  VkDeviceSize rhsSize) const
4857  {
4858  return lhs->size < rhsSize;
4859  }
4860 };
4861 
4863 // class VmaBlockMetadata
4864 
4865 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4866  m_Size(0),
4867  m_FreeCount(0),
4868  m_SumFreeSize(0),
4869  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4870  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4871 {
4872 }
4873 
4874 VmaBlockMetadata::~VmaBlockMetadata()
4875 {
4876 }
4877 
4878 void VmaBlockMetadata::Init(VkDeviceSize size)
4879 {
4880  m_Size = size;
4881  m_FreeCount = 1;
4882  m_SumFreeSize = size;
4883 
4884  VmaSuballocation suballoc = {};
4885  suballoc.offset = 0;
4886  suballoc.size = size;
4887  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4888  suballoc.hAllocation = VK_NULL_HANDLE;
4889 
4890  m_Suballocations.push_back(suballoc);
4891  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4892  --suballocItem;
4893  m_FreeSuballocationsBySize.push_back(suballocItem);
4894 }
4895 
4896 bool VmaBlockMetadata::Validate() const
4897 {
4898  if(m_Suballocations.empty())
4899  {
4900  return false;
4901  }
4902 
4903  // Expected offset of new suballocation as calculates from previous ones.
4904  VkDeviceSize calculatedOffset = 0;
4905  // Expected number of free suballocations as calculated from traversing their list.
4906  uint32_t calculatedFreeCount = 0;
4907  // Expected sum size of free suballocations as calculated from traversing their list.
4908  VkDeviceSize calculatedSumFreeSize = 0;
4909  // Expected number of free suballocations that should be registered in
4910  // m_FreeSuballocationsBySize calculated from traversing their list.
4911  size_t freeSuballocationsToRegister = 0;
4912  // True if previous visisted suballocation was free.
4913  bool prevFree = false;
4914 
4915  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4916  suballocItem != m_Suballocations.cend();
4917  ++suballocItem)
4918  {
4919  const VmaSuballocation& subAlloc = *suballocItem;
4920 
4921  // Actual offset of this suballocation doesn't match expected one.
4922  if(subAlloc.offset != calculatedOffset)
4923  {
4924  return false;
4925  }
4926 
4927  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4928  // Two adjacent free suballocations are invalid. They should be merged.
4929  if(prevFree && currFree)
4930  {
4931  return false;
4932  }
4933 
4934  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4935  {
4936  return false;
4937  }
4938 
4939  if(currFree)
4940  {
4941  calculatedSumFreeSize += subAlloc.size;
4942  ++calculatedFreeCount;
4943  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4944  {
4945  ++freeSuballocationsToRegister;
4946  }
4947  }
4948  else
4949  {
4950  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
4951  {
4952  return false;
4953  }
4954  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
4955  {
4956  return false;
4957  }
4958  }
4959 
4960  calculatedOffset += subAlloc.size;
4961  prevFree = currFree;
4962  }
4963 
4964  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4965  // match expected one.
4966  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4967  {
4968  return false;
4969  }
4970 
4971  VkDeviceSize lastSize = 0;
4972  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4973  {
4974  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4975 
4976  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4977  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4978  {
4979  return false;
4980  }
4981  // They must be sorted by size ascending.
4982  if(suballocItem->size < lastSize)
4983  {
4984  return false;
4985  }
4986 
4987  lastSize = suballocItem->size;
4988  }
4989 
4990  // Check if totals match calculacted values.
4991  if(!ValidateFreeSuballocationList() ||
4992  (calculatedOffset != m_Size) ||
4993  (calculatedSumFreeSize != m_SumFreeSize) ||
4994  (calculatedFreeCount != m_FreeCount))
4995  {
4996  return false;
4997  }
4998 
4999  return true;
5000 }
5001 
5002 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5003 {
5004  if(!m_FreeSuballocationsBySize.empty())
5005  {
5006  return m_FreeSuballocationsBySize.back()->size;
5007  }
5008  else
5009  {
5010  return 0;
5011  }
5012 }
5013 
5014 bool VmaBlockMetadata::IsEmpty() const
5015 {
5016  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5017 }
5018 
5019 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5020 {
5021  outInfo.blockCount = 1;
5022 
5023  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5024  outInfo.allocationCount = rangeCount - m_FreeCount;
5025  outInfo.unusedRangeCount = m_FreeCount;
5026 
5027  outInfo.unusedBytes = m_SumFreeSize;
5028  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5029 
5030  outInfo.allocationSizeMin = UINT64_MAX;
5031  outInfo.allocationSizeMax = 0;
5032  outInfo.unusedRangeSizeMin = UINT64_MAX;
5033  outInfo.unusedRangeSizeMax = 0;
5034 
5035  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5036  suballocItem != m_Suballocations.cend();
5037  ++suballocItem)
5038  {
5039  const VmaSuballocation& suballoc = *suballocItem;
5040  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5041  {
5042  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5043  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5044  }
5045  else
5046  {
5047  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5048  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5049  }
5050  }
5051 }
5052 
5053 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5054 {
5055  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5056 
5057  inoutStats.size += m_Size;
5058  inoutStats.unusedSize += m_SumFreeSize;
5059  inoutStats.allocationCount += rangeCount - m_FreeCount;
5060  inoutStats.unusedRangeCount += m_FreeCount;
5061  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5062 }
5063 
5064 #if VMA_STATS_STRING_ENABLED
5065 
5066 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5067 {
5068  json.BeginObject();
5069 
5070  json.WriteString("TotalBytes");
5071  json.WriteNumber(m_Size);
5072 
5073  json.WriteString("UnusedBytes");
5074  json.WriteNumber(m_SumFreeSize);
5075 
5076  json.WriteString("Allocations");
5077  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5078 
5079  json.WriteString("UnusedRanges");
5080  json.WriteNumber(m_FreeCount);
5081 
5082  json.WriteString("Suballocations");
5083  json.BeginArray();
5084  size_t i = 0;
5085  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5086  suballocItem != m_Suballocations.cend();
5087  ++suballocItem, ++i)
5088  {
5089  json.BeginObject(true);
5090 
5091  json.WriteString("Type");
5092  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5093 
5094  json.WriteString("Size");
5095  json.WriteNumber(suballocItem->size);
5096 
5097  json.WriteString("Offset");
5098  json.WriteNumber(suballocItem->offset);
5099 
5100  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5101  {
5102  const void* pUserData = suballocItem->hAllocation->GetUserData();
5103  if(pUserData != VMA_NULL)
5104  {
5105  json.WriteString("UserData");
5106  if(suballocItem->hAllocation->IsUserDataString())
5107  {
5108  json.WriteString((const char*)pUserData);
5109  }
5110  else
5111  {
5112  json.BeginString();
5113  json.ContinueString_Pointer(pUserData);
5114  json.EndString();
5115  }
5116  }
5117  }
5118 
5119  json.EndObject();
5120  }
5121  json.EndArray();
5122 
5123  json.EndObject();
5124 }
5125 
5126 #endif // #if VMA_STATS_STRING_ENABLED
5127 
5128 /*
5129 How many suitable free suballocations to analyze before choosing best one.
5130 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5131  be chosen.
5132 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5133  suballocations will be analized and best one will be chosen.
5134 - Any other value is also acceptable.
5135 */
5136 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5137 
5138 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5139 {
5140  VMA_ASSERT(IsEmpty());
5141  pAllocationRequest->offset = 0;
5142  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5143  pAllocationRequest->sumItemSize = 0;
5144  pAllocationRequest->item = m_Suballocations.begin();
5145  pAllocationRequest->itemsToMakeLostCount = 0;
5146 }
5147 
5148 bool VmaBlockMetadata::CreateAllocationRequest(
5149  uint32_t currentFrameIndex,
5150  uint32_t frameInUseCount,
5151  VkDeviceSize bufferImageGranularity,
5152  VkDeviceSize allocSize,
5153  VkDeviceSize allocAlignment,
5154  VmaSuballocationType allocType,
5155  bool canMakeOtherLost,
5156  VmaAllocationRequest* pAllocationRequest)
5157 {
5158  VMA_ASSERT(allocSize > 0);
5159  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5160  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5161  VMA_HEAVY_ASSERT(Validate());
5162 
5163  // There is not enough total free space in this block to fullfill the request: Early return.
5164  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5165  {
5166  return false;
5167  }
5168 
5169  // New algorithm, efficiently searching freeSuballocationsBySize.
5170  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5171  if(freeSuballocCount > 0)
5172  {
5173  if(VMA_BEST_FIT)
5174  {
5175  // Find first free suballocation with size not less than allocSize.
5176  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5177  m_FreeSuballocationsBySize.data(),
5178  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5179  allocSize,
5180  VmaSuballocationItemSizeLess());
5181  size_t index = it - m_FreeSuballocationsBySize.data();
5182  for(; index < freeSuballocCount; ++index)
5183  {
5184  if(CheckAllocation(
5185  currentFrameIndex,
5186  frameInUseCount,
5187  bufferImageGranularity,
5188  allocSize,
5189  allocAlignment,
5190  allocType,
5191  m_FreeSuballocationsBySize[index],
5192  false, // canMakeOtherLost
5193  &pAllocationRequest->offset,
5194  &pAllocationRequest->itemsToMakeLostCount,
5195  &pAllocationRequest->sumFreeSize,
5196  &pAllocationRequest->sumItemSize))
5197  {
5198  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5199  return true;
5200  }
5201  }
5202  }
5203  else
5204  {
5205  // Search staring from biggest suballocations.
5206  for(size_t index = freeSuballocCount; index--; )
5207  {
5208  if(CheckAllocation(
5209  currentFrameIndex,
5210  frameInUseCount,
5211  bufferImageGranularity,
5212  allocSize,
5213  allocAlignment,
5214  allocType,
5215  m_FreeSuballocationsBySize[index],
5216  false, // canMakeOtherLost
5217  &pAllocationRequest->offset,
5218  &pAllocationRequest->itemsToMakeLostCount,
5219  &pAllocationRequest->sumFreeSize,
5220  &pAllocationRequest->sumItemSize))
5221  {
5222  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5223  return true;
5224  }
5225  }
5226  }
5227  }
5228 
5229  if(canMakeOtherLost)
5230  {
5231  // Brute-force algorithm. TODO: Come up with something better.
5232 
5233  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5234  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5235 
5236  VmaAllocationRequest tmpAllocRequest = {};
5237  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5238  suballocIt != m_Suballocations.end();
5239  ++suballocIt)
5240  {
5241  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5242  suballocIt->hAllocation->CanBecomeLost())
5243  {
5244  if(CheckAllocation(
5245  currentFrameIndex,
5246  frameInUseCount,
5247  bufferImageGranularity,
5248  allocSize,
5249  allocAlignment,
5250  allocType,
5251  suballocIt,
5252  canMakeOtherLost,
5253  &tmpAllocRequest.offset,
5254  &tmpAllocRequest.itemsToMakeLostCount,
5255  &tmpAllocRequest.sumFreeSize,
5256  &tmpAllocRequest.sumItemSize))
5257  {
5258  tmpAllocRequest.item = suballocIt;
5259 
5260  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5261  {
5262  *pAllocationRequest = tmpAllocRequest;
5263  }
5264  }
5265  }
5266  }
5267 
5268  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5269  {
5270  return true;
5271  }
5272  }
5273 
5274  return false;
5275 }
5276 
5277 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5278  uint32_t currentFrameIndex,
5279  uint32_t frameInUseCount,
5280  VmaAllocationRequest* pAllocationRequest)
5281 {
5282  while(pAllocationRequest->itemsToMakeLostCount > 0)
5283  {
5284  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5285  {
5286  ++pAllocationRequest->item;
5287  }
5288  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5289  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5290  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5291  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5292  {
5293  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5294  --pAllocationRequest->itemsToMakeLostCount;
5295  }
5296  else
5297  {
5298  return false;
5299  }
5300  }
5301 
5302  VMA_HEAVY_ASSERT(Validate());
5303  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5304  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5305 
5306  return true;
5307 }
5308 
5309 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5310 {
5311  uint32_t lostAllocationCount = 0;
5312  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5313  it != m_Suballocations.end();
5314  ++it)
5315  {
5316  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5317  it->hAllocation->CanBecomeLost() &&
5318  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5319  {
5320  it = FreeSuballocation(it);
5321  ++lostAllocationCount;
5322  }
5323  }
5324  return lostAllocationCount;
5325 }
5326 
5327 void VmaBlockMetadata::Alloc(
5328  const VmaAllocationRequest& request,
5329  VmaSuballocationType type,
5330  VkDeviceSize allocSize,
5331  VmaAllocation hAllocation)
5332 {
5333  VMA_ASSERT(request.item != m_Suballocations.end());
5334  VmaSuballocation& suballoc = *request.item;
5335  // Given suballocation is a free block.
5336  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5337  // Given offset is inside this suballocation.
5338  VMA_ASSERT(request.offset >= suballoc.offset);
5339  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5340  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5341  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5342 
5343  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5344  // it to become used.
5345  UnregisterFreeSuballocation(request.item);
5346 
5347  suballoc.offset = request.offset;
5348  suballoc.size = allocSize;
5349  suballoc.type = type;
5350  suballoc.hAllocation = hAllocation;
5351 
5352  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5353  if(paddingEnd)
5354  {
5355  VmaSuballocation paddingSuballoc = {};
5356  paddingSuballoc.offset = request.offset + allocSize;
5357  paddingSuballoc.size = paddingEnd;
5358  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5359  VmaSuballocationList::iterator next = request.item;
5360  ++next;
5361  const VmaSuballocationList::iterator paddingEndItem =
5362  m_Suballocations.insert(next, paddingSuballoc);
5363  RegisterFreeSuballocation(paddingEndItem);
5364  }
5365 
5366  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5367  if(paddingBegin)
5368  {
5369  VmaSuballocation paddingSuballoc = {};
5370  paddingSuballoc.offset = request.offset - paddingBegin;
5371  paddingSuballoc.size = paddingBegin;
5372  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5373  const VmaSuballocationList::iterator paddingBeginItem =
5374  m_Suballocations.insert(request.item, paddingSuballoc);
5375  RegisterFreeSuballocation(paddingBeginItem);
5376  }
5377 
5378  // Update totals.
5379  m_FreeCount = m_FreeCount - 1;
5380  if(paddingBegin > 0)
5381  {
5382  ++m_FreeCount;
5383  }
5384  if(paddingEnd > 0)
5385  {
5386  ++m_FreeCount;
5387  }
5388  m_SumFreeSize -= allocSize;
5389 }
5390 
5391 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5392 {
5393  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5394  suballocItem != m_Suballocations.end();
5395  ++suballocItem)
5396  {
5397  VmaSuballocation& suballoc = *suballocItem;
5398  if(suballoc.hAllocation == allocation)
5399  {
5400  FreeSuballocation(suballocItem);
5401  VMA_HEAVY_ASSERT(Validate());
5402  return;
5403  }
5404  }
5405  VMA_ASSERT(0 && "Not found!");
5406 }
5407 
5408 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5409 {
5410  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5411  suballocItem != m_Suballocations.end();
5412  ++suballocItem)
5413  {
5414  VmaSuballocation& suballoc = *suballocItem;
5415  if(suballoc.offset == offset)
5416  {
5417  FreeSuballocation(suballocItem);
5418  return;
5419  }
5420  }
5421  VMA_ASSERT(0 && "Not found!");
5422 }
5423 
5424 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5425 {
5426  VkDeviceSize lastSize = 0;
5427  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5428  {
5429  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5430 
5431  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5432  {
5433  VMA_ASSERT(0);
5434  return false;
5435  }
5436  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5437  {
5438  VMA_ASSERT(0);
5439  return false;
5440  }
5441  if(it->size < lastSize)
5442  {
5443  VMA_ASSERT(0);
5444  return false;
5445  }
5446 
5447  lastSize = it->size;
5448  }
5449  return true;
5450 }
5451 
5452 bool VmaBlockMetadata::CheckAllocation(
5453  uint32_t currentFrameIndex,
5454  uint32_t frameInUseCount,
5455  VkDeviceSize bufferImageGranularity,
5456  VkDeviceSize allocSize,
5457  VkDeviceSize allocAlignment,
5458  VmaSuballocationType allocType,
5459  VmaSuballocationList::const_iterator suballocItem,
5460  bool canMakeOtherLost,
5461  VkDeviceSize* pOffset,
5462  size_t* itemsToMakeLostCount,
5463  VkDeviceSize* pSumFreeSize,
5464  VkDeviceSize* pSumItemSize) const
5465 {
5466  VMA_ASSERT(allocSize > 0);
5467  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5468  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5469  VMA_ASSERT(pOffset != VMA_NULL);
5470 
5471  *itemsToMakeLostCount = 0;
5472  *pSumFreeSize = 0;
5473  *pSumItemSize = 0;
5474 
5475  if(canMakeOtherLost)
5476  {
5477  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5478  {
5479  *pSumFreeSize = suballocItem->size;
5480  }
5481  else
5482  {
5483  if(suballocItem->hAllocation->CanBecomeLost() &&
5484  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5485  {
5486  ++*itemsToMakeLostCount;
5487  *pSumItemSize = suballocItem->size;
5488  }
5489  else
5490  {
5491  return false;
5492  }
5493  }
5494 
5495  // Remaining size is too small for this request: Early return.
5496  if(m_Size - suballocItem->offset < allocSize)
5497  {
5498  return false;
5499  }
5500 
5501  // Start from offset equal to beginning of this suballocation.
5502  *pOffset = suballocItem->offset;
5503 
5504  // Apply VMA_DEBUG_MARGIN at the beginning.
5505  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5506  {
5507  *pOffset += VMA_DEBUG_MARGIN;
5508  }
5509 
5510  // Apply alignment.
5511  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5512  *pOffset = VmaAlignUp(*pOffset, alignment);
5513 
5514  // Check previous suballocations for BufferImageGranularity conflicts.
5515  // Make bigger alignment if necessary.
5516  if(bufferImageGranularity > 1)
5517  {
5518  bool bufferImageGranularityConflict = false;
5519  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5520  while(prevSuballocItem != m_Suballocations.cbegin())
5521  {
5522  --prevSuballocItem;
5523  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5524  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5525  {
5526  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5527  {
5528  bufferImageGranularityConflict = true;
5529  break;
5530  }
5531  }
5532  else
5533  // Already on previous page.
5534  break;
5535  }
5536  if(bufferImageGranularityConflict)
5537  {
5538  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5539  }
5540  }
5541 
5542  // Now that we have final *pOffset, check if we are past suballocItem.
5543  // If yes, return false - this function should be called for another suballocItem as starting point.
5544  if(*pOffset >= suballocItem->offset + suballocItem->size)
5545  {
5546  return false;
5547  }
5548 
5549  // Calculate padding at the beginning based on current offset.
5550  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5551 
5552  // Calculate required margin at the end if this is not last suballocation.
5553  VmaSuballocationList::const_iterator next = suballocItem;
5554  ++next;
5555  const VkDeviceSize requiredEndMargin =
5556  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5557 
5558  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5559  // Another early return check.
5560  if(suballocItem->offset + totalSize > m_Size)
5561  {
5562  return false;
5563  }
5564 
5565  // Advance lastSuballocItem until desired size is reached.
5566  // Update itemsToMakeLostCount.
5567  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5568  if(totalSize > suballocItem->size)
5569  {
5570  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5571  while(remainingSize > 0)
5572  {
5573  ++lastSuballocItem;
5574  if(lastSuballocItem == m_Suballocations.cend())
5575  {
5576  return false;
5577  }
5578  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5579  {
5580  *pSumFreeSize += lastSuballocItem->size;
5581  }
5582  else
5583  {
5584  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5585  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5586  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5587  {
5588  ++*itemsToMakeLostCount;
5589  *pSumItemSize += lastSuballocItem->size;
5590  }
5591  else
5592  {
5593  return false;
5594  }
5595  }
5596  remainingSize = (lastSuballocItem->size < remainingSize) ?
5597  remainingSize - lastSuballocItem->size : 0;
5598  }
5599  }
5600 
5601  // Check next suballocations for BufferImageGranularity conflicts.
5602  // If conflict exists, we must mark more allocations lost or fail.
5603  if(bufferImageGranularity > 1)
5604  {
5605  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5606  ++nextSuballocItem;
5607  while(nextSuballocItem != m_Suballocations.cend())
5608  {
5609  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5610  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5611  {
5612  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5613  {
5614  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5615  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5616  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5617  {
5618  ++*itemsToMakeLostCount;
5619  }
5620  else
5621  {
5622  return false;
5623  }
5624  }
5625  }
5626  else
5627  {
5628  // Already on next page.
5629  break;
5630  }
5631  ++nextSuballocItem;
5632  }
5633  }
5634  }
5635  else
5636  {
5637  const VmaSuballocation& suballoc = *suballocItem;
5638  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5639 
5640  *pSumFreeSize = suballoc.size;
5641 
5642  // Size of this suballocation is too small for this request: Early return.
5643  if(suballoc.size < allocSize)
5644  {
5645  return false;
5646  }
5647 
5648  // Start from offset equal to beginning of this suballocation.
5649  *pOffset = suballoc.offset;
5650 
5651  // Apply VMA_DEBUG_MARGIN at the beginning.
5652  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5653  {
5654  *pOffset += VMA_DEBUG_MARGIN;
5655  }
5656 
5657  // Apply alignment.
5658  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5659  *pOffset = VmaAlignUp(*pOffset, alignment);
5660 
5661  // Check previous suballocations for BufferImageGranularity conflicts.
5662  // Make bigger alignment if necessary.
5663  if(bufferImageGranularity > 1)
5664  {
5665  bool bufferImageGranularityConflict = false;
5666  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5667  while(prevSuballocItem != m_Suballocations.cbegin())
5668  {
5669  --prevSuballocItem;
5670  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5671  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5672  {
5673  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5674  {
5675  bufferImageGranularityConflict = true;
5676  break;
5677  }
5678  }
5679  else
5680  // Already on previous page.
5681  break;
5682  }
5683  if(bufferImageGranularityConflict)
5684  {
5685  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5686  }
5687  }
5688 
5689  // Calculate padding at the beginning based on current offset.
5690  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5691 
5692  // Calculate required margin at the end if this is not last suballocation.
5693  VmaSuballocationList::const_iterator next = suballocItem;
5694  ++next;
5695  const VkDeviceSize requiredEndMargin =
5696  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5697 
5698  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5699  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5700  {
5701  return false;
5702  }
5703 
5704  // Check next suballocations for BufferImageGranularity conflicts.
5705  // If conflict exists, allocation cannot be made here.
5706  if(bufferImageGranularity > 1)
5707  {
5708  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5709  ++nextSuballocItem;
5710  while(nextSuballocItem != m_Suballocations.cend())
5711  {
5712  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5713  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5714  {
5715  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5716  {
5717  return false;
5718  }
5719  }
5720  else
5721  {
5722  // Already on next page.
5723  break;
5724  }
5725  ++nextSuballocItem;
5726  }
5727  }
5728  }
5729 
5730  // All tests passed: Success. pOffset is already filled.
5731  return true;
5732 }
5733 
5734 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5735 {
5736  VMA_ASSERT(item != m_Suballocations.end());
5737  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5738 
5739  VmaSuballocationList::iterator nextItem = item;
5740  ++nextItem;
5741  VMA_ASSERT(nextItem != m_Suballocations.end());
5742  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5743 
5744  item->size += nextItem->size;
5745  --m_FreeCount;
5746  m_Suballocations.erase(nextItem);
5747 }
5748 
5749 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5750 {
5751  // Change this suballocation to be marked as free.
5752  VmaSuballocation& suballoc = *suballocItem;
5753  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5754  suballoc.hAllocation = VK_NULL_HANDLE;
5755 
5756  // Update totals.
5757  ++m_FreeCount;
5758  m_SumFreeSize += suballoc.size;
5759 
5760  // Merge with previous and/or next suballocation if it's also free.
5761  bool mergeWithNext = false;
5762  bool mergeWithPrev = false;
5763 
5764  VmaSuballocationList::iterator nextItem = suballocItem;
5765  ++nextItem;
5766  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5767  {
5768  mergeWithNext = true;
5769  }
5770 
5771  VmaSuballocationList::iterator prevItem = suballocItem;
5772  if(suballocItem != m_Suballocations.begin())
5773  {
5774  --prevItem;
5775  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5776  {
5777  mergeWithPrev = true;
5778  }
5779  }
5780 
5781  if(mergeWithNext)
5782  {
5783  UnregisterFreeSuballocation(nextItem);
5784  MergeFreeWithNext(suballocItem);
5785  }
5786 
5787  if(mergeWithPrev)
5788  {
5789  UnregisterFreeSuballocation(prevItem);
5790  MergeFreeWithNext(prevItem);
5791  RegisterFreeSuballocation(prevItem);
5792  return prevItem;
5793  }
5794  else
5795  {
5796  RegisterFreeSuballocation(suballocItem);
5797  return suballocItem;
5798  }
5799 }
5800 
5801 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5802 {
5803  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5804  VMA_ASSERT(item->size > 0);
5805 
5806  // You may want to enable this validation at the beginning or at the end of
5807  // this function, depending on what do you want to check.
5808  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5809 
5810  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5811  {
5812  if(m_FreeSuballocationsBySize.empty())
5813  {
5814  m_FreeSuballocationsBySize.push_back(item);
5815  }
5816  else
5817  {
5818  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5819  }
5820  }
5821 
5822  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5823 }
5824 
5825 
5826 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5827 {
5828  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5829  VMA_ASSERT(item->size > 0);
5830 
5831  // You may want to enable this validation at the beginning or at the end of
5832  // this function, depending on what do you want to check.
5833  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5834 
5835  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5836  {
5837  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5838  m_FreeSuballocationsBySize.data(),
5839  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5840  item,
5841  VmaSuballocationItemSizeLess());
5842  for(size_t index = it - m_FreeSuballocationsBySize.data();
5843  index < m_FreeSuballocationsBySize.size();
5844  ++index)
5845  {
5846  if(m_FreeSuballocationsBySize[index] == item)
5847  {
5848  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5849  return;
5850  }
5851  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5852  }
5853  VMA_ASSERT(0 && "Not found.");
5854  }
5855 
5856  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5857 }
5858 
5860 // class VmaDeviceMemoryMapping
5861 
5862 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
5863  m_MapCount(0),
5864  m_pMappedData(VMA_NULL)
5865 {
5866 }
5867 
5868 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
5869 {
5870  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
5871 }
5872 
5873 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
5874 {
5875  if(count == 0)
5876  {
5877  return VK_SUCCESS;
5878  }
5879 
5880  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5881  if(m_MapCount != 0)
5882  {
5883  m_MapCount += count;
5884  VMA_ASSERT(m_pMappedData != VMA_NULL);
5885  if(ppData != VMA_NULL)
5886  {
5887  *ppData = m_pMappedData;
5888  }
5889  return VK_SUCCESS;
5890  }
5891  else
5892  {
5893  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5894  hAllocator->m_hDevice,
5895  hMemory,
5896  0, // offset
5897  VK_WHOLE_SIZE,
5898  0, // flags
5899  &m_pMappedData);
5900  if(result == VK_SUCCESS)
5901  {
5902  if(ppData != VMA_NULL)
5903  {
5904  *ppData = m_pMappedData;
5905  }
5906  m_MapCount = count;
5907  }
5908  return result;
5909  }
5910 }
5911 
5912 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
5913 {
5914  if(count == 0)
5915  {
5916  return;
5917  }
5918 
5919  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5920  if(m_MapCount >= count)
5921  {
5922  m_MapCount -= count;
5923  if(m_MapCount == 0)
5924  {
5925  m_pMappedData = VMA_NULL;
5926  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
5927  }
5928  }
5929  else
5930  {
5931  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
5932  }
5933 }
5934 
5936 // class VmaDeviceMemoryBlock
5937 
5938 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5939  m_MemoryTypeIndex(UINT32_MAX),
5940  m_hMemory(VK_NULL_HANDLE),
5941  m_Metadata(hAllocator)
5942 {
5943 }
5944 
5945 void VmaDeviceMemoryBlock::Init(
5946  uint32_t newMemoryTypeIndex,
5947  VkDeviceMemory newMemory,
5948  VkDeviceSize newSize)
5949 {
5950  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5951 
5952  m_MemoryTypeIndex = newMemoryTypeIndex;
5953  m_hMemory = newMemory;
5954 
5955  m_Metadata.Init(newSize);
5956 }
5957 
5958 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5959 {
5960  // This is the most important assert in the entire library.
5961  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5962  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5963 
5964  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5965  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5966  m_hMemory = VK_NULL_HANDLE;
5967 }
5968 
5969 bool VmaDeviceMemoryBlock::Validate() const
5970 {
5971  if((m_hMemory == VK_NULL_HANDLE) ||
5972  (m_Metadata.GetSize() == 0))
5973  {
5974  return false;
5975  }
5976 
5977  return m_Metadata.Validate();
5978 }
5979 
5980 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
5981 {
5982  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
5983 }
5984 
5985 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
5986 {
5987  m_Mapping.Unmap(hAllocator, m_hMemory, count);
5988 }
5989 
5990 static void InitStatInfo(VmaStatInfo& outInfo)
5991 {
5992  memset(&outInfo, 0, sizeof(outInfo));
5993  outInfo.allocationSizeMin = UINT64_MAX;
5994  outInfo.unusedRangeSizeMin = UINT64_MAX;
5995 }
5996 
5997 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5998 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5999 {
6000  inoutInfo.blockCount += srcInfo.blockCount;
6001  inoutInfo.allocationCount += srcInfo.allocationCount;
6002  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6003  inoutInfo.usedBytes += srcInfo.usedBytes;
6004  inoutInfo.unusedBytes += srcInfo.unusedBytes;
6005  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6006  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6007  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6008  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6009 }
6010 
6011 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6012 {
6013  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6014  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6015  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6016  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6017 }
6018 
6019 VmaPool_T::VmaPool_T(
6020  VmaAllocator hAllocator,
6021  const VmaPoolCreateInfo& createInfo) :
6022  m_BlockVector(
6023  hAllocator,
6024  createInfo.memoryTypeIndex,
6025  createInfo.blockSize,
6026  createInfo.minBlockCount,
6027  createInfo.maxBlockCount,
6028  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6029  createInfo.frameInUseCount,
6030  true) // isCustomPool
6031 {
6032 }
6033 
6034 VmaPool_T::~VmaPool_T()
6035 {
6036 }
6037 
6038 #if VMA_STATS_STRING_ENABLED
6039 
6040 #endif // #if VMA_STATS_STRING_ENABLED
6041 
6042 VmaBlockVector::VmaBlockVector(
6043  VmaAllocator hAllocator,
6044  uint32_t memoryTypeIndex,
6045  VkDeviceSize preferredBlockSize,
6046  size_t minBlockCount,
6047  size_t maxBlockCount,
6048  VkDeviceSize bufferImageGranularity,
6049  uint32_t frameInUseCount,
6050  bool isCustomPool) :
6051  m_hAllocator(hAllocator),
6052  m_MemoryTypeIndex(memoryTypeIndex),
6053  m_PreferredBlockSize(preferredBlockSize),
6054  m_MinBlockCount(minBlockCount),
6055  m_MaxBlockCount(maxBlockCount),
6056  m_BufferImageGranularity(bufferImageGranularity),
6057  m_FrameInUseCount(frameInUseCount),
6058  m_IsCustomPool(isCustomPool),
6059  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6060  m_HasEmptyBlock(false),
6061  m_pDefragmentator(VMA_NULL)
6062 {
6063 }
6064 
6065 VmaBlockVector::~VmaBlockVector()
6066 {
6067  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6068 
6069  for(size_t i = m_Blocks.size(); i--; )
6070  {
6071  m_Blocks[i]->Destroy(m_hAllocator);
6072  vma_delete(m_hAllocator, m_Blocks[i]);
6073  }
6074 }
6075 
6076 VkResult VmaBlockVector::CreateMinBlocks()
6077 {
6078  for(size_t i = 0; i < m_MinBlockCount; ++i)
6079  {
6080  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6081  if(res != VK_SUCCESS)
6082  {
6083  return res;
6084  }
6085  }
6086  return VK_SUCCESS;
6087 }
6088 
6089 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6090 {
6091  pStats->size = 0;
6092  pStats->unusedSize = 0;
6093  pStats->allocationCount = 0;
6094  pStats->unusedRangeCount = 0;
6095  pStats->unusedRangeSizeMax = 0;
6096 
6097  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6098 
6099  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6100  {
6101  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6102  VMA_ASSERT(pBlock);
6103  VMA_HEAVY_ASSERT(pBlock->Validate());
6104  pBlock->m_Metadata.AddPoolStats(*pStats);
6105  }
6106 }
6107 
6108 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6109 
6110 VkResult VmaBlockVector::Allocate(
6111  VmaPool hCurrentPool,
6112  uint32_t currentFrameIndex,
6113  const VkMemoryRequirements& vkMemReq,
6114  const VmaAllocationCreateInfo& createInfo,
6115  VmaSuballocationType suballocType,
6116  VmaAllocation* pAllocation)
6117 {
6118  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6119  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6120 
6121  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6122 
6123  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6124  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6125  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6126  {
6127  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6128  VMA_ASSERT(pCurrBlock);
6129  VmaAllocationRequest currRequest = {};
6130  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6131  currentFrameIndex,
6132  m_FrameInUseCount,
6133  m_BufferImageGranularity,
6134  vkMemReq.size,
6135  vkMemReq.alignment,
6136  suballocType,
6137  false, // canMakeOtherLost
6138  &currRequest))
6139  {
6140  // Allocate from pCurrBlock.
6141  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6142 
6143  if(mapped)
6144  {
6145  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6146  if(res != VK_SUCCESS)
6147  {
6148  return res;
6149  }
6150  }
6151 
6152  // We no longer have an empty Allocation.
6153  if(pCurrBlock->m_Metadata.IsEmpty())
6154  {
6155  m_HasEmptyBlock = false;
6156  }
6157 
6158  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6159  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6160  (*pAllocation)->InitBlockAllocation(
6161  hCurrentPool,
6162  pCurrBlock,
6163  currRequest.offset,
6164  vkMemReq.alignment,
6165  vkMemReq.size,
6166  suballocType,
6167  mapped,
6168  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6169  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6170  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6171  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6172  return VK_SUCCESS;
6173  }
6174  }
6175 
6176  const bool canCreateNewBlock =
6177  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6178  (m_Blocks.size() < m_MaxBlockCount);
6179 
6180  // 2. Try to create new block.
6181  if(canCreateNewBlock)
6182  {
6183  // Calculate optimal size for new block.
6184  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6185  uint32_t newBlockSizeShift = 0;
6186  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6187 
6188  // Allocating blocks of other sizes is allowed only in default pools.
6189  // In custom pools block size is fixed.
6190  if(m_IsCustomPool == false)
6191  {
6192  // Allocate 1/8, 1/4, 1/2 as first blocks.
6193  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6194  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6195  {
6196  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6197  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6198  {
6199  newBlockSize = smallerNewBlockSize;
6200  ++newBlockSizeShift;
6201  }
6202  else
6203  {
6204  break;
6205  }
6206  }
6207  }
6208 
6209  size_t newBlockIndex = 0;
6210  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6211  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6212  if(m_IsCustomPool == false)
6213  {
6214  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6215  {
6216  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6217  if(smallerNewBlockSize >= vkMemReq.size)
6218  {
6219  newBlockSize = smallerNewBlockSize;
6220  ++newBlockSizeShift;
6221  res = CreateBlock(newBlockSize, &newBlockIndex);
6222  }
6223  else
6224  {
6225  break;
6226  }
6227  }
6228  }
6229 
6230  if(res == VK_SUCCESS)
6231  {
6232  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6233  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6234 
6235  if(mapped)
6236  {
6237  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6238  if(res != VK_SUCCESS)
6239  {
6240  return res;
6241  }
6242  }
6243 
6244  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6245  VmaAllocationRequest allocRequest;
6246  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6247  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6248  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6249  (*pAllocation)->InitBlockAllocation(
6250  hCurrentPool,
6251  pBlock,
6252  allocRequest.offset,
6253  vkMemReq.alignment,
6254  vkMemReq.size,
6255  suballocType,
6256  mapped,
6257  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6258  VMA_HEAVY_ASSERT(pBlock->Validate());
6259  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6260  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6261  return VK_SUCCESS;
6262  }
6263  }
6264 
6265  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6266 
6267  // 3. Try to allocate from existing blocks with making other allocations lost.
6268  if(canMakeOtherLost)
6269  {
6270  uint32_t tryIndex = 0;
6271  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6272  {
6273  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6274  VmaAllocationRequest bestRequest = {};
6275  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6276 
6277  // 1. Search existing allocations.
6278  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6279  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6280  {
6281  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6282  VMA_ASSERT(pCurrBlock);
6283  VmaAllocationRequest currRequest = {};
6284  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6285  currentFrameIndex,
6286  m_FrameInUseCount,
6287  m_BufferImageGranularity,
6288  vkMemReq.size,
6289  vkMemReq.alignment,
6290  suballocType,
6291  canMakeOtherLost,
6292  &currRequest))
6293  {
6294  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6295  if(pBestRequestBlock == VMA_NULL ||
6296  currRequestCost < bestRequestCost)
6297  {
6298  pBestRequestBlock = pCurrBlock;
6299  bestRequest = currRequest;
6300  bestRequestCost = currRequestCost;
6301 
6302  if(bestRequestCost == 0)
6303  {
6304  break;
6305  }
6306  }
6307  }
6308  }
6309 
6310  if(pBestRequestBlock != VMA_NULL)
6311  {
6312  if(mapped)
6313  {
6314  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6315  if(res != VK_SUCCESS)
6316  {
6317  return res;
6318  }
6319  }
6320 
6321  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6322  currentFrameIndex,
6323  m_FrameInUseCount,
6324  &bestRequest))
6325  {
6326  // We no longer have an empty Allocation.
6327  if(pBestRequestBlock->m_Metadata.IsEmpty())
6328  {
6329  m_HasEmptyBlock = false;
6330  }
6331  // Allocate from this pBlock.
6332  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6333  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6334  (*pAllocation)->InitBlockAllocation(
6335  hCurrentPool,
6336  pBestRequestBlock,
6337  bestRequest.offset,
6338  vkMemReq.alignment,
6339  vkMemReq.size,
6340  suballocType,
6341  mapped,
6342  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6343  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6344  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6345  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6346  return VK_SUCCESS;
6347  }
6348  // else: Some allocations must have been touched while we are here. Next try.
6349  }
6350  else
6351  {
6352  // Could not find place in any of the blocks - break outer loop.
6353  break;
6354  }
6355  }
6356  /* Maximum number of tries exceeded - a very unlike event when many other
6357  threads are simultaneously touching allocations making it impossible to make
6358  lost at the same time as we try to allocate. */
6359  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6360  {
6361  return VK_ERROR_TOO_MANY_OBJECTS;
6362  }
6363  }
6364 
6365  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6366 }
6367 
6368 void VmaBlockVector::Free(
6369  VmaAllocation hAllocation)
6370 {
6371  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6372 
6373  // Scope for lock.
6374  {
6375  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6376 
6377  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6378 
6379  if(hAllocation->IsPersistentMap())
6380  {
6381  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6382  }
6383 
6384  pBlock->m_Metadata.Free(hAllocation);
6385  VMA_HEAVY_ASSERT(pBlock->Validate());
6386 
6387  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6388 
6389  // pBlock became empty after this deallocation.
6390  if(pBlock->m_Metadata.IsEmpty())
6391  {
6392  // Already has empty Allocation. We don't want to have two, so delete this one.
6393  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6394  {
6395  pBlockToDelete = pBlock;
6396  Remove(pBlock);
6397  }
6398  // We now have first empty Allocation.
6399  else
6400  {
6401  m_HasEmptyBlock = true;
6402  }
6403  }
6404  // pBlock didn't become empty, but we have another empty block - find and free that one.
6405  // (This is optional, heuristics.)
6406  else if(m_HasEmptyBlock)
6407  {
6408  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6409  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6410  {
6411  pBlockToDelete = pLastBlock;
6412  m_Blocks.pop_back();
6413  m_HasEmptyBlock = false;
6414  }
6415  }
6416 
6417  IncrementallySortBlocks();
6418  }
6419 
6420  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6421  // lock, for performance reason.
6422  if(pBlockToDelete != VMA_NULL)
6423  {
6424  VMA_DEBUG_LOG(" Deleted empty allocation");
6425  pBlockToDelete->Destroy(m_hAllocator);
6426  vma_delete(m_hAllocator, pBlockToDelete);
6427  }
6428 }
6429 
6430 size_t VmaBlockVector::CalcMaxBlockSize() const
6431 {
6432  size_t result = 0;
6433  for(size_t i = m_Blocks.size(); i--; )
6434  {
6435  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6436  if(result >= m_PreferredBlockSize)
6437  {
6438  break;
6439  }
6440  }
6441  return result;
6442 }
6443 
6444 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6445 {
6446  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6447  {
6448  if(m_Blocks[blockIndex] == pBlock)
6449  {
6450  VmaVectorRemove(m_Blocks, blockIndex);
6451  return;
6452  }
6453  }
6454  VMA_ASSERT(0);
6455 }
6456 
6457 void VmaBlockVector::IncrementallySortBlocks()
6458 {
6459  // Bubble sort only until first swap.
6460  for(size_t i = 1; i < m_Blocks.size(); ++i)
6461  {
6462  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6463  {
6464  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6465  return;
6466  }
6467  }
6468 }
6469 
6470 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6471 {
6472  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6473  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6474  allocInfo.allocationSize = blockSize;
6475  VkDeviceMemory mem = VK_NULL_HANDLE;
6476  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6477  if(res < 0)
6478  {
6479  return res;
6480  }
6481 
6482  // New VkDeviceMemory successfully created.
6483 
6484  // Create new Allocation for it.
6485  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6486  pBlock->Init(
6487  m_MemoryTypeIndex,
6488  mem,
6489  allocInfo.allocationSize);
6490 
6491  m_Blocks.push_back(pBlock);
6492  if(pNewBlockIndex != VMA_NULL)
6493  {
6494  *pNewBlockIndex = m_Blocks.size() - 1;
6495  }
6496 
6497  return VK_SUCCESS;
6498 }
6499 
6500 #if VMA_STATS_STRING_ENABLED
6501 
6502 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6503 {
6504  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6505 
6506  json.BeginObject();
6507 
6508  if(m_IsCustomPool)
6509  {
6510  json.WriteString("MemoryTypeIndex");
6511  json.WriteNumber(m_MemoryTypeIndex);
6512 
6513  json.WriteString("BlockSize");
6514  json.WriteNumber(m_PreferredBlockSize);
6515 
6516  json.WriteString("BlockCount");
6517  json.BeginObject(true);
6518  if(m_MinBlockCount > 0)
6519  {
6520  json.WriteString("Min");
6521  json.WriteNumber((uint64_t)m_MinBlockCount);
6522  }
6523  if(m_MaxBlockCount < SIZE_MAX)
6524  {
6525  json.WriteString("Max");
6526  json.WriteNumber((uint64_t)m_MaxBlockCount);
6527  }
6528  json.WriteString("Cur");
6529  json.WriteNumber((uint64_t)m_Blocks.size());
6530  json.EndObject();
6531 
6532  if(m_FrameInUseCount > 0)
6533  {
6534  json.WriteString("FrameInUseCount");
6535  json.WriteNumber(m_FrameInUseCount);
6536  }
6537  }
6538  else
6539  {
6540  json.WriteString("PreferredBlockSize");
6541  json.WriteNumber(m_PreferredBlockSize);
6542  }
6543 
6544  json.WriteString("Blocks");
6545  json.BeginArray();
6546  for(size_t i = 0; i < m_Blocks.size(); ++i)
6547  {
6548  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6549  }
6550  json.EndArray();
6551 
6552  json.EndObject();
6553 }
6554 
6555 #endif // #if VMA_STATS_STRING_ENABLED
6556 
6557 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6558  VmaAllocator hAllocator,
6559  uint32_t currentFrameIndex)
6560 {
6561  if(m_pDefragmentator == VMA_NULL)
6562  {
6563  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6564  hAllocator,
6565  this,
6566  currentFrameIndex);
6567  }
6568 
6569  return m_pDefragmentator;
6570 }
6571 
6572 VkResult VmaBlockVector::Defragment(
6573  VmaDefragmentationStats* pDefragmentationStats,
6574  VkDeviceSize& maxBytesToMove,
6575  uint32_t& maxAllocationsToMove)
6576 {
6577  if(m_pDefragmentator == VMA_NULL)
6578  {
6579  return VK_SUCCESS;
6580  }
6581 
6582  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6583 
6584  // Defragment.
6585  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6586 
6587  // Accumulate statistics.
6588  if(pDefragmentationStats != VMA_NULL)
6589  {
6590  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6591  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6592  pDefragmentationStats->bytesMoved += bytesMoved;
6593  pDefragmentationStats->allocationsMoved += allocationsMoved;
6594  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6595  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6596  maxBytesToMove -= bytesMoved;
6597  maxAllocationsToMove -= allocationsMoved;
6598  }
6599 
6600  // Free empty blocks.
6601  m_HasEmptyBlock = false;
6602  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6603  {
6604  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6605  if(pBlock->m_Metadata.IsEmpty())
6606  {
6607  if(m_Blocks.size() > m_MinBlockCount)
6608  {
6609  if(pDefragmentationStats != VMA_NULL)
6610  {
6611  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6612  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6613  }
6614 
6615  VmaVectorRemove(m_Blocks, blockIndex);
6616  pBlock->Destroy(m_hAllocator);
6617  vma_delete(m_hAllocator, pBlock);
6618  }
6619  else
6620  {
6621  m_HasEmptyBlock = true;
6622  }
6623  }
6624  }
6625 
6626  return result;
6627 }
6628 
6629 void VmaBlockVector::DestroyDefragmentator()
6630 {
6631  if(m_pDefragmentator != VMA_NULL)
6632  {
6633  vma_delete(m_hAllocator, m_pDefragmentator);
6634  m_pDefragmentator = VMA_NULL;
6635  }
6636 }
6637 
6638 void VmaBlockVector::MakePoolAllocationsLost(
6639  uint32_t currentFrameIndex,
6640  size_t* pLostAllocationCount)
6641 {
6642  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6643  size_t lostAllocationCount = 0;
6644  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6645  {
6646  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6647  VMA_ASSERT(pBlock);
6648  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6649  }
6650  if(pLostAllocationCount != VMA_NULL)
6651  {
6652  *pLostAllocationCount = lostAllocationCount;
6653  }
6654 }
6655 
6656 void VmaBlockVector::AddStats(VmaStats* pStats)
6657 {
6658  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6659  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6660 
6661  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6662 
6663  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6664  {
6665  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6666  VMA_ASSERT(pBlock);
6667  VMA_HEAVY_ASSERT(pBlock->Validate());
6668  VmaStatInfo allocationStatInfo;
6669  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6670  VmaAddStatInfo(pStats->total, allocationStatInfo);
6671  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6672  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6673  }
6674 }
6675 
6677 // VmaDefragmentator members definition
6678 
6679 VmaDefragmentator::VmaDefragmentator(
6680  VmaAllocator hAllocator,
6681  VmaBlockVector* pBlockVector,
6682  uint32_t currentFrameIndex) :
6683  m_hAllocator(hAllocator),
6684  m_pBlockVector(pBlockVector),
6685  m_CurrentFrameIndex(currentFrameIndex),
6686  m_BytesMoved(0),
6687  m_AllocationsMoved(0),
6688  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6689  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6690 {
6691 }
6692 
6693 VmaDefragmentator::~VmaDefragmentator()
6694 {
6695  for(size_t i = m_Blocks.size(); i--; )
6696  {
6697  vma_delete(m_hAllocator, m_Blocks[i]);
6698  }
6699 }
6700 
6701 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6702 {
6703  AllocationInfo allocInfo;
6704  allocInfo.m_hAllocation = hAlloc;
6705  allocInfo.m_pChanged = pChanged;
6706  m_Allocations.push_back(allocInfo);
6707 }
6708 
6709 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6710 {
6711  // It has already been mapped for defragmentation.
6712  if(m_pMappedDataForDefragmentation)
6713  {
6714  *ppMappedData = m_pMappedDataForDefragmentation;
6715  return VK_SUCCESS;
6716  }
6717 
6718  // It is originally mapped.
6719  if(m_pBlock->m_Mapping.GetMappedData())
6720  {
6721  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6722  return VK_SUCCESS;
6723  }
6724 
6725  // Map on first usage.
6726  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6727  *ppMappedData = m_pMappedDataForDefragmentation;
6728  return res;
6729 }
6730 
6731 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6732 {
6733  if(m_pMappedDataForDefragmentation != VMA_NULL)
6734  {
6735  m_pBlock->Unmap(hAllocator, 1);
6736  }
6737 }
6738 
6739 VkResult VmaDefragmentator::DefragmentRound(
6740  VkDeviceSize maxBytesToMove,
6741  uint32_t maxAllocationsToMove)
6742 {
6743  if(m_Blocks.empty())
6744  {
6745  return VK_SUCCESS;
6746  }
6747 
6748  size_t srcBlockIndex = m_Blocks.size() - 1;
6749  size_t srcAllocIndex = SIZE_MAX;
6750  for(;;)
6751  {
6752  // 1. Find next allocation to move.
6753  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6754  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6755  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6756  {
6757  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6758  {
6759  // Finished: no more allocations to process.
6760  if(srcBlockIndex == 0)
6761  {
6762  return VK_SUCCESS;
6763  }
6764  else
6765  {
6766  --srcBlockIndex;
6767  srcAllocIndex = SIZE_MAX;
6768  }
6769  }
6770  else
6771  {
6772  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6773  }
6774  }
6775 
6776  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6777  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6778 
6779  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6780  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6781  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6782  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6783 
6784  // 2. Try to find new place for this allocation in preceding or current block.
6785  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6786  {
6787  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6788  VmaAllocationRequest dstAllocRequest;
6789  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6790  m_CurrentFrameIndex,
6791  m_pBlockVector->GetFrameInUseCount(),
6792  m_pBlockVector->GetBufferImageGranularity(),
6793  size,
6794  alignment,
6795  suballocType,
6796  false, // canMakeOtherLost
6797  &dstAllocRequest) &&
6798  MoveMakesSense(
6799  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6800  {
6801  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6802 
6803  // Reached limit on number of allocations or bytes to move.
6804  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6805  (m_BytesMoved + size > maxBytesToMove))
6806  {
6807  return VK_INCOMPLETE;
6808  }
6809 
6810  void* pDstMappedData = VMA_NULL;
6811  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6812  if(res != VK_SUCCESS)
6813  {
6814  return res;
6815  }
6816 
6817  void* pSrcMappedData = VMA_NULL;
6818  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6819  if(res != VK_SUCCESS)
6820  {
6821  return res;
6822  }
6823 
6824  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6825  memcpy(
6826  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6827  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6828  static_cast<size_t>(size));
6829 
6830  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6831  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
6832 
6833  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6834 
6835  if(allocInfo.m_pChanged != VMA_NULL)
6836  {
6837  *allocInfo.m_pChanged = VK_TRUE;
6838  }
6839 
6840  ++m_AllocationsMoved;
6841  m_BytesMoved += size;
6842 
6843  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6844 
6845  break;
6846  }
6847  }
6848 
6849  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6850 
6851  if(srcAllocIndex > 0)
6852  {
6853  --srcAllocIndex;
6854  }
6855  else
6856  {
6857  if(srcBlockIndex > 0)
6858  {
6859  --srcBlockIndex;
6860  srcAllocIndex = SIZE_MAX;
6861  }
6862  else
6863  {
6864  return VK_SUCCESS;
6865  }
6866  }
6867  }
6868 }
6869 
6870 VkResult VmaDefragmentator::Defragment(
6871  VkDeviceSize maxBytesToMove,
6872  uint32_t maxAllocationsToMove)
6873 {
6874  if(m_Allocations.empty())
6875  {
6876  return VK_SUCCESS;
6877  }
6878 
6879  // Create block info for each block.
6880  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6881  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6882  {
6883  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6884  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6885  m_Blocks.push_back(pBlockInfo);
6886  }
6887 
6888  // Sort them by m_pBlock pointer value.
6889  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6890 
6891  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6892  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6893  {
6894  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6895  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6896  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6897  {
6898  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6899  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6900  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6901  {
6902  (*it)->m_Allocations.push_back(allocInfo);
6903  }
6904  else
6905  {
6906  VMA_ASSERT(0);
6907  }
6908  }
6909  }
6910  m_Allocations.clear();
6911 
6912  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6913  {
6914  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6915  pBlockInfo->CalcHasNonMovableAllocations();
6916  pBlockInfo->SortAllocationsBySizeDescecnding();
6917  }
6918 
6919  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6920  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6921 
6922  // Execute defragmentation rounds (the main part).
6923  VkResult result = VK_SUCCESS;
6924  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6925  {
6926  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6927  }
6928 
6929  // Unmap blocks that were mapped for defragmentation.
6930  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6931  {
6932  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6933  }
6934 
6935  return result;
6936 }
6937 
6938 bool VmaDefragmentator::MoveMakesSense(
6939  size_t dstBlockIndex, VkDeviceSize dstOffset,
6940  size_t srcBlockIndex, VkDeviceSize srcOffset)
6941 {
6942  if(dstBlockIndex < srcBlockIndex)
6943  {
6944  return true;
6945  }
6946  if(dstBlockIndex > srcBlockIndex)
6947  {
6948  return false;
6949  }
6950  if(dstOffset < srcOffset)
6951  {
6952  return true;
6953  }
6954  return false;
6955 }
6956 
6958 // VmaAllocator_T
6959 
6960 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6961  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6962  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
6963  m_hDevice(pCreateInfo->device),
6964  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6965  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6966  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6967  m_PreferredLargeHeapBlockSize(0),
6968  m_PhysicalDevice(pCreateInfo->physicalDevice),
6969  m_CurrentFrameIndex(0),
6970  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6971 {
6972  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6973 
6974  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6975  memset(&m_MemProps, 0, sizeof(m_MemProps));
6976  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6977 
6978  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6979  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
6980 
6981  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6982  {
6983  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6984  }
6985 
6986  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6987  {
6988  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6989  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6990  }
6991 
6992  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6993 
6994  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6995  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6996 
6997  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6998  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6999 
7000  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7001  {
7002  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7003  {
7004  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7005  if(limit != VK_WHOLE_SIZE)
7006  {
7007  m_HeapSizeLimit[heapIndex] = limit;
7008  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7009  {
7010  m_MemProps.memoryHeaps[heapIndex].size = limit;
7011  }
7012  }
7013  }
7014  }
7015 
7016  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7017  {
7018  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7019 
7020  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7021  this,
7022  memTypeIndex,
7023  preferredBlockSize,
7024  0,
7025  SIZE_MAX,
7026  GetBufferImageGranularity(),
7027  pCreateInfo->frameInUseCount,
7028  false); // isCustomPool
7029  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7030  // becase minBlockCount is 0.
7031  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7032  }
7033 }
7034 
7035 VmaAllocator_T::~VmaAllocator_T()
7036 {
7037  VMA_ASSERT(m_Pools.empty());
7038 
7039  for(size_t i = GetMemoryTypeCount(); i--; )
7040  {
7041  vma_delete(this, m_pDedicatedAllocations[i]);
7042  vma_delete(this, m_pBlockVectors[i]);
7043  }
7044 }
7045 
7046 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7047 {
7048 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7049  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7050  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7051  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7052  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7053  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7054  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7055  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7056  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7057  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7058  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7059  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7060  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7061  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7062  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7063  if(m_UseKhrDedicatedAllocation)
7064  {
7065  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7066  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7067  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7068  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7069  }
7070 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7071 
7072 #define VMA_COPY_IF_NOT_NULL(funcName) \
7073  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7074 
7075  if(pVulkanFunctions != VMA_NULL)
7076  {
7077  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7078  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7079  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7080  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7081  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7082  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7083  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7084  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7085  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7086  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7087  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7088  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7089  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7090  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7091  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7092  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7093  }
7094 
7095 #undef VMA_COPY_IF_NOT_NULL
7096 
7097  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7098  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7099  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7100  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7101  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7102  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7103  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7104  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7105  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7106  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7107  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7108  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7109  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7110  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7111  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7112  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7113  if(m_UseKhrDedicatedAllocation)
7114  {
7115  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7116  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7117  }
7118 }
7119 
7120 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7121 {
7122  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7123  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7124  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7125  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7126 }
7127 
7128 VkResult VmaAllocator_T::AllocateMemoryOfType(
7129  const VkMemoryRequirements& vkMemReq,
7130  bool dedicatedAllocation,
7131  VkBuffer dedicatedBuffer,
7132  VkImage dedicatedImage,
7133  const VmaAllocationCreateInfo& createInfo,
7134  uint32_t memTypeIndex,
7135  VmaSuballocationType suballocType,
7136  VmaAllocation* pAllocation)
7137 {
7138  VMA_ASSERT(pAllocation != VMA_NULL);
7139  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7140 
7141  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7142 
7143  // If memory type is not HOST_VISIBLE, disable MAPPED.
7144  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7145  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7146  {
7147  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7148  }
7149 
7150  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7151  VMA_ASSERT(blockVector);
7152 
7153  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7154  bool preferDedicatedMemory =
7155  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7156  dedicatedAllocation ||
7157  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7158  vkMemReq.size > preferredBlockSize / 2;
7159 
7160  if(preferDedicatedMemory &&
7161  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7162  finalCreateInfo.pool == VK_NULL_HANDLE)
7163  {
7164  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7165  }
7166 
7167  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7168  {
7169  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7170  {
7171  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7172  }
7173  else
7174  {
7175  return AllocateDedicatedMemory(
7176  vkMemReq.size,
7177  suballocType,
7178  memTypeIndex,
7179  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7180  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7181  finalCreateInfo.pUserData,
7182  dedicatedBuffer,
7183  dedicatedImage,
7184  pAllocation);
7185  }
7186  }
7187  else
7188  {
7189  VkResult res = blockVector->Allocate(
7190  VK_NULL_HANDLE, // hCurrentPool
7191  m_CurrentFrameIndex.load(),
7192  vkMemReq,
7193  finalCreateInfo,
7194  suballocType,
7195  pAllocation);
7196  if(res == VK_SUCCESS)
7197  {
7198  return res;
7199  }
7200 
7201  // 5. Try dedicated memory.
7202  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7203  {
7204  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7205  }
7206  else
7207  {
7208  res = AllocateDedicatedMemory(
7209  vkMemReq.size,
7210  suballocType,
7211  memTypeIndex,
7212  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7213  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7214  finalCreateInfo.pUserData,
7215  dedicatedBuffer,
7216  dedicatedImage,
7217  pAllocation);
7218  if(res == VK_SUCCESS)
7219  {
7220  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7221  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7222  return VK_SUCCESS;
7223  }
7224  else
7225  {
7226  // Everything failed: Return error code.
7227  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7228  return res;
7229  }
7230  }
7231  }
7232 }
7233 
7234 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7235  VkDeviceSize size,
7236  VmaSuballocationType suballocType,
7237  uint32_t memTypeIndex,
7238  bool map,
7239  bool isUserDataString,
7240  void* pUserData,
7241  VkBuffer dedicatedBuffer,
7242  VkImage dedicatedImage,
7243  VmaAllocation* pAllocation)
7244 {
7245  VMA_ASSERT(pAllocation);
7246 
7247  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7248  allocInfo.memoryTypeIndex = memTypeIndex;
7249  allocInfo.allocationSize = size;
7250 
7251  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7252  if(m_UseKhrDedicatedAllocation)
7253  {
7254  if(dedicatedBuffer != VK_NULL_HANDLE)
7255  {
7256  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7257  dedicatedAllocInfo.buffer = dedicatedBuffer;
7258  allocInfo.pNext = &dedicatedAllocInfo;
7259  }
7260  else if(dedicatedImage != VK_NULL_HANDLE)
7261  {
7262  dedicatedAllocInfo.image = dedicatedImage;
7263  allocInfo.pNext = &dedicatedAllocInfo;
7264  }
7265  }
7266 
7267  // Allocate VkDeviceMemory.
7268  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7269  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7270  if(res < 0)
7271  {
7272  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7273  return res;
7274  }
7275 
7276  void* pMappedData = VMA_NULL;
7277  if(map)
7278  {
7279  res = (*m_VulkanFunctions.vkMapMemory)(
7280  m_hDevice,
7281  hMemory,
7282  0,
7283  VK_WHOLE_SIZE,
7284  0,
7285  &pMappedData);
7286  if(res < 0)
7287  {
7288  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7289  FreeVulkanMemory(memTypeIndex, size, hMemory);
7290  return res;
7291  }
7292  }
7293 
7294  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7295  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7296  (*pAllocation)->SetUserData(this, pUserData);
7297 
7298  // Register it in m_pDedicatedAllocations.
7299  {
7300  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7301  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7302  VMA_ASSERT(pDedicatedAllocations);
7303  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7304  }
7305 
7306  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7307 
7308  return VK_SUCCESS;
7309 }
7310 
7311 void VmaAllocator_T::GetBufferMemoryRequirements(
7312  VkBuffer hBuffer,
7313  VkMemoryRequirements& memReq,
7314  bool& requiresDedicatedAllocation,
7315  bool& prefersDedicatedAllocation) const
7316 {
7317  if(m_UseKhrDedicatedAllocation)
7318  {
7319  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7320  memReqInfo.buffer = hBuffer;
7321 
7322  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7323 
7324  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7325  memReq2.pNext = &memDedicatedReq;
7326 
7327  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7328 
7329  memReq = memReq2.memoryRequirements;
7330  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7331  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7332  }
7333  else
7334  {
7335  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7336  requiresDedicatedAllocation = false;
7337  prefersDedicatedAllocation = false;
7338  }
7339 }
7340 
7341 void VmaAllocator_T::GetImageMemoryRequirements(
7342  VkImage hImage,
7343  VkMemoryRequirements& memReq,
7344  bool& requiresDedicatedAllocation,
7345  bool& prefersDedicatedAllocation) const
7346 {
7347  if(m_UseKhrDedicatedAllocation)
7348  {
7349  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7350  memReqInfo.image = hImage;
7351 
7352  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7353 
7354  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7355  memReq2.pNext = &memDedicatedReq;
7356 
7357  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7358 
7359  memReq = memReq2.memoryRequirements;
7360  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7361  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7362  }
7363  else
7364  {
7365  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7366  requiresDedicatedAllocation = false;
7367  prefersDedicatedAllocation = false;
7368  }
7369 }
7370 
7371 VkResult VmaAllocator_T::AllocateMemory(
7372  const VkMemoryRequirements& vkMemReq,
7373  bool requiresDedicatedAllocation,
7374  bool prefersDedicatedAllocation,
7375  VkBuffer dedicatedBuffer,
7376  VkImage dedicatedImage,
7377  const VmaAllocationCreateInfo& createInfo,
7378  VmaSuballocationType suballocType,
7379  VmaAllocation* pAllocation)
7380 {
7381  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7382  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7383  {
7384  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7385  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7386  }
7387  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7388  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7389  {
7390  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7391  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7392  }
7393  if(requiresDedicatedAllocation)
7394  {
7395  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7396  {
7397  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7398  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7399  }
7400  if(createInfo.pool != VK_NULL_HANDLE)
7401  {
7402  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7403  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7404  }
7405  }
7406  if((createInfo.pool != VK_NULL_HANDLE) &&
7407  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7408  {
7409  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7410  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7411  }
7412 
7413  if(createInfo.pool != VK_NULL_HANDLE)
7414  {
7415  return createInfo.pool->m_BlockVector.Allocate(
7416  createInfo.pool,
7417  m_CurrentFrameIndex.load(),
7418  vkMemReq,
7419  createInfo,
7420  suballocType,
7421  pAllocation);
7422  }
7423  else
7424  {
7425  // Bit mask of memory Vulkan types acceptable for this allocation.
7426  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7427  uint32_t memTypeIndex = UINT32_MAX;
7428  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7429  if(res == VK_SUCCESS)
7430  {
7431  res = AllocateMemoryOfType(
7432  vkMemReq,
7433  requiresDedicatedAllocation || prefersDedicatedAllocation,
7434  dedicatedBuffer,
7435  dedicatedImage,
7436  createInfo,
7437  memTypeIndex,
7438  suballocType,
7439  pAllocation);
7440  // Succeeded on first try.
7441  if(res == VK_SUCCESS)
7442  {
7443  return res;
7444  }
7445  // Allocation from this memory type failed. Try other compatible memory types.
7446  else
7447  {
7448  for(;;)
7449  {
7450  // Remove old memTypeIndex from list of possibilities.
7451  memoryTypeBits &= ~(1u << memTypeIndex);
7452  // Find alternative memTypeIndex.
7453  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7454  if(res == VK_SUCCESS)
7455  {
7456  res = AllocateMemoryOfType(
7457  vkMemReq,
7458  requiresDedicatedAllocation || prefersDedicatedAllocation,
7459  dedicatedBuffer,
7460  dedicatedImage,
7461  createInfo,
7462  memTypeIndex,
7463  suballocType,
7464  pAllocation);
7465  // Allocation from this alternative memory type succeeded.
7466  if(res == VK_SUCCESS)
7467  {
7468  return res;
7469  }
7470  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7471  }
7472  // No other matching memory type index could be found.
7473  else
7474  {
7475  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7476  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7477  }
7478  }
7479  }
7480  }
7481  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7482  else
7483  return res;
7484  }
7485 }
7486 
7487 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7488 {
7489  VMA_ASSERT(allocation);
7490 
7491  if(allocation->CanBecomeLost() == false ||
7492  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7493  {
7494  switch(allocation->GetType())
7495  {
7496  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7497  {
7498  VmaBlockVector* pBlockVector = VMA_NULL;
7499  VmaPool hPool = allocation->GetPool();
7500  if(hPool != VK_NULL_HANDLE)
7501  {
7502  pBlockVector = &hPool->m_BlockVector;
7503  }
7504  else
7505  {
7506  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7507  pBlockVector = m_pBlockVectors[memTypeIndex];
7508  }
7509  pBlockVector->Free(allocation);
7510  }
7511  break;
7512  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7513  FreeDedicatedMemory(allocation);
7514  break;
7515  default:
7516  VMA_ASSERT(0);
7517  }
7518  }
7519 
7520  allocation->SetUserData(this, VMA_NULL);
7521  vma_delete(this, allocation);
7522 }
7523 
7524 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7525 {
7526  // Initialize.
7527  InitStatInfo(pStats->total);
7528  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7529  InitStatInfo(pStats->memoryType[i]);
7530  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7531  InitStatInfo(pStats->memoryHeap[i]);
7532 
7533  // Process default pools.
7534  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7535  {
7536  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7537  VMA_ASSERT(pBlockVector);
7538  pBlockVector->AddStats(pStats);
7539  }
7540 
7541  // Process custom pools.
7542  {
7543  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7544  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7545  {
7546  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7547  }
7548  }
7549 
7550  // Process dedicated allocations.
7551  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7552  {
7553  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7554  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7555  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7556  VMA_ASSERT(pDedicatedAllocVector);
7557  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7558  {
7559  VmaStatInfo allocationStatInfo;
7560  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7561  VmaAddStatInfo(pStats->total, allocationStatInfo);
7562  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7563  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7564  }
7565  }
7566 
7567  // Postprocess.
7568  VmaPostprocessCalcStatInfo(pStats->total);
7569  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7570  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7571  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7572  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7573 }
7574 
7575 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7576 
7577 VkResult VmaAllocator_T::Defragment(
7578  VmaAllocation* pAllocations,
7579  size_t allocationCount,
7580  VkBool32* pAllocationsChanged,
7581  const VmaDefragmentationInfo* pDefragmentationInfo,
7582  VmaDefragmentationStats* pDefragmentationStats)
7583 {
7584  if(pAllocationsChanged != VMA_NULL)
7585  {
7586  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7587  }
7588  if(pDefragmentationStats != VMA_NULL)
7589  {
7590  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7591  }
7592 
7593  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7594 
7595  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7596 
7597  const size_t poolCount = m_Pools.size();
7598 
7599  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7600  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7601  {
7602  VmaAllocation hAlloc = pAllocations[allocIndex];
7603  VMA_ASSERT(hAlloc);
7604  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7605  // DedicatedAlloc cannot be defragmented.
7606  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7607  // Only HOST_VISIBLE memory types can be defragmented.
7608  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7609  // Lost allocation cannot be defragmented.
7610  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7611  {
7612  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7613 
7614  const VmaPool hAllocPool = hAlloc->GetPool();
7615  // This allocation belongs to custom pool.
7616  if(hAllocPool != VK_NULL_HANDLE)
7617  {
7618  pAllocBlockVector = &hAllocPool->GetBlockVector();
7619  }
7620  // This allocation belongs to general pool.
7621  else
7622  {
7623  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7624  }
7625 
7626  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7627 
7628  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7629  &pAllocationsChanged[allocIndex] : VMA_NULL;
7630  pDefragmentator->AddAllocation(hAlloc, pChanged);
7631  }
7632  }
7633 
7634  VkResult result = VK_SUCCESS;
7635 
7636  // ======== Main processing.
7637 
7638  VkDeviceSize maxBytesToMove = SIZE_MAX;
7639  uint32_t maxAllocationsToMove = UINT32_MAX;
7640  if(pDefragmentationInfo != VMA_NULL)
7641  {
7642  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7643  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7644  }
7645 
7646  // Process standard memory.
7647  for(uint32_t memTypeIndex = 0;
7648  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7649  ++memTypeIndex)
7650  {
7651  // Only HOST_VISIBLE memory types can be defragmented.
7652  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7653  {
7654  result = m_pBlockVectors[memTypeIndex]->Defragment(
7655  pDefragmentationStats,
7656  maxBytesToMove,
7657  maxAllocationsToMove);
7658  }
7659  }
7660 
7661  // Process custom pools.
7662  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7663  {
7664  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7665  pDefragmentationStats,
7666  maxBytesToMove,
7667  maxAllocationsToMove);
7668  }
7669 
7670  // ======== Destroy defragmentators.
7671 
7672  // Process custom pools.
7673  for(size_t poolIndex = poolCount; poolIndex--; )
7674  {
7675  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7676  }
7677 
7678  // Process standard memory.
7679  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7680  {
7681  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7682  {
7683  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7684  }
7685  }
7686 
7687  return result;
7688 }
7689 
7690 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7691 {
7692  if(hAllocation->CanBecomeLost())
7693  {
7694  /*
7695  Warning: This is a carefully designed algorithm.
7696  Do not modify unless you really know what you're doing :)
7697  */
7698  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7699  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7700  for(;;)
7701  {
7702  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7703  {
7704  pAllocationInfo->memoryType = UINT32_MAX;
7705  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7706  pAllocationInfo->offset = 0;
7707  pAllocationInfo->size = hAllocation->GetSize();
7708  pAllocationInfo->pMappedData = VMA_NULL;
7709  pAllocationInfo->pUserData = hAllocation->GetUserData();
7710  return;
7711  }
7712  else if(localLastUseFrameIndex == localCurrFrameIndex)
7713  {
7714  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7715  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7716  pAllocationInfo->offset = hAllocation->GetOffset();
7717  pAllocationInfo->size = hAllocation->GetSize();
7718  pAllocationInfo->pMappedData = VMA_NULL;
7719  pAllocationInfo->pUserData = hAllocation->GetUserData();
7720  return;
7721  }
7722  else // Last use time earlier than current time.
7723  {
7724  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7725  {
7726  localLastUseFrameIndex = localCurrFrameIndex;
7727  }
7728  }
7729  }
7730  }
7731  else
7732  {
7733  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7734  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7735  pAllocationInfo->offset = hAllocation->GetOffset();
7736  pAllocationInfo->size = hAllocation->GetSize();
7737  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7738  pAllocationInfo->pUserData = hAllocation->GetUserData();
7739  }
7740 }
7741 
7742 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7743 {
7744  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7745 
7746  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7747 
7748  if(newCreateInfo.maxBlockCount == 0)
7749  {
7750  newCreateInfo.maxBlockCount = SIZE_MAX;
7751  }
7752  if(newCreateInfo.blockSize == 0)
7753  {
7754  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7755  }
7756 
7757  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7758 
7759  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7760  if(res != VK_SUCCESS)
7761  {
7762  vma_delete(this, *pPool);
7763  *pPool = VMA_NULL;
7764  return res;
7765  }
7766 
7767  // Add to m_Pools.
7768  {
7769  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7770  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7771  }
7772 
7773  return VK_SUCCESS;
7774 }
7775 
7776 void VmaAllocator_T::DestroyPool(VmaPool pool)
7777 {
7778  // Remove from m_Pools.
7779  {
7780  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7781  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
7782  VMA_ASSERT(success && "Pool not found in Allocator.");
7783  }
7784 
7785  vma_delete(this, pool);
7786 }
7787 
7788 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
7789 {
7790  pool->m_BlockVector.GetPoolStats(pPoolStats);
7791 }
7792 
7793 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
7794 {
7795  m_CurrentFrameIndex.store(frameIndex);
7796 }
7797 
7798 void VmaAllocator_T::MakePoolAllocationsLost(
7799  VmaPool hPool,
7800  size_t* pLostAllocationCount)
7801 {
7802  hPool->m_BlockVector.MakePoolAllocationsLost(
7803  m_CurrentFrameIndex.load(),
7804  pLostAllocationCount);
7805 }
7806 
7807 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
7808 {
7809  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
7810  (*pAllocation)->InitLost();
7811 }
7812 
7813 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
7814 {
7815  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
7816 
7817  VkResult res;
7818  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7819  {
7820  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7821  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
7822  {
7823  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7824  if(res == VK_SUCCESS)
7825  {
7826  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
7827  }
7828  }
7829  else
7830  {
7831  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
7832  }
7833  }
7834  else
7835  {
7836  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7837  }
7838 
7839  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
7840  {
7841  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
7842  }
7843 
7844  return res;
7845 }
7846 
7847 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
7848 {
7849  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
7850  {
7851  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
7852  }
7853 
7854  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7855 
7856  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7857  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7858  {
7859  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7860  m_HeapSizeLimit[heapIndex] += size;
7861  }
7862 }
7863 
7864 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
7865 {
7866  if(hAllocation->CanBecomeLost())
7867  {
7868  return VK_ERROR_MEMORY_MAP_FAILED;
7869  }
7870 
7871  switch(hAllocation->GetType())
7872  {
7873  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7874  {
7875  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7876  char *pBytes = VMA_NULL;
7877  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
7878  if(res == VK_SUCCESS)
7879  {
7880  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
7881  hAllocation->BlockAllocMap();
7882  }
7883  return res;
7884  }
7885  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7886  return hAllocation->DedicatedAllocMap(this, ppData);
7887  default:
7888  VMA_ASSERT(0);
7889  return VK_ERROR_MEMORY_MAP_FAILED;
7890  }
7891 }
7892 
7893 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
7894 {
7895  switch(hAllocation->GetType())
7896  {
7897  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7898  {
7899  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7900  hAllocation->BlockAllocUnmap();
7901  pBlock->Unmap(this, 1);
7902  }
7903  break;
7904  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7905  hAllocation->DedicatedAllocUnmap(this);
7906  break;
7907  default:
7908  VMA_ASSERT(0);
7909  }
7910 }
7911 
7912 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
7913 {
7914  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
7915 
7916  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7917  {
7918  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7919  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7920  VMA_ASSERT(pDedicatedAllocations);
7921  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
7922  VMA_ASSERT(success);
7923  }
7924 
7925  VkDeviceMemory hMemory = allocation->GetMemory();
7926 
7927  if(allocation->GetMappedData() != VMA_NULL)
7928  {
7929  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7930  }
7931 
7932  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7933 
7934  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
7935 }
7936 
7937 #if VMA_STATS_STRING_ENABLED
7938 
7939 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7940 {
7941  bool dedicatedAllocationsStarted = false;
7942  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7943  {
7944  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7945  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7946  VMA_ASSERT(pDedicatedAllocVector);
7947  if(pDedicatedAllocVector->empty() == false)
7948  {
7949  if(dedicatedAllocationsStarted == false)
7950  {
7951  dedicatedAllocationsStarted = true;
7952  json.WriteString("DedicatedAllocations");
7953  json.BeginObject();
7954  }
7955 
7956  json.BeginString("Type ");
7957  json.ContinueString(memTypeIndex);
7958  json.EndString();
7959 
7960  json.BeginArray();
7961 
7962  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
7963  {
7964  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
7965  json.BeginObject(true);
7966 
7967  json.WriteString("Type");
7968  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7969 
7970  json.WriteString("Size");
7971  json.WriteNumber(hAlloc->GetSize());
7972 
7973  const void* pUserData = hAlloc->GetUserData();
7974  if(pUserData != VMA_NULL)
7975  {
7976  json.WriteString("UserData");
7977  if(hAlloc->IsUserDataString())
7978  {
7979  json.WriteString((const char*)pUserData);
7980  }
7981  else
7982  {
7983  json.BeginString();
7984  json.ContinueString_Pointer(pUserData);
7985  json.EndString();
7986  }
7987  }
7988 
7989  json.EndObject();
7990  }
7991 
7992  json.EndArray();
7993  }
7994  }
7995  if(dedicatedAllocationsStarted)
7996  {
7997  json.EndObject();
7998  }
7999 
8000  {
8001  bool allocationsStarted = false;
8002  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8003  {
8004  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8005  {
8006  if(allocationsStarted == false)
8007  {
8008  allocationsStarted = true;
8009  json.WriteString("DefaultPools");
8010  json.BeginObject();
8011  }
8012 
8013  json.BeginString("Type ");
8014  json.ContinueString(memTypeIndex);
8015  json.EndString();
8016 
8017  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8018  }
8019  }
8020  if(allocationsStarted)
8021  {
8022  json.EndObject();
8023  }
8024  }
8025 
8026  {
8027  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8028  const size_t poolCount = m_Pools.size();
8029  if(poolCount > 0)
8030  {
8031  json.WriteString("Pools");
8032  json.BeginArray();
8033  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8034  {
8035  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8036  }
8037  json.EndArray();
8038  }
8039  }
8040 }
8041 
8042 #endif // #if VMA_STATS_STRING_ENABLED
8043 
8044 static VkResult AllocateMemoryForImage(
8045  VmaAllocator allocator,
8046  VkImage image,
8047  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8048  VmaSuballocationType suballocType,
8049  VmaAllocation* pAllocation)
8050 {
8051  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8052 
8053  VkMemoryRequirements vkMemReq = {};
8054  bool requiresDedicatedAllocation = false;
8055  bool prefersDedicatedAllocation = false;
8056  allocator->GetImageMemoryRequirements(image, vkMemReq,
8057  requiresDedicatedAllocation, prefersDedicatedAllocation);
8058 
8059  return allocator->AllocateMemory(
8060  vkMemReq,
8061  requiresDedicatedAllocation,
8062  prefersDedicatedAllocation,
8063  VK_NULL_HANDLE, // dedicatedBuffer
8064  image, // dedicatedImage
8065  *pAllocationCreateInfo,
8066  suballocType,
8067  pAllocation);
8068 }
8069 
8071 // Public interface
8072 
8073 VkResult vmaCreateAllocator(
8074  const VmaAllocatorCreateInfo* pCreateInfo,
8075  VmaAllocator* pAllocator)
8076 {
8077  VMA_ASSERT(pCreateInfo && pAllocator);
8078  VMA_DEBUG_LOG("vmaCreateAllocator");
8079  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8080  return VK_SUCCESS;
8081 }
8082 
8083 void vmaDestroyAllocator(
8084  VmaAllocator allocator)
8085 {
8086  if(allocator != VK_NULL_HANDLE)
8087  {
8088  VMA_DEBUG_LOG("vmaDestroyAllocator");
8089  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8090  vma_delete(&allocationCallbacks, allocator);
8091  }
8092 }
8093 
8094 void vmaGetPhysicalDeviceProperties(
8095  VmaAllocator allocator,
8096  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8097 {
8098  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8099  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8100 }
8101 
8102 void vmaGetMemoryProperties(
8103  VmaAllocator allocator,
8104  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8105 {
8106  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8107  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8108 }
8109 
8110 void vmaGetMemoryTypeProperties(
8111  VmaAllocator allocator,
8112  uint32_t memoryTypeIndex,
8113  VkMemoryPropertyFlags* pFlags)
8114 {
8115  VMA_ASSERT(allocator && pFlags);
8116  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8117  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8118 }
8119 
8120 void vmaSetCurrentFrameIndex(
8121  VmaAllocator allocator,
8122  uint32_t frameIndex)
8123 {
8124  VMA_ASSERT(allocator);
8125  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8126 
8127  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8128 
8129  allocator->SetCurrentFrameIndex(frameIndex);
8130 }
8131 
8132 void vmaCalculateStats(
8133  VmaAllocator allocator,
8134  VmaStats* pStats)
8135 {
8136  VMA_ASSERT(allocator && pStats);
8137  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8138  allocator->CalculateStats(pStats);
8139 }
8140 
8141 #if VMA_STATS_STRING_ENABLED
8142 
8143 void vmaBuildStatsString(
8144  VmaAllocator allocator,
8145  char** ppStatsString,
8146  VkBool32 detailedMap)
8147 {
8148  VMA_ASSERT(allocator && ppStatsString);
8149  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8150 
8151  VmaStringBuilder sb(allocator);
8152  {
8153  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8154  json.BeginObject();
8155 
8156  VmaStats stats;
8157  allocator->CalculateStats(&stats);
8158 
8159  json.WriteString("Total");
8160  VmaPrintStatInfo(json, stats.total);
8161 
8162  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8163  {
8164  json.BeginString("Heap ");
8165  json.ContinueString(heapIndex);
8166  json.EndString();
8167  json.BeginObject();
8168 
8169  json.WriteString("Size");
8170  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8171 
8172  json.WriteString("Flags");
8173  json.BeginArray(true);
8174  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8175  {
8176  json.WriteString("DEVICE_LOCAL");
8177  }
8178  json.EndArray();
8179 
8180  if(stats.memoryHeap[heapIndex].blockCount > 0)
8181  {
8182  json.WriteString("Stats");
8183  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8184  }
8185 
8186  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8187  {
8188  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8189  {
8190  json.BeginString("Type ");
8191  json.ContinueString(typeIndex);
8192  json.EndString();
8193 
8194  json.BeginObject();
8195 
8196  json.WriteString("Flags");
8197  json.BeginArray(true);
8198  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8199  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8200  {
8201  json.WriteString("DEVICE_LOCAL");
8202  }
8203  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8204  {
8205  json.WriteString("HOST_VISIBLE");
8206  }
8207  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8208  {
8209  json.WriteString("HOST_COHERENT");
8210  }
8211  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8212  {
8213  json.WriteString("HOST_CACHED");
8214  }
8215  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8216  {
8217  json.WriteString("LAZILY_ALLOCATED");
8218  }
8219  json.EndArray();
8220 
8221  if(stats.memoryType[typeIndex].blockCount > 0)
8222  {
8223  json.WriteString("Stats");
8224  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8225  }
8226 
8227  json.EndObject();
8228  }
8229  }
8230 
8231  json.EndObject();
8232  }
8233  if(detailedMap == VK_TRUE)
8234  {
8235  allocator->PrintDetailedMap(json);
8236  }
8237 
8238  json.EndObject();
8239  }
8240 
8241  const size_t len = sb.GetLength();
8242  char* const pChars = vma_new_array(allocator, char, len + 1);
8243  if(len > 0)
8244  {
8245  memcpy(pChars, sb.GetData(), len);
8246  }
8247  pChars[len] = '\0';
8248  *ppStatsString = pChars;
8249 }
8250 
8251 void vmaFreeStatsString(
8252  VmaAllocator allocator,
8253  char* pStatsString)
8254 {
8255  if(pStatsString != VMA_NULL)
8256  {
8257  VMA_ASSERT(allocator);
8258  size_t len = strlen(pStatsString);
8259  vma_delete_array(allocator, pStatsString, len + 1);
8260  }
8261 }
8262 
8263 #endif // #if VMA_STATS_STRING_ENABLED
8264 
8265 /*
8266 This function is not protected by any mutex because it just reads immutable data.
8267 */
8268 VkResult vmaFindMemoryTypeIndex(
8269  VmaAllocator allocator,
8270  uint32_t memoryTypeBits,
8271  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8272  uint32_t* pMemoryTypeIndex)
8273 {
8274  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8275  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8276  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8277 
8278  if(pAllocationCreateInfo->memoryTypeBits != 0)
8279  {
8280  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8281  }
8282 
8283  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8284  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8285 
8286  // Convert usage to requiredFlags and preferredFlags.
8287  switch(pAllocationCreateInfo->usage)
8288  {
8289  case VMA_MEMORY_USAGE_UNKNOWN:
8290  break;
8291  case VMA_MEMORY_USAGE_GPU_ONLY:
8292  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8293  break;
8294  case VMA_MEMORY_USAGE_CPU_ONLY:
8295  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8296  break;
8297  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8298  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8299  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8300  break;
8301  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8302  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8303  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8304  break;
8305  default:
8306  break;
8307  }
8308 
8309  *pMemoryTypeIndex = UINT32_MAX;
8310  uint32_t minCost = UINT32_MAX;
8311  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8312  memTypeIndex < allocator->GetMemoryTypeCount();
8313  ++memTypeIndex, memTypeBit <<= 1)
8314  {
8315  // This memory type is acceptable according to memoryTypeBits bitmask.
8316  if((memTypeBit & memoryTypeBits) != 0)
8317  {
8318  const VkMemoryPropertyFlags currFlags =
8319  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8320  // This memory type contains requiredFlags.
8321  if((requiredFlags & ~currFlags) == 0)
8322  {
8323  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8324  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8325  // Remember memory type with lowest cost.
8326  if(currCost < minCost)
8327  {
8328  *pMemoryTypeIndex = memTypeIndex;
8329  if(currCost == 0)
8330  {
8331  return VK_SUCCESS;
8332  }
8333  minCost = currCost;
8334  }
8335  }
8336  }
8337  }
8338  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8339 }
8340 
8341 VkResult vmaCreatePool(
8342  VmaAllocator allocator,
8343  const VmaPoolCreateInfo* pCreateInfo,
8344  VmaPool* pPool)
8345 {
8346  VMA_ASSERT(allocator && pCreateInfo && pPool);
8347 
8348  VMA_DEBUG_LOG("vmaCreatePool");
8349 
8350  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8351 
8352  return allocator->CreatePool(pCreateInfo, pPool);
8353 }
8354 
8355 void vmaDestroyPool(
8356  VmaAllocator allocator,
8357  VmaPool pool)
8358 {
8359  VMA_ASSERT(allocator);
8360 
8361  if(pool == VK_NULL_HANDLE)
8362  {
8363  return;
8364  }
8365 
8366  VMA_DEBUG_LOG("vmaDestroyPool");
8367 
8368  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8369 
8370  allocator->DestroyPool(pool);
8371 }
8372 
8373 void vmaGetPoolStats(
8374  VmaAllocator allocator,
8375  VmaPool pool,
8376  VmaPoolStats* pPoolStats)
8377 {
8378  VMA_ASSERT(allocator && pool && pPoolStats);
8379 
8380  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8381 
8382  allocator->GetPoolStats(pool, pPoolStats);
8383 }
8384 
8385 void vmaMakePoolAllocationsLost(
8386  VmaAllocator allocator,
8387  VmaPool pool,
8388  size_t* pLostAllocationCount)
8389 {
8390  VMA_ASSERT(allocator && pool);
8391 
8392  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8393 
8394  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8395 }
8396 
8397 VkResult vmaAllocateMemory(
8398  VmaAllocator allocator,
8399  const VkMemoryRequirements* pVkMemoryRequirements,
8400  const VmaAllocationCreateInfo* pCreateInfo,
8401  VmaAllocation* pAllocation,
8402  VmaAllocationInfo* pAllocationInfo)
8403 {
8404  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8405 
8406  VMA_DEBUG_LOG("vmaAllocateMemory");
8407 
8408  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8409 
8410  VkResult result = allocator->AllocateMemory(
8411  *pVkMemoryRequirements,
8412  false, // requiresDedicatedAllocation
8413  false, // prefersDedicatedAllocation
8414  VK_NULL_HANDLE, // dedicatedBuffer
8415  VK_NULL_HANDLE, // dedicatedImage
8416  *pCreateInfo,
8417  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8418  pAllocation);
8419 
8420  if(pAllocationInfo && result == VK_SUCCESS)
8421  {
8422  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8423  }
8424 
8425  return result;
8426 }
8427 
8428 VkResult vmaAllocateMemoryForBuffer(
8429  VmaAllocator allocator,
8430  VkBuffer buffer,
8431  const VmaAllocationCreateInfo* pCreateInfo,
8432  VmaAllocation* pAllocation,
8433  VmaAllocationInfo* pAllocationInfo)
8434 {
8435  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8436 
8437  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8438 
8439  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8440 
8441  VkMemoryRequirements vkMemReq = {};
8442  bool requiresDedicatedAllocation = false;
8443  bool prefersDedicatedAllocation = false;
8444  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8445  requiresDedicatedAllocation,
8446  prefersDedicatedAllocation);
8447 
8448  VkResult result = allocator->AllocateMemory(
8449  vkMemReq,
8450  requiresDedicatedAllocation,
8451  prefersDedicatedAllocation,
8452  buffer, // dedicatedBuffer
8453  VK_NULL_HANDLE, // dedicatedImage
8454  *pCreateInfo,
8455  VMA_SUBALLOCATION_TYPE_BUFFER,
8456  pAllocation);
8457 
8458  if(pAllocationInfo && result == VK_SUCCESS)
8459  {
8460  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8461  }
8462 
8463  return result;
8464 }
8465 
8466 VkResult vmaAllocateMemoryForImage(
8467  VmaAllocator allocator,
8468  VkImage image,
8469  const VmaAllocationCreateInfo* pCreateInfo,
8470  VmaAllocation* pAllocation,
8471  VmaAllocationInfo* pAllocationInfo)
8472 {
8473  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8474 
8475  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8476 
8477  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8478 
8479  VkResult result = AllocateMemoryForImage(
8480  allocator,
8481  image,
8482  pCreateInfo,
8483  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8484  pAllocation);
8485 
8486  if(pAllocationInfo && result == VK_SUCCESS)
8487  {
8488  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8489  }
8490 
8491  return result;
8492 }
8493 
8494 void vmaFreeMemory(
8495  VmaAllocator allocator,
8496  VmaAllocation allocation)
8497 {
8498  VMA_ASSERT(allocator && allocation);
8499 
8500  VMA_DEBUG_LOG("vmaFreeMemory");
8501 
8502  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8503 
8504  allocator->FreeMemory(allocation);
8505 }
8506 
8507 void vmaGetAllocationInfo(
8508  VmaAllocator allocator,
8509  VmaAllocation allocation,
8510  VmaAllocationInfo* pAllocationInfo)
8511 {
8512  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8513 
8514  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8515 
8516  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8517 }
8518 
8519 void vmaSetAllocationUserData(
8520  VmaAllocator allocator,
8521  VmaAllocation allocation,
8522  void* pUserData)
8523 {
8524  VMA_ASSERT(allocator && allocation);
8525 
8526  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8527 
8528  allocation->SetUserData(allocator, pUserData);
8529 }
8530 
8531 void vmaCreateLostAllocation(
8532  VmaAllocator allocator,
8533  VmaAllocation* pAllocation)
8534 {
8535  VMA_ASSERT(allocator && pAllocation);
8536 
8537  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8538 
8539  allocator->CreateLostAllocation(pAllocation);
8540 }
8541 
8542 VkResult vmaMapMemory(
8543  VmaAllocator allocator,
8544  VmaAllocation allocation,
8545  void** ppData)
8546 {
8547  VMA_ASSERT(allocator && allocation && ppData);
8548 
8549  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8550 
8551  return allocator->Map(allocation, ppData);
8552 }
8553 
8554 void vmaUnmapMemory(
8555  VmaAllocator allocator,
8556  VmaAllocation allocation)
8557 {
8558  VMA_ASSERT(allocator && allocation);
8559 
8560  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8561 
8562  allocator->Unmap(allocation);
8563 }
8564 
8565 VkResult vmaDefragment(
8566  VmaAllocator allocator,
8567  VmaAllocation* pAllocations,
8568  size_t allocationCount,
8569  VkBool32* pAllocationsChanged,
8570  const VmaDefragmentationInfo *pDefragmentationInfo,
8571  VmaDefragmentationStats* pDefragmentationStats)
8572 {
8573  VMA_ASSERT(allocator && pAllocations);
8574 
8575  VMA_DEBUG_LOG("vmaDefragment");
8576 
8577  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8578 
8579  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8580 }
8581 
8582 VkResult vmaCreateBuffer(
8583  VmaAllocator allocator,
8584  const VkBufferCreateInfo* pBufferCreateInfo,
8585  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8586  VkBuffer* pBuffer,
8587  VmaAllocation* pAllocation,
8588  VmaAllocationInfo* pAllocationInfo)
8589 {
8590  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8591 
8592  VMA_DEBUG_LOG("vmaCreateBuffer");
8593 
8594  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8595 
8596  *pBuffer = VK_NULL_HANDLE;
8597  *pAllocation = VK_NULL_HANDLE;
8598 
8599  // 1. Create VkBuffer.
8600  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8601  allocator->m_hDevice,
8602  pBufferCreateInfo,
8603  allocator->GetAllocationCallbacks(),
8604  pBuffer);
8605  if(res >= 0)
8606  {
8607  // 2. vkGetBufferMemoryRequirements.
8608  VkMemoryRequirements vkMemReq = {};
8609  bool requiresDedicatedAllocation = false;
8610  bool prefersDedicatedAllocation = false;
8611  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8612  requiresDedicatedAllocation, prefersDedicatedAllocation);
8613 
8614  // Make sure alignment requirements for specific buffer usages reported
8615  // in Physical Device Properties are included in alignment reported by memory requirements.
8616  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8617  {
8618  VMA_ASSERT(vkMemReq.alignment %
8619  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8620  }
8621  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8622  {
8623  VMA_ASSERT(vkMemReq.alignment %
8624  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8625  }
8626  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8627  {
8628  VMA_ASSERT(vkMemReq.alignment %
8629  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8630  }
8631 
8632  // 3. Allocate memory using allocator.
8633  res = allocator->AllocateMemory(
8634  vkMemReq,
8635  requiresDedicatedAllocation,
8636  prefersDedicatedAllocation,
8637  *pBuffer, // dedicatedBuffer
8638  VK_NULL_HANDLE, // dedicatedImage
8639  *pAllocationCreateInfo,
8640  VMA_SUBALLOCATION_TYPE_BUFFER,
8641  pAllocation);
8642  if(res >= 0)
8643  {
8644  // 3. Bind buffer with memory.
8645  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8646  allocator->m_hDevice,
8647  *pBuffer,
8648  (*pAllocation)->GetMemory(),
8649  (*pAllocation)->GetOffset());
8650  if(res >= 0)
8651  {
8652  // All steps succeeded.
8653  if(pAllocationInfo != VMA_NULL)
8654  {
8655  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8656  }
8657  return VK_SUCCESS;
8658  }
8659  allocator->FreeMemory(*pAllocation);
8660  *pAllocation = VK_NULL_HANDLE;
8661  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8662  *pBuffer = VK_NULL_HANDLE;
8663  return res;
8664  }
8665  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8666  *pBuffer = VK_NULL_HANDLE;
8667  return res;
8668  }
8669  return res;
8670 }
8671 
8672 void vmaDestroyBuffer(
8673  VmaAllocator allocator,
8674  VkBuffer buffer,
8675  VmaAllocation allocation)
8676 {
8677  if(buffer != VK_NULL_HANDLE)
8678  {
8679  VMA_ASSERT(allocator);
8680 
8681  VMA_DEBUG_LOG("vmaDestroyBuffer");
8682 
8683  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8684 
8685  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8686 
8687  allocator->FreeMemory(allocation);
8688  }
8689 }
8690 
8691 VkResult vmaCreateImage(
8692  VmaAllocator allocator,
8693  const VkImageCreateInfo* pImageCreateInfo,
8694  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8695  VkImage* pImage,
8696  VmaAllocation* pAllocation,
8697  VmaAllocationInfo* pAllocationInfo)
8698 {
8699  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
8700 
8701  VMA_DEBUG_LOG("vmaCreateImage");
8702 
8703  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8704 
8705  *pImage = VK_NULL_HANDLE;
8706  *pAllocation = VK_NULL_HANDLE;
8707 
8708  // 1. Create VkImage.
8709  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
8710  allocator->m_hDevice,
8711  pImageCreateInfo,
8712  allocator->GetAllocationCallbacks(),
8713  pImage);
8714  if(res >= 0)
8715  {
8716  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
8717  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
8718  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
8719 
8720  // 2. Allocate memory using allocator.
8721  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
8722  if(res >= 0)
8723  {
8724  // 3. Bind image with memory.
8725  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
8726  allocator->m_hDevice,
8727  *pImage,
8728  (*pAllocation)->GetMemory(),
8729  (*pAllocation)->GetOffset());
8730  if(res >= 0)
8731  {
8732  // All steps succeeded.
8733  if(pAllocationInfo != VMA_NULL)
8734  {
8735  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8736  }
8737  return VK_SUCCESS;
8738  }
8739  allocator->FreeMemory(*pAllocation);
8740  *pAllocation = VK_NULL_HANDLE;
8741  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8742  *pImage = VK_NULL_HANDLE;
8743  return res;
8744  }
8745  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8746  *pImage = VK_NULL_HANDLE;
8747  return res;
8748  }
8749  return res;
8750 }
8751 
8752 void vmaDestroyImage(
8753  VmaAllocator allocator,
8754  VkImage image,
8755  VmaAllocation allocation)
8756 {
8757  if(image != VK_NULL_HANDLE)
8758  {
8759  VMA_ASSERT(allocator);
8760 
8761  VMA_DEBUG_LOG("vmaDestroyImage");
8762 
8763  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8764 
8765  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
8766 
8767  allocator->FreeMemory(allocation);
8768  }
8769 }
8770 
8771 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:893
-
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1147
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
826 #include <vulkan/vulkan.h>
827 
828 VK_DEFINE_HANDLE(VmaAllocator)
829 
830 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
832  VmaAllocator allocator,
833  uint32_t memoryType,
834  VkDeviceMemory memory,
835  VkDeviceSize size);
837 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
838  VmaAllocator allocator,
839  uint32_t memoryType,
840  VkDeviceMemory memory,
841  VkDeviceSize size);
842 
850 typedef struct VmaDeviceMemoryCallbacks {
852  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
854  PFN_vmaFreeDeviceMemoryFunction pfnFree;
855 } VmaDeviceMemoryCallbacks;
856 
858 typedef enum VmaAllocatorCreateFlagBits {
863  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
885  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
886 
887  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
888 } VmaAllocatorCreateFlagBits;
889 typedef VkFlags VmaAllocatorCreateFlags;
890 
895 typedef struct VmaVulkanFunctions {
896  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
897  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
898  PFN_vkAllocateMemory vkAllocateMemory;
899  PFN_vkFreeMemory vkFreeMemory;
900  PFN_vkMapMemory vkMapMemory;
901  PFN_vkUnmapMemory vkUnmapMemory;
902  PFN_vkBindBufferMemory vkBindBufferMemory;
903  PFN_vkBindImageMemory vkBindImageMemory;
904  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
905  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
906  PFN_vkCreateBuffer vkCreateBuffer;
907  PFN_vkDestroyBuffer vkDestroyBuffer;
908  PFN_vkCreateImage vkCreateImage;
909  PFN_vkDestroyImage vkDestroyImage;
910  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
911  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
912 } VmaVulkanFunctions;
913 
915 typedef struct VmaAllocatorCreateInfo
916 {
918  VmaAllocatorCreateFlags flags;
920 
921  VkPhysicalDevice physicalDevice;
923 
924  VkDevice device;
926 
927  VkDeviceSize preferredLargeHeapBlockSize;
929 
930  const VkAllocationCallbacks* pAllocationCallbacks;
932 
933  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
947  uint32_t frameInUseCount;
971  const VkDeviceSize* pHeapSizeLimit;
983  const VmaVulkanFunctions* pVulkanFunctions;
984 } VmaAllocatorCreateInfo;
985 
987 VkResult vmaCreateAllocator(
988  const VmaAllocatorCreateInfo* pCreateInfo,
989  VmaAllocator* pAllocator);
990 
992 void vmaDestroyAllocator(
993  VmaAllocator allocator);
994 
999 void vmaGetPhysicalDeviceProperties(
1000  VmaAllocator allocator,
1001  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1002 
1007 void vmaGetMemoryProperties(
1008  VmaAllocator allocator,
1009  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1010 
1017 void vmaGetMemoryTypeProperties(
1018  VmaAllocator allocator,
1019  uint32_t memoryTypeIndex,
1020  VkMemoryPropertyFlags* pFlags);
1021 
1030 void vmaSetCurrentFrameIndex(
1031  VmaAllocator allocator,
1032  uint32_t frameIndex);
1033 
1036 typedef struct VmaStatInfo
1037 {
1039  uint32_t blockCount;
1041  uint32_t allocationCount;
1043  uint32_t unusedRangeCount;
1045  VkDeviceSize usedBytes;
1047  VkDeviceSize unusedBytes;
1048  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1049  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1050 } VmaStatInfo;
1051 
1053 typedef struct VmaStats
1054 {
1055  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1056  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1057  VmaStatInfo total;
1058 } VmaStats;
1059 
1061 void vmaCalculateStats(
1062  VmaAllocator allocator,
1063  VmaStats* pStats);
1064 
1065 #define VMA_STATS_STRING_ENABLED 1
1066 
1067 #if VMA_STATS_STRING_ENABLED
1068 
1070 
1072 void vmaBuildStatsString(
1073  VmaAllocator allocator,
1074  char** ppStatsString,
1075  VkBool32 detailedMap);
1076 
1077 void vmaFreeStatsString(
1078  VmaAllocator allocator,
1079  char* pStatsString);
1080 
1081 #endif // #if VMA_STATS_STRING_ENABLED
1082 
1083 VK_DEFINE_HANDLE(VmaPool)
1084 
1085 typedef enum VmaMemoryUsage
1086 {
1090  VMA_MEMORY_USAGE_UNKNOWN = 0,
1107  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1117  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1124  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1133  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1134  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1135 } VmaMemoryUsage;
1136 
1138 typedef enum VmaAllocationCreateFlagBits {
1150  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1151 
1161  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1174  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1187  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1194  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1200  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1201 
1202  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1203 } VmaAllocationCreateFlagBits;
1204 typedef VkFlags VmaAllocationCreateFlags;
1205 
1206 typedef struct VmaAllocationCreateInfo
1207 {
1209  VmaAllocationCreateFlags flags;
1215  VmaMemoryUsage usage;
1220  VkMemoryPropertyFlags requiredFlags;
1225  VkMemoryPropertyFlags preferredFlags;
1233  uint32_t memoryTypeBits;
1239  VmaPool pool;
1246  void* pUserData;
1247 } VmaAllocationCreateInfo;
1248 
1263 VkResult vmaFindMemoryTypeIndex(
1264  VmaAllocator allocator,
1265  uint32_t memoryTypeBits,
1266  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1267  uint32_t* pMemoryTypeIndex);
1268 
1270 typedef enum VmaPoolCreateFlagBits {
1288  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1289 
1290  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1291 } VmaPoolCreateFlagBits;
1292 typedef VkFlags VmaPoolCreateFlags;
1293 
1296 typedef struct VmaPoolCreateInfo {
1299  uint32_t memoryTypeIndex;
1302  VmaPoolCreateFlags flags;
1307  VkDeviceSize blockSize;
1312  size_t minBlockCount;
1320  size_t maxBlockCount;
1334  uint32_t frameInUseCount;
1335 } VmaPoolCreateInfo;
1336 
1339 typedef struct VmaPoolStats {
1342  VkDeviceSize size;
1345  VkDeviceSize unusedSize;
1348  size_t allocationCount;
1351  size_t unusedRangeCount;
1358  VkDeviceSize unusedRangeSizeMax;
1359 } VmaPoolStats;
1360 
1367 VkResult vmaCreatePool(
1368  VmaAllocator allocator,
1369  const VmaPoolCreateInfo* pCreateInfo,
1370  VmaPool* pPool);
1371 
1374 void vmaDestroyPool(
1375  VmaAllocator allocator,
1376  VmaPool pool);
1377 
1384 void vmaGetPoolStats(
1385  VmaAllocator allocator,
1386  VmaPool pool,
1387  VmaPoolStats* pPoolStats);
1388 
1395 void vmaMakePoolAllocationsLost(
1396  VmaAllocator allocator,
1397  VmaPool pool,
1398  size_t* pLostAllocationCount);
1399 
1400 VK_DEFINE_HANDLE(VmaAllocation)
1401 
1402 
1404 typedef struct VmaAllocationInfo {
1409  uint32_t memoryType;
1418  VkDeviceMemory deviceMemory;
1423  VkDeviceSize offset;
1428  VkDeviceSize size;
1437  void* pMappedData;
1442  void* pUserData;
1443 } VmaAllocationInfo;
1444 
1455 VkResult vmaAllocateMemory(
1456  VmaAllocator allocator,
1457  const VkMemoryRequirements* pVkMemoryRequirements,
1458  const VmaAllocationCreateInfo* pCreateInfo,
1459  VmaAllocation* pAllocation,
1460  VmaAllocationInfo* pAllocationInfo);
1461 
1468 VkResult vmaAllocateMemoryForBuffer(
1469  VmaAllocator allocator,
1470  VkBuffer buffer,
1471  const VmaAllocationCreateInfo* pCreateInfo,
1472  VmaAllocation* pAllocation,
1473  VmaAllocationInfo* pAllocationInfo);
1474 
1476 VkResult vmaAllocateMemoryForImage(
1477  VmaAllocator allocator,
1478  VkImage image,
1479  const VmaAllocationCreateInfo* pCreateInfo,
1480  VmaAllocation* pAllocation,
1481  VmaAllocationInfo* pAllocationInfo);
1482 
1484 void vmaFreeMemory(
1485  VmaAllocator allocator,
1486  VmaAllocation allocation);
1487 
1489 void vmaGetAllocationInfo(
1490  VmaAllocator allocator,
1491  VmaAllocation allocation,
1492  VmaAllocationInfo* pAllocationInfo);
1493 
1507 void vmaSetAllocationUserData(
1508  VmaAllocator allocator,
1509  VmaAllocation allocation,
1510  void* pUserData);
1511 
1522 void vmaCreateLostAllocation(
1523  VmaAllocator allocator,
1524  VmaAllocation* pAllocation);
1525 
1560 VkResult vmaMapMemory(
1561  VmaAllocator allocator,
1562  VmaAllocation allocation,
1563  void** ppData);
1564 
1569 void vmaUnmapMemory(
1570  VmaAllocator allocator,
1571  VmaAllocation allocation);
1572 
1574 typedef struct VmaDefragmentationInfo {
1579  VkDeviceSize maxBytesToMove;
1584  uint32_t maxAllocationsToMove;
1585 } VmaDefragmentationInfo;
1586 
1588 typedef struct VmaDefragmentationStats {
1590  VkDeviceSize bytesMoved;
1592  VkDeviceSize bytesFreed;
1594  uint32_t allocationsMoved;
1596  uint32_t deviceMemoryBlocksFreed;
1597 } VmaDefragmentationStats;
1598 
1681 VkResult vmaDefragment(
1682  VmaAllocator allocator,
1683  VmaAllocation* pAllocations,
1684  size_t allocationCount,
1685  VkBool32* pAllocationsChanged,
1686  const VmaDefragmentationInfo *pDefragmentationInfo,
1687  VmaDefragmentationStats* pDefragmentationStats);
1688 
1715 VkResult vmaCreateBuffer(
1716  VmaAllocator allocator,
1717  const VkBufferCreateInfo* pBufferCreateInfo,
1718  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1719  VkBuffer* pBuffer,
1720  VmaAllocation* pAllocation,
1721  VmaAllocationInfo* pAllocationInfo);
1722 
1734 void vmaDestroyBuffer(
1735  VmaAllocator allocator,
1736  VkBuffer buffer,
1737  VmaAllocation allocation);
1738 
1740 VkResult vmaCreateImage(
1741  VmaAllocator allocator,
1742  const VkImageCreateInfo* pImageCreateInfo,
1743  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1744  VkImage* pImage,
1745  VmaAllocation* pAllocation,
1746  VmaAllocationInfo* pAllocationInfo);
1747 
1759 void vmaDestroyImage(
1760  VmaAllocator allocator,
1761  VkImage image,
1762  VmaAllocation allocation);
1763 
1764 #ifdef __cplusplus
1765 }
1766 #endif
1767 
1768 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1769 
1770 // For Visual Studio IntelliSense.
1771 #ifdef __INTELLISENSE__
1772 #define VMA_IMPLEMENTATION
1773 #endif
1774 
1775 #ifdef VMA_IMPLEMENTATION
1776 #undef VMA_IMPLEMENTATION
1777 
1778 #include <cstdint>
1779 #include <cstdlib>
1780 #include <cstring>
1781 
1782 /*******************************************************************************
1783 CONFIGURATION SECTION
1784 
1785 Define some of these macros before each #include of this header or change them
1786 here if you need other then default behavior depending on your environment.
1787 */
1788 
1789 /*
1790 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1791 internally, like:
1792 
1793  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1794 
1795 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1796 VmaAllocatorCreateInfo::pVulkanFunctions.
1797 */
1798 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
1799 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1800 #endif
1801 
1802 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1803 //#define VMA_USE_STL_CONTAINERS 1
1804 
1805 /* Set this macro to 1 to make the library including and using STL containers:
1806 std::pair, std::vector, std::list, std::unordered_map.
1807 
1808 Set it to 0 or undefined to make the library using its own implementation of
1809 the containers.
1810 */
1811 #if VMA_USE_STL_CONTAINERS
1812  #define VMA_USE_STL_VECTOR 1
1813  #define VMA_USE_STL_UNORDERED_MAP 1
1814  #define VMA_USE_STL_LIST 1
1815 #endif
1816 
1817 #if VMA_USE_STL_VECTOR
1818  #include <vector>
1819 #endif
1820 
1821 #if VMA_USE_STL_UNORDERED_MAP
1822  #include <unordered_map>
1823 #endif
1824 
1825 #if VMA_USE_STL_LIST
1826  #include <list>
1827 #endif
1828 
1829 /*
1830 Following headers are used in this CONFIGURATION section only, so feel free to
1831 remove them if not needed.
1832 */
1833 #include <cassert> // for assert
1834 #include <algorithm> // for min, max
1835 #include <mutex> // for std::mutex
1836 #include <atomic> // for std::atomic
1837 
1838 #if !defined(_WIN32) && !defined(__APPLE__)
1839  #include <malloc.h> // for aligned_alloc()
1840 #endif
1841 
1842 #if defined(__APPLE__)
1843 #include <cstdlib>
1844 void *aligned_alloc(size_t alignment, size_t size)
1845 {
1846  // alignment must be >= sizeof(void*)
1847  if(alignment < sizeof(void*))
1848  {
1849  alignment = sizeof(void*);
1850  }
1851 
1852  void *pointer;
1853  if(posix_memalign(&pointer, alignment, size) == 0)
1854  return pointer;
1855  return VMA_NULL;
1856 }
1857 #endif
1858 
1859 // Normal assert to check for programmer's errors, especially in Debug configuration.
1860 #ifndef VMA_ASSERT
1861  #ifdef _DEBUG
1862  #define VMA_ASSERT(expr) assert(expr)
1863  #else
1864  #define VMA_ASSERT(expr)
1865  #endif
1866 #endif
1867 
1868 // Assert that will be called very often, like inside data structures e.g. operator[].
1869 // Making it non-empty can make program slow.
1870 #ifndef VMA_HEAVY_ASSERT
1871  #ifdef _DEBUG
1872  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1873  #else
1874  #define VMA_HEAVY_ASSERT(expr)
1875  #endif
1876 #endif
1877 
1878 #ifndef VMA_NULL
1879  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1880  #define VMA_NULL nullptr
1881 #endif
1882 
1883 #ifndef VMA_ALIGN_OF
1884  #define VMA_ALIGN_OF(type) (__alignof(type))
1885 #endif
1886 
1887 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1888  #if defined(_WIN32)
1889  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1890  #else
1891  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1892  #endif
1893 #endif
1894 
1895 #ifndef VMA_SYSTEM_FREE
1896  #if defined(_WIN32)
1897  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1898  #else
1899  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1900  #endif
1901 #endif
1902 
1903 #ifndef VMA_MIN
1904  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1905 #endif
1906 
1907 #ifndef VMA_MAX
1908  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1909 #endif
1910 
1911 #ifndef VMA_SWAP
1912  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1913 #endif
1914 
1915 #ifndef VMA_SORT
1916  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1917 #endif
1918 
1919 #ifndef VMA_DEBUG_LOG
1920  #define VMA_DEBUG_LOG(format, ...)
1921  /*
1922  #define VMA_DEBUG_LOG(format, ...) do { \
1923  printf(format, __VA_ARGS__); \
1924  printf("\n"); \
1925  } while(false)
1926  */
1927 #endif
1928 
1929 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1930 #if VMA_STATS_STRING_ENABLED
1931  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1932  {
1933  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1934  }
1935  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1936  {
1937  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1938  }
1939  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1940  {
1941  snprintf(outStr, strLen, "%p", ptr);
1942  }
1943 #endif
1944 
1945 #ifndef VMA_MUTEX
1946  class VmaMutex
1947  {
1948  public:
1949  VmaMutex() { }
1950  ~VmaMutex() { }
1951  void Lock() { m_Mutex.lock(); }
1952  void Unlock() { m_Mutex.unlock(); }
1953  private:
1954  std::mutex m_Mutex;
1955  };
1956  #define VMA_MUTEX VmaMutex
1957 #endif
1958 
1959 /*
1960 If providing your own implementation, you need to implement a subset of std::atomic:
1961 
1962 - Constructor(uint32_t desired)
1963 - uint32_t load() const
1964 - void store(uint32_t desired)
1965 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1966 */
1967 #ifndef VMA_ATOMIC_UINT32
1968  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1969 #endif
1970 
1971 #ifndef VMA_BEST_FIT
1972 
1984  #define VMA_BEST_FIT (1)
1985 #endif
1986 
1987 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
1988 
1992  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
1993 #endif
1994 
1995 #ifndef VMA_DEBUG_ALIGNMENT
1996 
2000  #define VMA_DEBUG_ALIGNMENT (1)
2001 #endif
2002 
2003 #ifndef VMA_DEBUG_MARGIN
2004 
2008  #define VMA_DEBUG_MARGIN (0)
2009 #endif
2010 
2011 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2012 
2016  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2017 #endif
2018 
2019 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2020 
2024  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2025 #endif
2026 
2027 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2028  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2030 #endif
2031 
2032 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2033  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2035 #endif
2036 
2037 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2038 
2039 /*******************************************************************************
2040 END OF CONFIGURATION
2041 */
2042 
2043 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2044  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2045 
2046 // Returns number of bits set to 1 in (v).
2047 static inline uint32_t VmaCountBitsSet(uint32_t v)
2048 {
2049  uint32_t c = v - ((v >> 1) & 0x55555555);
2050  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2051  c = ((c >> 4) + c) & 0x0F0F0F0F;
2052  c = ((c >> 8) + c) & 0x00FF00FF;
2053  c = ((c >> 16) + c) & 0x0000FFFF;
2054  return c;
2055 }
2056 
2057 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2058 // Use types like uint32_t, uint64_t as T.
2059 template <typename T>
2060 static inline T VmaAlignUp(T val, T align)
2061 {
2062  return (val + align - 1) / align * align;
2063 }
2064 
2065 // Division with mathematical rounding to nearest number.
2066 template <typename T>
2067 inline T VmaRoundDiv(T x, T y)
2068 {
2069  return (x + (y / (T)2)) / y;
2070 }
2071 
2072 #ifndef VMA_SORT
2073 
2074 template<typename Iterator, typename Compare>
2075 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2076 {
2077  Iterator centerValue = end; --centerValue;
2078  Iterator insertIndex = beg;
2079  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2080  {
2081  if(cmp(*memTypeIndex, *centerValue))
2082  {
2083  if(insertIndex != memTypeIndex)
2084  {
2085  VMA_SWAP(*memTypeIndex, *insertIndex);
2086  }
2087  ++insertIndex;
2088  }
2089  }
2090  if(insertIndex != centerValue)
2091  {
2092  VMA_SWAP(*insertIndex, *centerValue);
2093  }
2094  return insertIndex;
2095 }
2096 
2097 template<typename Iterator, typename Compare>
2098 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2099 {
2100  if(beg < end)
2101  {
2102  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2103  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2104  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2105  }
2106 }
2107 
2108 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2109 
2110 #endif // #ifndef VMA_SORT
2111 
2112 /*
2113 Returns true if two memory blocks occupy overlapping pages.
2114 ResourceA must be in less memory offset than ResourceB.
2115 
2116 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2117 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2118 */
2119 static inline bool VmaBlocksOnSamePage(
2120  VkDeviceSize resourceAOffset,
2121  VkDeviceSize resourceASize,
2122  VkDeviceSize resourceBOffset,
2123  VkDeviceSize pageSize)
2124 {
2125  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2126  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2127  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2128  VkDeviceSize resourceBStart = resourceBOffset;
2129  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2130  return resourceAEndPage == resourceBStartPage;
2131 }
2132 
2133 enum VmaSuballocationType
2134 {
2135  VMA_SUBALLOCATION_TYPE_FREE = 0,
2136  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2137  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2138  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2139  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2140  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2141  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2142 };
2143 
2144 /*
2145 Returns true if given suballocation types could conflict and must respect
2146 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2147 or linear image and another one is optimal image. If type is unknown, behave
2148 conservatively.
2149 */
2150 static inline bool VmaIsBufferImageGranularityConflict(
2151  VmaSuballocationType suballocType1,
2152  VmaSuballocationType suballocType2)
2153 {
2154  if(suballocType1 > suballocType2)
2155  {
2156  VMA_SWAP(suballocType1, suballocType2);
2157  }
2158 
2159  switch(suballocType1)
2160  {
2161  case VMA_SUBALLOCATION_TYPE_FREE:
2162  return false;
2163  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2164  return true;
2165  case VMA_SUBALLOCATION_TYPE_BUFFER:
2166  return
2167  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2168  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2169  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2170  return
2171  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2172  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2173  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2174  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2175  return
2176  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2177  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2178  return false;
2179  default:
2180  VMA_ASSERT(0);
2181  return true;
2182  }
2183 }
2184 
2185 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2186 struct VmaMutexLock
2187 {
2188 public:
2189  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2190  m_pMutex(useMutex ? &mutex : VMA_NULL)
2191  {
2192  if(m_pMutex)
2193  {
2194  m_pMutex->Lock();
2195  }
2196  }
2197 
2198  ~VmaMutexLock()
2199  {
2200  if(m_pMutex)
2201  {
2202  m_pMutex->Unlock();
2203  }
2204  }
2205 
2206 private:
2207  VMA_MUTEX* m_pMutex;
2208 };
2209 
2210 #if VMA_DEBUG_GLOBAL_MUTEX
2211  static VMA_MUTEX gDebugGlobalMutex;
2212  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2213 #else
2214  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2215 #endif
2216 
2217 // Minimum size of a free suballocation to register it in the free suballocation collection.
2218 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2219 
2220 /*
2221 Performs binary search and returns iterator to first element that is greater or
2222 equal to (key), according to comparison (cmp).
2223 
2224 Cmp should return true if first argument is less than second argument.
2225 
2226 Returned value is the found element, if present in the collection or place where
2227 new element with value (key) should be inserted.
2228 */
2229 template <typename IterT, typename KeyT, typename CmpT>
2230 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2231 {
2232  size_t down = 0, up = (end - beg);
2233  while(down < up)
2234  {
2235  const size_t mid = (down + up) / 2;
2236  if(cmp(*(beg+mid), key))
2237  {
2238  down = mid + 1;
2239  }
2240  else
2241  {
2242  up = mid;
2243  }
2244  }
2245  return beg + down;
2246 }
2247 
2249 // Memory allocation
2250 
2251 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2252 {
2253  if((pAllocationCallbacks != VMA_NULL) &&
2254  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2255  {
2256  return (*pAllocationCallbacks->pfnAllocation)(
2257  pAllocationCallbacks->pUserData,
2258  size,
2259  alignment,
2260  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2261  }
2262  else
2263  {
2264  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2265  }
2266 }
2267 
2268 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2269 {
2270  if((pAllocationCallbacks != VMA_NULL) &&
2271  (pAllocationCallbacks->pfnFree != VMA_NULL))
2272  {
2273  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2274  }
2275  else
2276  {
2277  VMA_SYSTEM_FREE(ptr);
2278  }
2279 }
2280 
2281 template<typename T>
2282 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2283 {
2284  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2285 }
2286 
2287 template<typename T>
2288 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2289 {
2290  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2291 }
2292 
2293 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2294 
2295 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2296 
2297 template<typename T>
2298 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2299 {
2300  ptr->~T();
2301  VmaFree(pAllocationCallbacks, ptr);
2302 }
2303 
2304 template<typename T>
2305 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2306 {
2307  if(ptr != VMA_NULL)
2308  {
2309  for(size_t i = count; i--; )
2310  {
2311  ptr[i].~T();
2312  }
2313  VmaFree(pAllocationCallbacks, ptr);
2314  }
2315 }
2316 
2317 // STL-compatible allocator.
2318 template<typename T>
2319 class VmaStlAllocator
2320 {
2321 public:
2322  const VkAllocationCallbacks* const m_pCallbacks;
2323  typedef T value_type;
2324 
2325  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
2326  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2327 
2328  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
2329  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2330 
2331  template<typename U>
2332  bool operator==(const VmaStlAllocator<U>& rhs) const
2333  {
2334  return m_pCallbacks == rhs.m_pCallbacks;
2335  }
2336  template<typename U>
2337  bool operator!=(const VmaStlAllocator<U>& rhs) const
2338  {
2339  return m_pCallbacks != rhs.m_pCallbacks;
2340  }
2341 
2342  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2343 };
2344 
2345 #if VMA_USE_STL_VECTOR
2346 
2347 #define VmaVector std::vector
2348 
2349 template<typename T, typename allocatorT>
2350 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2351 {
2352  vec.insert(vec.begin() + index, item);
2353 }
2354 
2355 template<typename T, typename allocatorT>
2356 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2357 {
2358  vec.erase(vec.begin() + index);
2359 }
2360 
2361 #else // #if VMA_USE_STL_VECTOR
2362 
2363 /* Class with interface compatible with subset of std::vector.
2364 T must be POD because constructors and destructors are not called and memcpy is
2365 used for these objects. */
2366 template<typename T, typename AllocatorT>
2367 class VmaVector
2368 {
2369 public:
2370  typedef T value_type;
2371 
2372  VmaVector(const AllocatorT& allocator) :
2373  m_Allocator(allocator),
2374  m_pArray(VMA_NULL),
2375  m_Count(0),
2376  m_Capacity(0)
2377  {
2378  }
2379 
2380  VmaVector(size_t count, const AllocatorT& allocator) :
2381  m_Allocator(allocator),
2382  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2383  m_Count(count),
2384  m_Capacity(count)
2385  {
2386  }
2387 
2388  VmaVector(const VmaVector<T, AllocatorT>& src) :
2389  m_Allocator(src.m_Allocator),
2390  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2391  m_Count(src.m_Count),
2392  m_Capacity(src.m_Count)
2393  {
2394  if(m_Count != 0)
2395  {
2396  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2397  }
2398  }
2399 
2400  ~VmaVector()
2401  {
2402  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2403  }
2404 
2405  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2406  {
2407  if(&rhs != this)
2408  {
2409  resize(rhs.m_Count);
2410  if(m_Count != 0)
2411  {
2412  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2413  }
2414  }
2415  return *this;
2416  }
2417 
2418  bool empty() const { return m_Count == 0; }
2419  size_t size() const { return m_Count; }
2420  T* data() { return m_pArray; }
2421  const T* data() const { return m_pArray; }
2422 
2423  T& operator[](size_t index)
2424  {
2425  VMA_HEAVY_ASSERT(index < m_Count);
2426  return m_pArray[index];
2427  }
2428  const T& operator[](size_t index) const
2429  {
2430  VMA_HEAVY_ASSERT(index < m_Count);
2431  return m_pArray[index];
2432  }
2433 
2434  T& front()
2435  {
2436  VMA_HEAVY_ASSERT(m_Count > 0);
2437  return m_pArray[0];
2438  }
2439  const T& front() const
2440  {
2441  VMA_HEAVY_ASSERT(m_Count > 0);
2442  return m_pArray[0];
2443  }
2444  T& back()
2445  {
2446  VMA_HEAVY_ASSERT(m_Count > 0);
2447  return m_pArray[m_Count - 1];
2448  }
2449  const T& back() const
2450  {
2451  VMA_HEAVY_ASSERT(m_Count > 0);
2452  return m_pArray[m_Count - 1];
2453  }
2454 
2455  void reserve(size_t newCapacity, bool freeMemory = false)
2456  {
2457  newCapacity = VMA_MAX(newCapacity, m_Count);
2458 
2459  if((newCapacity < m_Capacity) && !freeMemory)
2460  {
2461  newCapacity = m_Capacity;
2462  }
2463 
2464  if(newCapacity != m_Capacity)
2465  {
2466  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2467  if(m_Count != 0)
2468  {
2469  memcpy(newArray, m_pArray, m_Count * sizeof(T));
2470  }
2471  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2472  m_Capacity = newCapacity;
2473  m_pArray = newArray;
2474  }
2475  }
2476 
2477  void resize(size_t newCount, bool freeMemory = false)
2478  {
2479  size_t newCapacity = m_Capacity;
2480  if(newCount > m_Capacity)
2481  {
2482  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2483  }
2484  else if(freeMemory)
2485  {
2486  newCapacity = newCount;
2487  }
2488 
2489  if(newCapacity != m_Capacity)
2490  {
2491  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2492  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2493  if(elementsToCopy != 0)
2494  {
2495  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2496  }
2497  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2498  m_Capacity = newCapacity;
2499  m_pArray = newArray;
2500  }
2501 
2502  m_Count = newCount;
2503  }
2504 
2505  void clear(bool freeMemory = false)
2506  {
2507  resize(0, freeMemory);
2508  }
2509 
2510  void insert(size_t index, const T& src)
2511  {
2512  VMA_HEAVY_ASSERT(index <= m_Count);
2513  const size_t oldCount = size();
2514  resize(oldCount + 1);
2515  if(index < oldCount)
2516  {
2517  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2518  }
2519  m_pArray[index] = src;
2520  }
2521 
2522  void remove(size_t index)
2523  {
2524  VMA_HEAVY_ASSERT(index < m_Count);
2525  const size_t oldCount = size();
2526  if(index < oldCount - 1)
2527  {
2528  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2529  }
2530  resize(oldCount - 1);
2531  }
2532 
2533  void push_back(const T& src)
2534  {
2535  const size_t newIndex = size();
2536  resize(newIndex + 1);
2537  m_pArray[newIndex] = src;
2538  }
2539 
2540  void pop_back()
2541  {
2542  VMA_HEAVY_ASSERT(m_Count > 0);
2543  resize(size() - 1);
2544  }
2545 
2546  void push_front(const T& src)
2547  {
2548  insert(0, src);
2549  }
2550 
2551  void pop_front()
2552  {
2553  VMA_HEAVY_ASSERT(m_Count > 0);
2554  remove(0);
2555  }
2556 
2557  typedef T* iterator;
2558 
2559  iterator begin() { return m_pArray; }
2560  iterator end() { return m_pArray + m_Count; }
2561 
2562 private:
2563  AllocatorT m_Allocator;
2564  T* m_pArray;
2565  size_t m_Count;
2566  size_t m_Capacity;
2567 };
2568 
2569 template<typename T, typename allocatorT>
2570 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2571 {
2572  vec.insert(index, item);
2573 }
2574 
2575 template<typename T, typename allocatorT>
2576 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2577 {
2578  vec.remove(index);
2579 }
2580 
2581 #endif // #if VMA_USE_STL_VECTOR
2582 
2583 template<typename CmpLess, typename VectorT>
2584 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2585 {
2586  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2587  vector.data(),
2588  vector.data() + vector.size(),
2589  value,
2590  CmpLess()) - vector.data();
2591  VmaVectorInsert(vector, indexToInsert, value);
2592  return indexToInsert;
2593 }
2594 
2595 template<typename CmpLess, typename VectorT>
2596 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2597 {
2598  CmpLess comparator;
2599  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2600  vector.begin(),
2601  vector.end(),
2602  value,
2603  comparator);
2604  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2605  {
2606  size_t indexToRemove = it - vector.begin();
2607  VmaVectorRemove(vector, indexToRemove);
2608  return true;
2609  }
2610  return false;
2611 }
2612 
2613 template<typename CmpLess, typename VectorT>
2614 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2615 {
2616  CmpLess comparator;
2617  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2618  vector.data(),
2619  vector.data() + vector.size(),
2620  value,
2621  comparator);
2622  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2623  {
2624  return it - vector.begin();
2625  }
2626  else
2627  {
2628  return vector.size();
2629  }
2630 }
2631 
2633 // class VmaPoolAllocator
2634 
2635 /*
2636 Allocator for objects of type T using a list of arrays (pools) to speed up
2637 allocation. Number of elements that can be allocated is not bounded because
2638 allocator can create multiple blocks.
2639 */
2640 template<typename T>
2641 class VmaPoolAllocator
2642 {
2643 public:
2644  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2645  ~VmaPoolAllocator();
2646  void Clear();
2647  T* Alloc();
2648  void Free(T* ptr);
2649 
2650 private:
2651  union Item
2652  {
2653  uint32_t NextFreeIndex;
2654  T Value;
2655  };
2656 
2657  struct ItemBlock
2658  {
2659  Item* pItems;
2660  uint32_t FirstFreeIndex;
2661  };
2662 
2663  const VkAllocationCallbacks* m_pAllocationCallbacks;
2664  size_t m_ItemsPerBlock;
2665  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2666 
2667  ItemBlock& CreateNewBlock();
2668 };
2669 
2670 template<typename T>
2671 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2672  m_pAllocationCallbacks(pAllocationCallbacks),
2673  m_ItemsPerBlock(itemsPerBlock),
2674  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2675 {
2676  VMA_ASSERT(itemsPerBlock > 0);
2677 }
2678 
2679 template<typename T>
2680 VmaPoolAllocator<T>::~VmaPoolAllocator()
2681 {
2682  Clear();
2683 }
2684 
2685 template<typename T>
2686 void VmaPoolAllocator<T>::Clear()
2687 {
2688  for(size_t i = m_ItemBlocks.size(); i--; )
2689  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2690  m_ItemBlocks.clear();
2691 }
2692 
2693 template<typename T>
2694 T* VmaPoolAllocator<T>::Alloc()
2695 {
2696  for(size_t i = m_ItemBlocks.size(); i--; )
2697  {
2698  ItemBlock& block = m_ItemBlocks[i];
2699  // This block has some free items: Use first one.
2700  if(block.FirstFreeIndex != UINT32_MAX)
2701  {
2702  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2703  block.FirstFreeIndex = pItem->NextFreeIndex;
2704  return &pItem->Value;
2705  }
2706  }
2707 
2708  // No block has free item: Create new one and use it.
2709  ItemBlock& newBlock = CreateNewBlock();
2710  Item* const pItem = &newBlock.pItems[0];
2711  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2712  return &pItem->Value;
2713 }
2714 
2715 template<typename T>
2716 void VmaPoolAllocator<T>::Free(T* ptr)
2717 {
2718  // Search all memory blocks to find ptr.
2719  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2720  {
2721  ItemBlock& block = m_ItemBlocks[i];
2722 
2723  // Casting to union.
2724  Item* pItemPtr;
2725  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2726 
2727  // Check if pItemPtr is in address range of this block.
2728  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2729  {
2730  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2731  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2732  block.FirstFreeIndex = index;
2733  return;
2734  }
2735  }
2736  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2737 }
2738 
2739 template<typename T>
2740 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2741 {
2742  ItemBlock newBlock = {
2743  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2744 
2745  m_ItemBlocks.push_back(newBlock);
2746 
2747  // Setup singly-linked list of all free items in this block.
2748  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2749  newBlock.pItems[i].NextFreeIndex = i + 1;
2750  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2751  return m_ItemBlocks.back();
2752 }
2753 
2755 // class VmaRawList, VmaList
2756 
2757 #if VMA_USE_STL_LIST
2758 
2759 #define VmaList std::list
2760 
2761 #else // #if VMA_USE_STL_LIST
2762 
2763 template<typename T>
2764 struct VmaListItem
2765 {
2766  VmaListItem* pPrev;
2767  VmaListItem* pNext;
2768  T Value;
2769 };
2770 
2771 // Doubly linked list.
2772 template<typename T>
2773 class VmaRawList
2774 {
2775 public:
2776  typedef VmaListItem<T> ItemType;
2777 
2778  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2779  ~VmaRawList();
2780  void Clear();
2781 
2782  size_t GetCount() const { return m_Count; }
2783  bool IsEmpty() const { return m_Count == 0; }
2784 
2785  ItemType* Front() { return m_pFront; }
2786  const ItemType* Front() const { return m_pFront; }
2787  ItemType* Back() { return m_pBack; }
2788  const ItemType* Back() const { return m_pBack; }
2789 
2790  ItemType* PushBack();
2791  ItemType* PushFront();
2792  ItemType* PushBack(const T& value);
2793  ItemType* PushFront(const T& value);
2794  void PopBack();
2795  void PopFront();
2796 
2797  // Item can be null - it means PushBack.
2798  ItemType* InsertBefore(ItemType* pItem);
2799  // Item can be null - it means PushFront.
2800  ItemType* InsertAfter(ItemType* pItem);
2801 
2802  ItemType* InsertBefore(ItemType* pItem, const T& value);
2803  ItemType* InsertAfter(ItemType* pItem, const T& value);
2804 
2805  void Remove(ItemType* pItem);
2806 
2807 private:
2808  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2809  VmaPoolAllocator<ItemType> m_ItemAllocator;
2810  ItemType* m_pFront;
2811  ItemType* m_pBack;
2812  size_t m_Count;
2813 
2814  // Declared not defined, to block copy constructor and assignment operator.
2815  VmaRawList(const VmaRawList<T>& src);
2816  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2817 };
2818 
2819 template<typename T>
2820 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2821  m_pAllocationCallbacks(pAllocationCallbacks),
2822  m_ItemAllocator(pAllocationCallbacks, 128),
2823  m_pFront(VMA_NULL),
2824  m_pBack(VMA_NULL),
2825  m_Count(0)
2826 {
2827 }
2828 
2829 template<typename T>
2830 VmaRawList<T>::~VmaRawList()
2831 {
2832  // Intentionally not calling Clear, because that would be unnecessary
2833  // computations to return all items to m_ItemAllocator as free.
2834 }
2835 
2836 template<typename T>
2837 void VmaRawList<T>::Clear()
2838 {
2839  if(IsEmpty() == false)
2840  {
2841  ItemType* pItem = m_pBack;
2842  while(pItem != VMA_NULL)
2843  {
2844  ItemType* const pPrevItem = pItem->pPrev;
2845  m_ItemAllocator.Free(pItem);
2846  pItem = pPrevItem;
2847  }
2848  m_pFront = VMA_NULL;
2849  m_pBack = VMA_NULL;
2850  m_Count = 0;
2851  }
2852 }
2853 
2854 template<typename T>
2855 VmaListItem<T>* VmaRawList<T>::PushBack()
2856 {
2857  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2858  pNewItem->pNext = VMA_NULL;
2859  if(IsEmpty())
2860  {
2861  pNewItem->pPrev = VMA_NULL;
2862  m_pFront = pNewItem;
2863  m_pBack = pNewItem;
2864  m_Count = 1;
2865  }
2866  else
2867  {
2868  pNewItem->pPrev = m_pBack;
2869  m_pBack->pNext = pNewItem;
2870  m_pBack = pNewItem;
2871  ++m_Count;
2872  }
2873  return pNewItem;
2874 }
2875 
2876 template<typename T>
2877 VmaListItem<T>* VmaRawList<T>::PushFront()
2878 {
2879  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2880  pNewItem->pPrev = VMA_NULL;
2881  if(IsEmpty())
2882  {
2883  pNewItem->pNext = VMA_NULL;
2884  m_pFront = pNewItem;
2885  m_pBack = pNewItem;
2886  m_Count = 1;
2887  }
2888  else
2889  {
2890  pNewItem->pNext = m_pFront;
2891  m_pFront->pPrev = pNewItem;
2892  m_pFront = pNewItem;
2893  ++m_Count;
2894  }
2895  return pNewItem;
2896 }
2897 
2898 template<typename T>
2899 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2900 {
2901  ItemType* const pNewItem = PushBack();
2902  pNewItem->Value = value;
2903  return pNewItem;
2904 }
2905 
2906 template<typename T>
2907 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2908 {
2909  ItemType* const pNewItem = PushFront();
2910  pNewItem->Value = value;
2911  return pNewItem;
2912 }
2913 
2914 template<typename T>
2915 void VmaRawList<T>::PopBack()
2916 {
2917  VMA_HEAVY_ASSERT(m_Count > 0);
2918  ItemType* const pBackItem = m_pBack;
2919  ItemType* const pPrevItem = pBackItem->pPrev;
2920  if(pPrevItem != VMA_NULL)
2921  {
2922  pPrevItem->pNext = VMA_NULL;
2923  }
2924  m_pBack = pPrevItem;
2925  m_ItemAllocator.Free(pBackItem);
2926  --m_Count;
2927 }
2928 
2929 template<typename T>
2930 void VmaRawList<T>::PopFront()
2931 {
2932  VMA_HEAVY_ASSERT(m_Count > 0);
2933  ItemType* const pFrontItem = m_pFront;
2934  ItemType* const pNextItem = pFrontItem->pNext;
2935  if(pNextItem != VMA_NULL)
2936  {
2937  pNextItem->pPrev = VMA_NULL;
2938  }
2939  m_pFront = pNextItem;
2940  m_ItemAllocator.Free(pFrontItem);
2941  --m_Count;
2942 }
2943 
2944 template<typename T>
2945 void VmaRawList<T>::Remove(ItemType* pItem)
2946 {
2947  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2948  VMA_HEAVY_ASSERT(m_Count > 0);
2949 
2950  if(pItem->pPrev != VMA_NULL)
2951  {
2952  pItem->pPrev->pNext = pItem->pNext;
2953  }
2954  else
2955  {
2956  VMA_HEAVY_ASSERT(m_pFront == pItem);
2957  m_pFront = pItem->pNext;
2958  }
2959 
2960  if(pItem->pNext != VMA_NULL)
2961  {
2962  pItem->pNext->pPrev = pItem->pPrev;
2963  }
2964  else
2965  {
2966  VMA_HEAVY_ASSERT(m_pBack == pItem);
2967  m_pBack = pItem->pPrev;
2968  }
2969 
2970  m_ItemAllocator.Free(pItem);
2971  --m_Count;
2972 }
2973 
2974 template<typename T>
2975 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2976 {
2977  if(pItem != VMA_NULL)
2978  {
2979  ItemType* const prevItem = pItem->pPrev;
2980  ItemType* const newItem = m_ItemAllocator.Alloc();
2981  newItem->pPrev = prevItem;
2982  newItem->pNext = pItem;
2983  pItem->pPrev = newItem;
2984  if(prevItem != VMA_NULL)
2985  {
2986  prevItem->pNext = newItem;
2987  }
2988  else
2989  {
2990  VMA_HEAVY_ASSERT(m_pFront == pItem);
2991  m_pFront = newItem;
2992  }
2993  ++m_Count;
2994  return newItem;
2995  }
2996  else
2997  return PushBack();
2998 }
2999 
3000 template<typename T>
3001 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3002 {
3003  if(pItem != VMA_NULL)
3004  {
3005  ItemType* const nextItem = pItem->pNext;
3006  ItemType* const newItem = m_ItemAllocator.Alloc();
3007  newItem->pNext = nextItem;
3008  newItem->pPrev = pItem;
3009  pItem->pNext = newItem;
3010  if(nextItem != VMA_NULL)
3011  {
3012  nextItem->pPrev = newItem;
3013  }
3014  else
3015  {
3016  VMA_HEAVY_ASSERT(m_pBack == pItem);
3017  m_pBack = newItem;
3018  }
3019  ++m_Count;
3020  return newItem;
3021  }
3022  else
3023  return PushFront();
3024 }
3025 
3026 template<typename T>
3027 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3028 {
3029  ItemType* const newItem = InsertBefore(pItem);
3030  newItem->Value = value;
3031  return newItem;
3032 }
3033 
3034 template<typename T>
3035 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3036 {
3037  ItemType* const newItem = InsertAfter(pItem);
3038  newItem->Value = value;
3039  return newItem;
3040 }
3041 
3042 template<typename T, typename AllocatorT>
3043 class VmaList
3044 {
3045 public:
3046  class iterator
3047  {
3048  public:
3049  iterator() :
3050  m_pList(VMA_NULL),
3051  m_pItem(VMA_NULL)
3052  {
3053  }
3054 
3055  T& operator*() const
3056  {
3057  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3058  return m_pItem->Value;
3059  }
3060  T* operator->() const
3061  {
3062  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3063  return &m_pItem->Value;
3064  }
3065 
3066  iterator& operator++()
3067  {
3068  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3069  m_pItem = m_pItem->pNext;
3070  return *this;
3071  }
3072  iterator& operator--()
3073  {
3074  if(m_pItem != VMA_NULL)
3075  {
3076  m_pItem = m_pItem->pPrev;
3077  }
3078  else
3079  {
3080  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3081  m_pItem = m_pList->Back();
3082  }
3083  return *this;
3084  }
3085 
3086  iterator operator++(int)
3087  {
3088  iterator result = *this;
3089  ++*this;
3090  return result;
3091  }
3092  iterator operator--(int)
3093  {
3094  iterator result = *this;
3095  --*this;
3096  return result;
3097  }
3098 
3099  bool operator==(const iterator& rhs) const
3100  {
3101  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3102  return m_pItem == rhs.m_pItem;
3103  }
3104  bool operator!=(const iterator& rhs) const
3105  {
3106  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3107  return m_pItem != rhs.m_pItem;
3108  }
3109 
3110  private:
3111  VmaRawList<T>* m_pList;
3112  VmaListItem<T>* m_pItem;
3113 
3114  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3115  m_pList(pList),
3116  m_pItem(pItem)
3117  {
3118  }
3119 
3120  friend class VmaList<T, AllocatorT>;
3121  };
3122 
3123  class const_iterator
3124  {
3125  public:
3126  const_iterator() :
3127  m_pList(VMA_NULL),
3128  m_pItem(VMA_NULL)
3129  {
3130  }
3131 
3132  const_iterator(const iterator& src) :
3133  m_pList(src.m_pList),
3134  m_pItem(src.m_pItem)
3135  {
3136  }
3137 
3138  const T& operator*() const
3139  {
3140  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3141  return m_pItem->Value;
3142  }
3143  const T* operator->() const
3144  {
3145  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3146  return &m_pItem->Value;
3147  }
3148 
3149  const_iterator& operator++()
3150  {
3151  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3152  m_pItem = m_pItem->pNext;
3153  return *this;
3154  }
3155  const_iterator& operator--()
3156  {
3157  if(m_pItem != VMA_NULL)
3158  {
3159  m_pItem = m_pItem->pPrev;
3160  }
3161  else
3162  {
3163  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3164  m_pItem = m_pList->Back();
3165  }
3166  return *this;
3167  }
3168 
3169  const_iterator operator++(int)
3170  {
3171  const_iterator result = *this;
3172  ++*this;
3173  return result;
3174  }
3175  const_iterator operator--(int)
3176  {
3177  const_iterator result = *this;
3178  --*this;
3179  return result;
3180  }
3181 
3182  bool operator==(const const_iterator& rhs) const
3183  {
3184  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3185  return m_pItem == rhs.m_pItem;
3186  }
3187  bool operator!=(const const_iterator& rhs) const
3188  {
3189  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3190  return m_pItem != rhs.m_pItem;
3191  }
3192 
3193  private:
3194  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3195  m_pList(pList),
3196  m_pItem(pItem)
3197  {
3198  }
3199 
3200  const VmaRawList<T>* m_pList;
3201  const VmaListItem<T>* m_pItem;
3202 
3203  friend class VmaList<T, AllocatorT>;
3204  };
3205 
3206  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3207 
3208  bool empty() const { return m_RawList.IsEmpty(); }
3209  size_t size() const { return m_RawList.GetCount(); }
3210 
3211  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
3212  iterator end() { return iterator(&m_RawList, VMA_NULL); }
3213 
3214  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
3215  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3216 
3217  void clear() { m_RawList.Clear(); }
3218  void push_back(const T& value) { m_RawList.PushBack(value); }
3219  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
3220  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3221 
3222 private:
3223  VmaRawList<T> m_RawList;
3224 };
3225 
3226 #endif // #if VMA_USE_STL_LIST
3227 
3229 // class VmaMap
3230 
3231 // Unused in this version.
3232 #if 0
3233 
3234 #if VMA_USE_STL_UNORDERED_MAP
3235 
3236 #define VmaPair std::pair
3237 
3238 #define VMA_MAP_TYPE(KeyT, ValueT) \
3239  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3240 
3241 #else // #if VMA_USE_STL_UNORDERED_MAP
3242 
3243 template<typename T1, typename T2>
3244 struct VmaPair
3245 {
3246  T1 first;
3247  T2 second;
3248 
3249  VmaPair() : first(), second() { }
3250  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3251 };
3252 
3253 /* Class compatible with subset of interface of std::unordered_map.
3254 KeyT, ValueT must be POD because they will be stored in VmaVector.
3255 */
3256 template<typename KeyT, typename ValueT>
3257 class VmaMap
3258 {
3259 public:
3260  typedef VmaPair<KeyT, ValueT> PairType;
3261  typedef PairType* iterator;
3262 
3263  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3264 
3265  iterator begin() { return m_Vector.begin(); }
3266  iterator end() { return m_Vector.end(); }
3267 
3268  void insert(const PairType& pair);
3269  iterator find(const KeyT& key);
3270  void erase(iterator it);
3271 
3272 private:
3273  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3274 };
3275 
3276 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3277 
3278 template<typename FirstT, typename SecondT>
3279 struct VmaPairFirstLess
3280 {
3281  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3282  {
3283  return lhs.first < rhs.first;
3284  }
3285  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3286  {
3287  return lhs.first < rhsFirst;
3288  }
3289 };
3290 
3291 template<typename KeyT, typename ValueT>
3292 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3293 {
3294  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3295  m_Vector.data(),
3296  m_Vector.data() + m_Vector.size(),
3297  pair,
3298  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3299  VmaVectorInsert(m_Vector, indexToInsert, pair);
3300 }
3301 
3302 template<typename KeyT, typename ValueT>
3303 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3304 {
3305  PairType* it = VmaBinaryFindFirstNotLess(
3306  m_Vector.data(),
3307  m_Vector.data() + m_Vector.size(),
3308  key,
3309  VmaPairFirstLess<KeyT, ValueT>());
3310  if((it != m_Vector.end()) && (it->first == key))
3311  {
3312  return it;
3313  }
3314  else
3315  {
3316  return m_Vector.end();
3317  }
3318 }
3319 
3320 template<typename KeyT, typename ValueT>
3321 void VmaMap<KeyT, ValueT>::erase(iterator it)
3322 {
3323  VmaVectorRemove(m_Vector, it - m_Vector.begin());
3324 }
3325 
3326 #endif // #if VMA_USE_STL_UNORDERED_MAP
3327 
3328 #endif // #if 0
3329 
3331 
3332 class VmaDeviceMemoryBlock;
3333 
3334 struct VmaAllocation_T
3335 {
3336 private:
3337  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3338 
3339  enum FLAGS
3340  {
3341  FLAG_USER_DATA_STRING = 0x01,
3342  };
3343 
3344 public:
3345  enum ALLOCATION_TYPE
3346  {
3347  ALLOCATION_TYPE_NONE,
3348  ALLOCATION_TYPE_BLOCK,
3349  ALLOCATION_TYPE_DEDICATED,
3350  };
3351 
3352  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3353  m_Alignment(1),
3354  m_Size(0),
3355  m_pUserData(VMA_NULL),
3356  m_LastUseFrameIndex(currentFrameIndex),
3357  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3358  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3359  m_MapCount(0),
3360  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3361  {
3362  }
3363 
3364  ~VmaAllocation_T()
3365  {
3366  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3367 
3368  // Check if owned string was freed.
3369  VMA_ASSERT(m_pUserData == VMA_NULL);
3370  }
3371 
3372  void InitBlockAllocation(
3373  VmaPool hPool,
3374  VmaDeviceMemoryBlock* block,
3375  VkDeviceSize offset,
3376  VkDeviceSize alignment,
3377  VkDeviceSize size,
3378  VmaSuballocationType suballocationType,
3379  bool mapped,
3380  bool canBecomeLost)
3381  {
3382  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3383  VMA_ASSERT(block != VMA_NULL);
3384  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3385  m_Alignment = alignment;
3386  m_Size = size;
3387  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3388  m_SuballocationType = (uint8_t)suballocationType;
3389  m_BlockAllocation.m_hPool = hPool;
3390  m_BlockAllocation.m_Block = block;
3391  m_BlockAllocation.m_Offset = offset;
3392  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3393  }
3394 
3395  void InitLost()
3396  {
3397  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3398  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3399  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3400  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3401  m_BlockAllocation.m_Block = VMA_NULL;
3402  m_BlockAllocation.m_Offset = 0;
3403  m_BlockAllocation.m_CanBecomeLost = true;
3404  }
3405 
3406  void ChangeBlockAllocation(
3407  VmaAllocator hAllocator,
3408  VmaDeviceMemoryBlock* block,
3409  VkDeviceSize offset);
3410 
3411  // pMappedData not null means allocation is created with MAPPED flag.
3412  void InitDedicatedAllocation(
3413  uint32_t memoryTypeIndex,
3414  VkDeviceMemory hMemory,
3415  VmaSuballocationType suballocationType,
3416  void* pMappedData,
3417  VkDeviceSize size)
3418  {
3419  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3420  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3421  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3422  m_Alignment = 0;
3423  m_Size = size;
3424  m_SuballocationType = (uint8_t)suballocationType;
3425  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3426  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3427  m_DedicatedAllocation.m_hMemory = hMemory;
3428  m_DedicatedAllocation.m_pMappedData = pMappedData;
3429  }
3430 
3431  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
3432  VkDeviceSize GetAlignment() const { return m_Alignment; }
3433  VkDeviceSize GetSize() const { return m_Size; }
3434  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
3435  void* GetUserData() const { return m_pUserData; }
3436  void SetUserData(VmaAllocator hAllocator, void* pUserData);
3437  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3438 
3439  VmaDeviceMemoryBlock* GetBlock() const
3440  {
3441  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3442  return m_BlockAllocation.m_Block;
3443  }
3444  VkDeviceSize GetOffset() const;
3445  VkDeviceMemory GetMemory() const;
3446  uint32_t GetMemoryTypeIndex() const;
3447  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3448  void* GetMappedData() const;
3449  bool CanBecomeLost() const;
3450  VmaPool GetPool() const;
3451 
3452  uint32_t GetLastUseFrameIndex() const
3453  {
3454  return m_LastUseFrameIndex.load();
3455  }
3456  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3457  {
3458  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3459  }
3460  /*
3461  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3462  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3463  - Else, returns false.
3464 
3465  If hAllocation is already lost, assert - you should not call it then.
3466  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3467  */
3468  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3469 
3470  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3471  {
3472  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3473  outInfo.blockCount = 1;
3474  outInfo.allocationCount = 1;
3475  outInfo.unusedRangeCount = 0;
3476  outInfo.usedBytes = m_Size;
3477  outInfo.unusedBytes = 0;
3478  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3479  outInfo.unusedRangeSizeMin = UINT64_MAX;
3480  outInfo.unusedRangeSizeMax = 0;
3481  }
3482 
3483  void BlockAllocMap();
3484  void BlockAllocUnmap();
3485  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3486  void DedicatedAllocUnmap(VmaAllocator hAllocator);
3487 
3488 private:
3489  VkDeviceSize m_Alignment;
3490  VkDeviceSize m_Size;
3491  void* m_pUserData;
3492  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3493  uint8_t m_Type; // ALLOCATION_TYPE
3494  uint8_t m_SuballocationType; // VmaSuballocationType
3495  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3496  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3497  uint8_t m_MapCount;
3498  uint8_t m_Flags; // enum FLAGS
3499 
3500  // Allocation out of VmaDeviceMemoryBlock.
3501  struct BlockAllocation
3502  {
3503  VmaPool m_hPool; // Null if belongs to general memory.
3504  VmaDeviceMemoryBlock* m_Block;
3505  VkDeviceSize m_Offset;
3506  bool m_CanBecomeLost;
3507  };
3508 
3509  // Allocation for an object that has its own private VkDeviceMemory.
3510  struct DedicatedAllocation
3511  {
3512  uint32_t m_MemoryTypeIndex;
3513  VkDeviceMemory m_hMemory;
3514  void* m_pMappedData; // Not null means memory is mapped.
3515  };
3516 
3517  union
3518  {
3519  // Allocation out of VmaDeviceMemoryBlock.
3520  BlockAllocation m_BlockAllocation;
3521  // Allocation for an object that has its own private VkDeviceMemory.
3522  DedicatedAllocation m_DedicatedAllocation;
3523  };
3524 
3525  void FreeUserDataString(VmaAllocator hAllocator);
3526 };
3527 
3528 /*
3529 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3530 allocated memory block or free.
3531 */
3532 struct VmaSuballocation
3533 {
3534  VkDeviceSize offset;
3535  VkDeviceSize size;
3536  VmaAllocation hAllocation;
3537  VmaSuballocationType type;
3538 };
3539 
3540 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3541 
3542 // Cost of one additional allocation lost, as equivalent in bytes.
3543 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3544 
3545 /*
3546 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3547 
3548 If canMakeOtherLost was false:
3549 - item points to a FREE suballocation.
3550 - itemsToMakeLostCount is 0.
3551 
3552 If canMakeOtherLost was true:
3553 - item points to first of sequence of suballocations, which are either FREE,
3554  or point to VmaAllocations that can become lost.
3555 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3556  the requested allocation to succeed.
3557 */
3558 struct VmaAllocationRequest
3559 {
3560  VkDeviceSize offset;
3561  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3562  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3563  VmaSuballocationList::iterator item;
3564  size_t itemsToMakeLostCount;
3565 
3566  VkDeviceSize CalcCost() const
3567  {
3568  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3569  }
3570 };
3571 
3572 /*
3573 Data structure used for bookkeeping of allocations and unused ranges of memory
3574 in a single VkDeviceMemory block.
3575 */
3576 class VmaBlockMetadata
3577 {
3578 public:
3579  VmaBlockMetadata(VmaAllocator hAllocator);
3580  ~VmaBlockMetadata();
3581  void Init(VkDeviceSize size);
3582 
3583  // Validates all data structures inside this object. If not valid, returns false.
3584  bool Validate() const;
3585  VkDeviceSize GetSize() const { return m_Size; }
3586  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3587  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3588  VkDeviceSize GetUnusedRangeSizeMax() const;
3589  // Returns true if this block is empty - contains only single free suballocation.
3590  bool IsEmpty() const;
3591 
3592  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3593  void AddPoolStats(VmaPoolStats& inoutStats) const;
3594 
3595 #if VMA_STATS_STRING_ENABLED
3596  void PrintDetailedMap(class VmaJsonWriter& json) const;
3597 #endif
3598 
3599  // Creates trivial request for case when block is empty.
3600  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3601 
3602  // Tries to find a place for suballocation with given parameters inside this block.
3603  // If succeeded, fills pAllocationRequest and returns true.
3604  // If failed, returns false.
3605  bool CreateAllocationRequest(
3606  uint32_t currentFrameIndex,
3607  uint32_t frameInUseCount,
3608  VkDeviceSize bufferImageGranularity,
3609  VkDeviceSize allocSize,
3610  VkDeviceSize allocAlignment,
3611  VmaSuballocationType allocType,
3612  bool canMakeOtherLost,
3613  VmaAllocationRequest* pAllocationRequest);
3614 
3615  bool MakeRequestedAllocationsLost(
3616  uint32_t currentFrameIndex,
3617  uint32_t frameInUseCount,
3618  VmaAllocationRequest* pAllocationRequest);
3619 
3620  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3621 
3622  // Makes actual allocation based on request. Request must already be checked and valid.
3623  void Alloc(
3624  const VmaAllocationRequest& request,
3625  VmaSuballocationType type,
3626  VkDeviceSize allocSize,
3627  VmaAllocation hAllocation);
3628 
3629  // Frees suballocation assigned to given memory region.
3630  void Free(const VmaAllocation allocation);
3631  void FreeAtOffset(VkDeviceSize offset);
3632 
3633 private:
3634  VkDeviceSize m_Size;
3635  uint32_t m_FreeCount;
3636  VkDeviceSize m_SumFreeSize;
3637  VmaSuballocationList m_Suballocations;
3638  // Suballocations that are free and have size greater than certain threshold.
3639  // Sorted by size, ascending.
3640  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3641 
3642  bool ValidateFreeSuballocationList() const;
3643 
3644  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3645  // If yes, fills pOffset and returns true. If no, returns false.
3646  bool CheckAllocation(
3647  uint32_t currentFrameIndex,
3648  uint32_t frameInUseCount,
3649  VkDeviceSize bufferImageGranularity,
3650  VkDeviceSize allocSize,
3651  VkDeviceSize allocAlignment,
3652  VmaSuballocationType allocType,
3653  VmaSuballocationList::const_iterator suballocItem,
3654  bool canMakeOtherLost,
3655  VkDeviceSize* pOffset,
3656  size_t* itemsToMakeLostCount,
3657  VkDeviceSize* pSumFreeSize,
3658  VkDeviceSize* pSumItemSize) const;
3659  // Given free suballocation, it merges it with following one, which must also be free.
3660  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3661  // Releases given suballocation, making it free.
3662  // Merges it with adjacent free suballocations if applicable.
3663  // Returns iterator to new free suballocation at this place.
3664  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3665  // Given free suballocation, it inserts it into sorted list of
3666  // m_FreeSuballocationsBySize if it's suitable.
3667  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3668  // Given free suballocation, it removes it from sorted list of
3669  // m_FreeSuballocationsBySize if it's suitable.
3670  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3671 };
3672 
3673 // Helper class that represents mapped memory. Synchronized internally.
3674 class VmaDeviceMemoryMapping
3675 {
3676 public:
3677  VmaDeviceMemoryMapping();
3678  ~VmaDeviceMemoryMapping();
3679 
3680  void* GetMappedData() const { return m_pMappedData; }
3681 
3682  // ppData can be null.
3683  VkResult Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData);
3684  void Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count);
3685 
3686 private:
3687  VMA_MUTEX m_Mutex;
3688  uint32_t m_MapCount;
3689  void* m_pMappedData;
3690 };
3691 
3692 /*
3693 Represents a single block of device memory (`VkDeviceMemory`) with all the
3694 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3695 
3696 Thread-safety: This class must be externally synchronized.
3697 */
3698 class VmaDeviceMemoryBlock
3699 {
3700 public:
3701  uint32_t m_MemoryTypeIndex;
3702  VkDeviceMemory m_hMemory;
3703  VmaDeviceMemoryMapping m_Mapping;
3704  VmaBlockMetadata m_Metadata;
3705 
3706  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3707 
3708  ~VmaDeviceMemoryBlock()
3709  {
3710  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3711  }
3712 
3713  // Always call after construction.
3714  void Init(
3715  uint32_t newMemoryTypeIndex,
3716  VkDeviceMemory newMemory,
3717  VkDeviceSize newSize);
3718  // Always call before destruction.
3719  void Destroy(VmaAllocator allocator);
3720 
3721  // Validates all data structures inside this object. If not valid, returns false.
3722  bool Validate() const;
3723 
3724  // ppData can be null.
3725  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
3726  void Unmap(VmaAllocator hAllocator, uint32_t count);
3727 };
3728 
3729 struct VmaPointerLess
3730 {
3731  bool operator()(const void* lhs, const void* rhs) const
3732  {
3733  return lhs < rhs;
3734  }
3735 };
3736 
3737 class VmaDefragmentator;
3738 
3739 /*
3740 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3741 Vulkan memory type.
3742 
3743 Synchronized internally with a mutex.
3744 */
3745 struct VmaBlockVector
3746 {
3747  VmaBlockVector(
3748  VmaAllocator hAllocator,
3749  uint32_t memoryTypeIndex,
3750  VkDeviceSize preferredBlockSize,
3751  size_t minBlockCount,
3752  size_t maxBlockCount,
3753  VkDeviceSize bufferImageGranularity,
3754  uint32_t frameInUseCount,
3755  bool isCustomPool);
3756  ~VmaBlockVector();
3757 
3758  VkResult CreateMinBlocks();
3759 
3760  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3761  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3762  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3763  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3764 
3765  void GetPoolStats(VmaPoolStats* pStats);
3766 
3767  bool IsEmpty() const { return m_Blocks.empty(); }
3768 
3769  VkResult Allocate(
3770  VmaPool hCurrentPool,
3771  uint32_t currentFrameIndex,
3772  const VkMemoryRequirements& vkMemReq,
3773  const VmaAllocationCreateInfo& createInfo,
3774  VmaSuballocationType suballocType,
3775  VmaAllocation* pAllocation);
3776 
3777  void Free(
3778  VmaAllocation hAllocation);
3779 
3780  // Adds statistics of this BlockVector to pStats.
3781  void AddStats(VmaStats* pStats);
3782 
3783 #if VMA_STATS_STRING_ENABLED
3784  void PrintDetailedMap(class VmaJsonWriter& json);
3785 #endif
3786 
3787  void MakePoolAllocationsLost(
3788  uint32_t currentFrameIndex,
3789  size_t* pLostAllocationCount);
3790 
3791  VmaDefragmentator* EnsureDefragmentator(
3792  VmaAllocator hAllocator,
3793  uint32_t currentFrameIndex);
3794 
3795  VkResult Defragment(
3796  VmaDefragmentationStats* pDefragmentationStats,
3797  VkDeviceSize& maxBytesToMove,
3798  uint32_t& maxAllocationsToMove);
3799 
3800  void DestroyDefragmentator();
3801 
3802 private:
3803  friend class VmaDefragmentator;
3804 
3805  const VmaAllocator m_hAllocator;
3806  const uint32_t m_MemoryTypeIndex;
3807  const VkDeviceSize m_PreferredBlockSize;
3808  const size_t m_MinBlockCount;
3809  const size_t m_MaxBlockCount;
3810  const VkDeviceSize m_BufferImageGranularity;
3811  const uint32_t m_FrameInUseCount;
3812  const bool m_IsCustomPool;
3813  VMA_MUTEX m_Mutex;
3814  // Incrementally sorted by sumFreeSize, ascending.
3815  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3816  /* There can be at most one allocation that is completely empty - a
3817  hysteresis to avoid pessimistic case of alternating creation and destruction
3818  of a VkDeviceMemory. */
3819  bool m_HasEmptyBlock;
3820  VmaDefragmentator* m_pDefragmentator;
3821 
3822  size_t CalcMaxBlockSize() const;
3823 
3824  // Finds and removes given block from vector.
3825  void Remove(VmaDeviceMemoryBlock* pBlock);
3826 
3827  // Performs single step in sorting m_Blocks. They may not be fully sorted
3828  // after this call.
3829  void IncrementallySortBlocks();
3830 
3831  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3832 };
3833 
3834 struct VmaPool_T
3835 {
3836 public:
3837  VmaBlockVector m_BlockVector;
3838 
3839  // Takes ownership.
3840  VmaPool_T(
3841  VmaAllocator hAllocator,
3842  const VmaPoolCreateInfo& createInfo);
3843  ~VmaPool_T();
3844 
3845  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3846 
3847 #if VMA_STATS_STRING_ENABLED
3848  //void PrintDetailedMap(class VmaStringBuilder& sb);
3849 #endif
3850 };
3851 
3852 class VmaDefragmentator
3853 {
3854  const VmaAllocator m_hAllocator;
3855  VmaBlockVector* const m_pBlockVector;
3856  uint32_t m_CurrentFrameIndex;
3857  VkDeviceSize m_BytesMoved;
3858  uint32_t m_AllocationsMoved;
3859 
3860  struct AllocationInfo
3861  {
3862  VmaAllocation m_hAllocation;
3863  VkBool32* m_pChanged;
3864 
3865  AllocationInfo() :
3866  m_hAllocation(VK_NULL_HANDLE),
3867  m_pChanged(VMA_NULL)
3868  {
3869  }
3870  };
3871 
3872  struct AllocationInfoSizeGreater
3873  {
3874  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3875  {
3876  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3877  }
3878  };
3879 
3880  // Used between AddAllocation and Defragment.
3881  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3882 
3883  struct BlockInfo
3884  {
3885  VmaDeviceMemoryBlock* m_pBlock;
3886  bool m_HasNonMovableAllocations;
3887  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3888 
3889  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3890  m_pBlock(VMA_NULL),
3891  m_HasNonMovableAllocations(true),
3892  m_Allocations(pAllocationCallbacks),
3893  m_pMappedDataForDefragmentation(VMA_NULL)
3894  {
3895  }
3896 
3897  void CalcHasNonMovableAllocations()
3898  {
3899  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3900  const size_t defragmentAllocCount = m_Allocations.size();
3901  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3902  }
3903 
3904  void SortAllocationsBySizeDescecnding()
3905  {
3906  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3907  }
3908 
3909  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3910  void Unmap(VmaAllocator hAllocator);
3911 
3912  private:
3913  // Not null if mapped for defragmentation only, not originally mapped.
3914  void* m_pMappedDataForDefragmentation;
3915  };
3916 
3917  struct BlockPointerLess
3918  {
3919  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3920  {
3921  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3922  }
3923  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3924  {
3925  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3926  }
3927  };
3928 
3929  // 1. Blocks with some non-movable allocations go first.
3930  // 2. Blocks with smaller sumFreeSize go first.
3931  struct BlockInfoCompareMoveDestination
3932  {
3933  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3934  {
3935  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3936  {
3937  return true;
3938  }
3939  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3940  {
3941  return false;
3942  }
3943  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3944  {
3945  return true;
3946  }
3947  return false;
3948  }
3949  };
3950 
3951  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3952  BlockInfoVector m_Blocks;
3953 
3954  VkResult DefragmentRound(
3955  VkDeviceSize maxBytesToMove,
3956  uint32_t maxAllocationsToMove);
3957 
3958  static bool MoveMakesSense(
3959  size_t dstBlockIndex, VkDeviceSize dstOffset,
3960  size_t srcBlockIndex, VkDeviceSize srcOffset);
3961 
3962 public:
3963  VmaDefragmentator(
3964  VmaAllocator hAllocator,
3965  VmaBlockVector* pBlockVector,
3966  uint32_t currentFrameIndex);
3967 
3968  ~VmaDefragmentator();
3969 
3970  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3971  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3972 
3973  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3974 
3975  VkResult Defragment(
3976  VkDeviceSize maxBytesToMove,
3977  uint32_t maxAllocationsToMove);
3978 };
3979 
3980 // Main allocator object.
3981 struct VmaAllocator_T
3982 {
3983  bool m_UseMutex;
3984  bool m_UseKhrDedicatedAllocation;
3985  VkDevice m_hDevice;
3986  bool m_AllocationCallbacksSpecified;
3987  VkAllocationCallbacks m_AllocationCallbacks;
3988  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3989 
3990  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3991  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3992  VMA_MUTEX m_HeapSizeLimitMutex;
3993 
3994  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3995  VkPhysicalDeviceMemoryProperties m_MemProps;
3996 
3997  // Default pools.
3998  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
3999 
4000  // Each vector is sorted by memory (handle value).
4001  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4002  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4003  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4004 
4005  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4006  ~VmaAllocator_T();
4007 
4008  const VkAllocationCallbacks* GetAllocationCallbacks() const
4009  {
4010  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4011  }
4012  const VmaVulkanFunctions& GetVulkanFunctions() const
4013  {
4014  return m_VulkanFunctions;
4015  }
4016 
4017  VkDeviceSize GetBufferImageGranularity() const
4018  {
4019  return VMA_MAX(
4020  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4021  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4022  }
4023 
4024  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
4025  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4026 
4027  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4028  {
4029  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4030  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4031  }
4032 
4033  void GetBufferMemoryRequirements(
4034  VkBuffer hBuffer,
4035  VkMemoryRequirements& memReq,
4036  bool& requiresDedicatedAllocation,
4037  bool& prefersDedicatedAllocation) const;
4038  void GetImageMemoryRequirements(
4039  VkImage hImage,
4040  VkMemoryRequirements& memReq,
4041  bool& requiresDedicatedAllocation,
4042  bool& prefersDedicatedAllocation) const;
4043 
4044  // Main allocation function.
4045  VkResult AllocateMemory(
4046  const VkMemoryRequirements& vkMemReq,
4047  bool requiresDedicatedAllocation,
4048  bool prefersDedicatedAllocation,
4049  VkBuffer dedicatedBuffer,
4050  VkImage dedicatedImage,
4051  const VmaAllocationCreateInfo& createInfo,
4052  VmaSuballocationType suballocType,
4053  VmaAllocation* pAllocation);
4054 
4055  // Main deallocation function.
4056  void FreeMemory(const VmaAllocation allocation);
4057 
4058  void CalculateStats(VmaStats* pStats);
4059 
4060 #if VMA_STATS_STRING_ENABLED
4061  void PrintDetailedMap(class VmaJsonWriter& json);
4062 #endif
4063 
4064  VkResult Defragment(
4065  VmaAllocation* pAllocations,
4066  size_t allocationCount,
4067  VkBool32* pAllocationsChanged,
4068  const VmaDefragmentationInfo* pDefragmentationInfo,
4069  VmaDefragmentationStats* pDefragmentationStats);
4070 
4071  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4072 
4073  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4074  void DestroyPool(VmaPool pool);
4075  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4076 
4077  void SetCurrentFrameIndex(uint32_t frameIndex);
4078 
4079  void MakePoolAllocationsLost(
4080  VmaPool hPool,
4081  size_t* pLostAllocationCount);
4082 
4083  void CreateLostAllocation(VmaAllocation* pAllocation);
4084 
4085  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4086  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4087 
4088  VkResult Map(VmaAllocation hAllocation, void** ppData);
4089  void Unmap(VmaAllocation hAllocation);
4090 
4091 private:
4092  VkDeviceSize m_PreferredLargeHeapBlockSize;
4093 
4094  VkPhysicalDevice m_PhysicalDevice;
4095  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4096 
4097  VMA_MUTEX m_PoolsMutex;
4098  // Protected by m_PoolsMutex. Sorted by pointer value.
4099  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4100 
4101  VmaVulkanFunctions m_VulkanFunctions;
4102 
4103  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4104 
4105  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4106 
4107  VkResult AllocateMemoryOfType(
4108  const VkMemoryRequirements& vkMemReq,
4109  bool dedicatedAllocation,
4110  VkBuffer dedicatedBuffer,
4111  VkImage dedicatedImage,
4112  const VmaAllocationCreateInfo& createInfo,
4113  uint32_t memTypeIndex,
4114  VmaSuballocationType suballocType,
4115  VmaAllocation* pAllocation);
4116 
4117  // Allocates and registers new VkDeviceMemory specifically for single allocation.
4118  VkResult AllocateDedicatedMemory(
4119  VkDeviceSize size,
4120  VmaSuballocationType suballocType,
4121  uint32_t memTypeIndex,
4122  bool map,
4123  bool isUserDataString,
4124  void* pUserData,
4125  VkBuffer dedicatedBuffer,
4126  VkImage dedicatedImage,
4127  VmaAllocation* pAllocation);
4128 
4129  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4130  void FreeDedicatedMemory(VmaAllocation allocation);
4131 };
4132 
4134 // Memory allocation #2 after VmaAllocator_T definition
4135 
4136 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4137 {
4138  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4139 }
4140 
4141 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4142 {
4143  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4144 }
4145 
4146 template<typename T>
4147 static T* VmaAllocate(VmaAllocator hAllocator)
4148 {
4149  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4150 }
4151 
4152 template<typename T>
4153 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4154 {
4155  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4156 }
4157 
4158 template<typename T>
4159 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4160 {
4161  if(ptr != VMA_NULL)
4162  {
4163  ptr->~T();
4164  VmaFree(hAllocator, ptr);
4165  }
4166 }
4167 
4168 template<typename T>
4169 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4170 {
4171  if(ptr != VMA_NULL)
4172  {
4173  for(size_t i = count; i--; )
4174  ptr[i].~T();
4175  VmaFree(hAllocator, ptr);
4176  }
4177 }
4178 
4180 // VmaStringBuilder
4181 
4182 #if VMA_STATS_STRING_ENABLED
4183 
4184 class VmaStringBuilder
4185 {
4186 public:
4187  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
4188  size_t GetLength() const { return m_Data.size(); }
4189  const char* GetData() const { return m_Data.data(); }
4190 
4191  void Add(char ch) { m_Data.push_back(ch); }
4192  void Add(const char* pStr);
4193  void AddNewLine() { Add('\n'); }
4194  void AddNumber(uint32_t num);
4195  void AddNumber(uint64_t num);
4196  void AddPointer(const void* ptr);
4197 
4198 private:
4199  VmaVector< char, VmaStlAllocator<char> > m_Data;
4200 };
4201 
4202 void VmaStringBuilder::Add(const char* pStr)
4203 {
4204  const size_t strLen = strlen(pStr);
4205  if(strLen > 0)
4206  {
4207  const size_t oldCount = m_Data.size();
4208  m_Data.resize(oldCount + strLen);
4209  memcpy(m_Data.data() + oldCount, pStr, strLen);
4210  }
4211 }
4212 
4213 void VmaStringBuilder::AddNumber(uint32_t num)
4214 {
4215  char buf[11];
4216  VmaUint32ToStr(buf, sizeof(buf), num);
4217  Add(buf);
4218 }
4219 
4220 void VmaStringBuilder::AddNumber(uint64_t num)
4221 {
4222  char buf[21];
4223  VmaUint64ToStr(buf, sizeof(buf), num);
4224  Add(buf);
4225 }
4226 
4227 void VmaStringBuilder::AddPointer(const void* ptr)
4228 {
4229  char buf[21];
4230  VmaPtrToStr(buf, sizeof(buf), ptr);
4231  Add(buf);
4232 }
4233 
4234 #endif // #if VMA_STATS_STRING_ENABLED
4235 
4237 // VmaJsonWriter
4238 
4239 #if VMA_STATS_STRING_ENABLED
4240 
4241 class VmaJsonWriter
4242 {
4243 public:
4244  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4245  ~VmaJsonWriter();
4246 
4247  void BeginObject(bool singleLine = false);
4248  void EndObject();
4249 
4250  void BeginArray(bool singleLine = false);
4251  void EndArray();
4252 
4253  void WriteString(const char* pStr);
4254  void BeginString(const char* pStr = VMA_NULL);
4255  void ContinueString(const char* pStr);
4256  void ContinueString(uint32_t n);
4257  void ContinueString(uint64_t n);
4258  void ContinueString_Pointer(const void* ptr);
4259  void EndString(const char* pStr = VMA_NULL);
4260 
4261  void WriteNumber(uint32_t n);
4262  void WriteNumber(uint64_t n);
4263  void WriteBool(bool b);
4264  void WriteNull();
4265 
4266 private:
4267  static const char* const INDENT;
4268 
4269  enum COLLECTION_TYPE
4270  {
4271  COLLECTION_TYPE_OBJECT,
4272  COLLECTION_TYPE_ARRAY,
4273  };
4274  struct StackItem
4275  {
4276  COLLECTION_TYPE type;
4277  uint32_t valueCount;
4278  bool singleLineMode;
4279  };
4280 
4281  VmaStringBuilder& m_SB;
4282  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4283  bool m_InsideString;
4284 
4285  void BeginValue(bool isString);
4286  void WriteIndent(bool oneLess = false);
4287 };
4288 
4289 const char* const VmaJsonWriter::INDENT = " ";
4290 
4291 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4292  m_SB(sb),
4293  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4294  m_InsideString(false)
4295 {
4296 }
4297 
4298 VmaJsonWriter::~VmaJsonWriter()
4299 {
4300  VMA_ASSERT(!m_InsideString);
4301  VMA_ASSERT(m_Stack.empty());
4302 }
4303 
4304 void VmaJsonWriter::BeginObject(bool singleLine)
4305 {
4306  VMA_ASSERT(!m_InsideString);
4307 
4308  BeginValue(false);
4309  m_SB.Add('{');
4310 
4311  StackItem item;
4312  item.type = COLLECTION_TYPE_OBJECT;
4313  item.valueCount = 0;
4314  item.singleLineMode = singleLine;
4315  m_Stack.push_back(item);
4316 }
4317 
4318 void VmaJsonWriter::EndObject()
4319 {
4320  VMA_ASSERT(!m_InsideString);
4321 
4322  WriteIndent(true);
4323  m_SB.Add('}');
4324 
4325  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4326  m_Stack.pop_back();
4327 }
4328 
4329 void VmaJsonWriter::BeginArray(bool singleLine)
4330 {
4331  VMA_ASSERT(!m_InsideString);
4332 
4333  BeginValue(false);
4334  m_SB.Add('[');
4335 
4336  StackItem item;
4337  item.type = COLLECTION_TYPE_ARRAY;
4338  item.valueCount = 0;
4339  item.singleLineMode = singleLine;
4340  m_Stack.push_back(item);
4341 }
4342 
4343 void VmaJsonWriter::EndArray()
4344 {
4345  VMA_ASSERT(!m_InsideString);
4346 
4347  WriteIndent(true);
4348  m_SB.Add(']');
4349 
4350  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4351  m_Stack.pop_back();
4352 }
4353 
4354 void VmaJsonWriter::WriteString(const char* pStr)
4355 {
4356  BeginString(pStr);
4357  EndString();
4358 }
4359 
4360 void VmaJsonWriter::BeginString(const char* pStr)
4361 {
4362  VMA_ASSERT(!m_InsideString);
4363 
4364  BeginValue(true);
4365  m_SB.Add('"');
4366  m_InsideString = true;
4367  if(pStr != VMA_NULL && pStr[0] != '\0')
4368  {
4369  ContinueString(pStr);
4370  }
4371 }
4372 
4373 void VmaJsonWriter::ContinueString(const char* pStr)
4374 {
4375  VMA_ASSERT(m_InsideString);
4376 
4377  const size_t strLen = strlen(pStr);
4378  for(size_t i = 0; i < strLen; ++i)
4379  {
4380  char ch = pStr[i];
4381  if(ch == '\'')
4382  {
4383  m_SB.Add("\\\\");
4384  }
4385  else if(ch == '"')
4386  {
4387  m_SB.Add("\\\"");
4388  }
4389  else if(ch >= 32)
4390  {
4391  m_SB.Add(ch);
4392  }
4393  else switch(ch)
4394  {
4395  case '\b':
4396  m_SB.Add("\\b");
4397  break;
4398  case '\f':
4399  m_SB.Add("\\f");
4400  break;
4401  case '\n':
4402  m_SB.Add("\\n");
4403  break;
4404  case '\r':
4405  m_SB.Add("\\r");
4406  break;
4407  case '\t':
4408  m_SB.Add("\\t");
4409  break;
4410  default:
4411  VMA_ASSERT(0 && "Character not currently supported.");
4412  break;
4413  }
4414  }
4415 }
4416 
4417 void VmaJsonWriter::ContinueString(uint32_t n)
4418 {
4419  VMA_ASSERT(m_InsideString);
4420  m_SB.AddNumber(n);
4421 }
4422 
4423 void VmaJsonWriter::ContinueString(uint64_t n)
4424 {
4425  VMA_ASSERT(m_InsideString);
4426  m_SB.AddNumber(n);
4427 }
4428 
4429 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4430 {
4431  VMA_ASSERT(m_InsideString);
4432  m_SB.AddPointer(ptr);
4433 }
4434 
4435 void VmaJsonWriter::EndString(const char* pStr)
4436 {
4437  VMA_ASSERT(m_InsideString);
4438  if(pStr != VMA_NULL && pStr[0] != '\0')
4439  {
4440  ContinueString(pStr);
4441  }
4442  m_SB.Add('"');
4443  m_InsideString = false;
4444 }
4445 
4446 void VmaJsonWriter::WriteNumber(uint32_t n)
4447 {
4448  VMA_ASSERT(!m_InsideString);
4449  BeginValue(false);
4450  m_SB.AddNumber(n);
4451 }
4452 
4453 void VmaJsonWriter::WriteNumber(uint64_t n)
4454 {
4455  VMA_ASSERT(!m_InsideString);
4456  BeginValue(false);
4457  m_SB.AddNumber(n);
4458 }
4459 
4460 void VmaJsonWriter::WriteBool(bool b)
4461 {
4462  VMA_ASSERT(!m_InsideString);
4463  BeginValue(false);
4464  m_SB.Add(b ? "true" : "false");
4465 }
4466 
4467 void VmaJsonWriter::WriteNull()
4468 {
4469  VMA_ASSERT(!m_InsideString);
4470  BeginValue(false);
4471  m_SB.Add("null");
4472 }
4473 
4474 void VmaJsonWriter::BeginValue(bool isString)
4475 {
4476  if(!m_Stack.empty())
4477  {
4478  StackItem& currItem = m_Stack.back();
4479  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4480  currItem.valueCount % 2 == 0)
4481  {
4482  VMA_ASSERT(isString);
4483  }
4484 
4485  if(currItem.type == COLLECTION_TYPE_OBJECT &&
4486  currItem.valueCount % 2 != 0)
4487  {
4488  m_SB.Add(": ");
4489  }
4490  else if(currItem.valueCount > 0)
4491  {
4492  m_SB.Add(", ");
4493  WriteIndent();
4494  }
4495  else
4496  {
4497  WriteIndent();
4498  }
4499  ++currItem.valueCount;
4500  }
4501 }
4502 
4503 void VmaJsonWriter::WriteIndent(bool oneLess)
4504 {
4505  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4506  {
4507  m_SB.AddNewLine();
4508 
4509  size_t count = m_Stack.size();
4510  if(count > 0 && oneLess)
4511  {
4512  --count;
4513  }
4514  for(size_t i = 0; i < count; ++i)
4515  {
4516  m_SB.Add(INDENT);
4517  }
4518  }
4519 }
4520 
4521 #endif // #if VMA_STATS_STRING_ENABLED
4522 
4524 
4525 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4526 {
4527  if(IsUserDataString())
4528  {
4529  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4530 
4531  FreeUserDataString(hAllocator);
4532 
4533  if(pUserData != VMA_NULL)
4534  {
4535  const char* const newStrSrc = (char*)pUserData;
4536  const size_t newStrLen = strlen(newStrSrc);
4537  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4538  memcpy(newStrDst, newStrSrc, newStrLen + 1);
4539  m_pUserData = newStrDst;
4540  }
4541  }
4542  else
4543  {
4544  m_pUserData = pUserData;
4545  }
4546 }
4547 
4548 void VmaAllocation_T::ChangeBlockAllocation(
4549  VmaAllocator hAllocator,
4550  VmaDeviceMemoryBlock* block,
4551  VkDeviceSize offset)
4552 {
4553  VMA_ASSERT(block != VMA_NULL);
4554  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4555 
4556  // Move mapping reference counter from old block to new block.
4557  if(block != m_BlockAllocation.m_Block)
4558  {
4559  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4560  if(IsPersistentMap())
4561  ++mapRefCount;
4562  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4563  block->Map(hAllocator, mapRefCount, VMA_NULL);
4564  }
4565 
4566  m_BlockAllocation.m_Block = block;
4567  m_BlockAllocation.m_Offset = offset;
4568 }
4569 
4570 VkDeviceSize VmaAllocation_T::GetOffset() const
4571 {
4572  switch(m_Type)
4573  {
4574  case ALLOCATION_TYPE_BLOCK:
4575  return m_BlockAllocation.m_Offset;
4576  case ALLOCATION_TYPE_DEDICATED:
4577  return 0;
4578  default:
4579  VMA_ASSERT(0);
4580  return 0;
4581  }
4582 }
4583 
4584 VkDeviceMemory VmaAllocation_T::GetMemory() const
4585 {
4586  switch(m_Type)
4587  {
4588  case ALLOCATION_TYPE_BLOCK:
4589  return m_BlockAllocation.m_Block->m_hMemory;
4590  case ALLOCATION_TYPE_DEDICATED:
4591  return m_DedicatedAllocation.m_hMemory;
4592  default:
4593  VMA_ASSERT(0);
4594  return VK_NULL_HANDLE;
4595  }
4596 }
4597 
4598 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
4599 {
4600  switch(m_Type)
4601  {
4602  case ALLOCATION_TYPE_BLOCK:
4603  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
4604  case ALLOCATION_TYPE_DEDICATED:
4605  return m_DedicatedAllocation.m_MemoryTypeIndex;
4606  default:
4607  VMA_ASSERT(0);
4608  return UINT32_MAX;
4609  }
4610 }
4611 
4612 void* VmaAllocation_T::GetMappedData() const
4613 {
4614  switch(m_Type)
4615  {
4616  case ALLOCATION_TYPE_BLOCK:
4617  if(m_MapCount != 0)
4618  {
4619  void* pBlockData = m_BlockAllocation.m_Block->m_Mapping.GetMappedData();
4620  VMA_ASSERT(pBlockData != VMA_NULL);
4621  return (char*)pBlockData + m_BlockAllocation.m_Offset;
4622  }
4623  else
4624  {
4625  return VMA_NULL;
4626  }
4627  break;
4628  case ALLOCATION_TYPE_DEDICATED:
4629  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
4630  return m_DedicatedAllocation.m_pMappedData;
4631  default:
4632  VMA_ASSERT(0);
4633  return VMA_NULL;
4634  }
4635 }
4636 
4637 bool VmaAllocation_T::CanBecomeLost() const
4638 {
4639  switch(m_Type)
4640  {
4641  case ALLOCATION_TYPE_BLOCK:
4642  return m_BlockAllocation.m_CanBecomeLost;
4643  case ALLOCATION_TYPE_DEDICATED:
4644  return false;
4645  default:
4646  VMA_ASSERT(0);
4647  return false;
4648  }
4649 }
4650 
4651 VmaPool VmaAllocation_T::GetPool() const
4652 {
4653  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4654  return m_BlockAllocation.m_hPool;
4655 }
4656 
4657 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4658 {
4659  VMA_ASSERT(CanBecomeLost());
4660 
4661  /*
4662  Warning: This is a carefully designed algorithm.
4663  Do not modify unless you really know what you're doing :)
4664  */
4665  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4666  for(;;)
4667  {
4668  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4669  {
4670  VMA_ASSERT(0);
4671  return false;
4672  }
4673  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4674  {
4675  return false;
4676  }
4677  else // Last use time earlier than current time.
4678  {
4679  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4680  {
4681  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4682  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4683  return true;
4684  }
4685  }
4686  }
4687 }
4688 
4689 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
4690 {
4691  VMA_ASSERT(IsUserDataString());
4692  if(m_pUserData != VMA_NULL)
4693  {
4694  char* const oldStr = (char*)m_pUserData;
4695  const size_t oldStrLen = strlen(oldStr);
4696  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
4697  m_pUserData = VMA_NULL;
4698  }
4699 }
4700 
4701 void VmaAllocation_T::BlockAllocMap()
4702 {
4703  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4704 
4705  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4706  {
4707  ++m_MapCount;
4708  }
4709  else
4710  {
4711  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
4712  }
4713 }
4714 
4715 void VmaAllocation_T::BlockAllocUnmap()
4716 {
4717  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
4718 
4719  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4720  {
4721  --m_MapCount;
4722  }
4723  else
4724  {
4725  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
4726  }
4727 }
4728 
4729 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
4730 {
4731  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4732 
4733  if(m_MapCount != 0)
4734  {
4735  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
4736  {
4737  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
4738  *ppData = m_DedicatedAllocation.m_pMappedData;
4739  ++m_MapCount;
4740  return VK_SUCCESS;
4741  }
4742  else
4743  {
4744  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
4745  return VK_ERROR_MEMORY_MAP_FAILED;
4746  }
4747  }
4748  else
4749  {
4750  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4751  hAllocator->m_hDevice,
4752  m_DedicatedAllocation.m_hMemory,
4753  0, // offset
4754  VK_WHOLE_SIZE,
4755  0, // flags
4756  ppData);
4757  if(result == VK_SUCCESS)
4758  {
4759  m_DedicatedAllocation.m_pMappedData = *ppData;
4760  m_MapCount = 1;
4761  }
4762  return result;
4763  }
4764 }
4765 
4766 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
4767 {
4768  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
4769 
4770  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
4771  {
4772  --m_MapCount;
4773  if(m_MapCount == 0)
4774  {
4775  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
4776  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
4777  hAllocator->m_hDevice,
4778  m_DedicatedAllocation.m_hMemory);
4779  }
4780  }
4781  else
4782  {
4783  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
4784  }
4785 }
4786 
4787 #if VMA_STATS_STRING_ENABLED
4788 
4789 // Correspond to values of enum VmaSuballocationType.
4790 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4791  "FREE",
4792  "UNKNOWN",
4793  "BUFFER",
4794  "IMAGE_UNKNOWN",
4795  "IMAGE_LINEAR",
4796  "IMAGE_OPTIMAL",
4797 };
4798 
4799 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4800 {
4801  json.BeginObject();
4802 
4803  json.WriteString("Blocks");
4804  json.WriteNumber(stat.blockCount);
4805 
4806  json.WriteString("Allocations");
4807  json.WriteNumber(stat.allocationCount);
4808 
4809  json.WriteString("UnusedRanges");
4810  json.WriteNumber(stat.unusedRangeCount);
4811 
4812  json.WriteString("UsedBytes");
4813  json.WriteNumber(stat.usedBytes);
4814 
4815  json.WriteString("UnusedBytes");
4816  json.WriteNumber(stat.unusedBytes);
4817 
4818  if(stat.allocationCount > 1)
4819  {
4820  json.WriteString("AllocationSize");
4821  json.BeginObject(true);
4822  json.WriteString("Min");
4823  json.WriteNumber(stat.allocationSizeMin);
4824  json.WriteString("Avg");
4825  json.WriteNumber(stat.allocationSizeAvg);
4826  json.WriteString("Max");
4827  json.WriteNumber(stat.allocationSizeMax);
4828  json.EndObject();
4829  }
4830 
4831  if(stat.unusedRangeCount > 1)
4832  {
4833  json.WriteString("UnusedRangeSize");
4834  json.BeginObject(true);
4835  json.WriteString("Min");
4836  json.WriteNumber(stat.unusedRangeSizeMin);
4837  json.WriteString("Avg");
4838  json.WriteNumber(stat.unusedRangeSizeAvg);
4839  json.WriteString("Max");
4840  json.WriteNumber(stat.unusedRangeSizeMax);
4841  json.EndObject();
4842  }
4843 
4844  json.EndObject();
4845 }
4846 
4847 #endif // #if VMA_STATS_STRING_ENABLED
4848 
4849 struct VmaSuballocationItemSizeLess
4850 {
4851  bool operator()(
4852  const VmaSuballocationList::iterator lhs,
4853  const VmaSuballocationList::iterator rhs) const
4854  {
4855  return lhs->size < rhs->size;
4856  }
4857  bool operator()(
4858  const VmaSuballocationList::iterator lhs,
4859  VkDeviceSize rhsSize) const
4860  {
4861  return lhs->size < rhsSize;
4862  }
4863 };
4864 
4866 // class VmaBlockMetadata
4867 
4868 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4869  m_Size(0),
4870  m_FreeCount(0),
4871  m_SumFreeSize(0),
4872  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4873  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4874 {
4875 }
4876 
4877 VmaBlockMetadata::~VmaBlockMetadata()
4878 {
4879 }
4880 
4881 void VmaBlockMetadata::Init(VkDeviceSize size)
4882 {
4883  m_Size = size;
4884  m_FreeCount = 1;
4885  m_SumFreeSize = size;
4886 
4887  VmaSuballocation suballoc = {};
4888  suballoc.offset = 0;
4889  suballoc.size = size;
4890  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4891  suballoc.hAllocation = VK_NULL_HANDLE;
4892 
4893  m_Suballocations.push_back(suballoc);
4894  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4895  --suballocItem;
4896  m_FreeSuballocationsBySize.push_back(suballocItem);
4897 }
4898 
4899 bool VmaBlockMetadata::Validate() const
4900 {
4901  if(m_Suballocations.empty())
4902  {
4903  return false;
4904  }
4905 
4906  // Expected offset of new suballocation as calculates from previous ones.
4907  VkDeviceSize calculatedOffset = 0;
4908  // Expected number of free suballocations as calculated from traversing their list.
4909  uint32_t calculatedFreeCount = 0;
4910  // Expected sum size of free suballocations as calculated from traversing their list.
4911  VkDeviceSize calculatedSumFreeSize = 0;
4912  // Expected number of free suballocations that should be registered in
4913  // m_FreeSuballocationsBySize calculated from traversing their list.
4914  size_t freeSuballocationsToRegister = 0;
4915  // True if previous visisted suballocation was free.
4916  bool prevFree = false;
4917 
4918  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4919  suballocItem != m_Suballocations.cend();
4920  ++suballocItem)
4921  {
4922  const VmaSuballocation& subAlloc = *suballocItem;
4923 
4924  // Actual offset of this suballocation doesn't match expected one.
4925  if(subAlloc.offset != calculatedOffset)
4926  {
4927  return false;
4928  }
4929 
4930  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4931  // Two adjacent free suballocations are invalid. They should be merged.
4932  if(prevFree && currFree)
4933  {
4934  return false;
4935  }
4936 
4937  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4938  {
4939  return false;
4940  }
4941 
4942  if(currFree)
4943  {
4944  calculatedSumFreeSize += subAlloc.size;
4945  ++calculatedFreeCount;
4946  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4947  {
4948  ++freeSuballocationsToRegister;
4949  }
4950  }
4951  else
4952  {
4953  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
4954  {
4955  return false;
4956  }
4957  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
4958  {
4959  return false;
4960  }
4961  }
4962 
4963  calculatedOffset += subAlloc.size;
4964  prevFree = currFree;
4965  }
4966 
4967  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4968  // match expected one.
4969  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4970  {
4971  return false;
4972  }
4973 
4974  VkDeviceSize lastSize = 0;
4975  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4976  {
4977  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4978 
4979  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4980  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4981  {
4982  return false;
4983  }
4984  // They must be sorted by size ascending.
4985  if(suballocItem->size < lastSize)
4986  {
4987  return false;
4988  }
4989 
4990  lastSize = suballocItem->size;
4991  }
4992 
4993  // Check if totals match calculacted values.
4994  if(!ValidateFreeSuballocationList() ||
4995  (calculatedOffset != m_Size) ||
4996  (calculatedSumFreeSize != m_SumFreeSize) ||
4997  (calculatedFreeCount != m_FreeCount))
4998  {
4999  return false;
5000  }
5001 
5002  return true;
5003 }
5004 
5005 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5006 {
5007  if(!m_FreeSuballocationsBySize.empty())
5008  {
5009  return m_FreeSuballocationsBySize.back()->size;
5010  }
5011  else
5012  {
5013  return 0;
5014  }
5015 }
5016 
5017 bool VmaBlockMetadata::IsEmpty() const
5018 {
5019  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5020 }
5021 
5022 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5023 {
5024  outInfo.blockCount = 1;
5025 
5026  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5027  outInfo.allocationCount = rangeCount - m_FreeCount;
5028  outInfo.unusedRangeCount = m_FreeCount;
5029 
5030  outInfo.unusedBytes = m_SumFreeSize;
5031  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5032 
5033  outInfo.allocationSizeMin = UINT64_MAX;
5034  outInfo.allocationSizeMax = 0;
5035  outInfo.unusedRangeSizeMin = UINT64_MAX;
5036  outInfo.unusedRangeSizeMax = 0;
5037 
5038  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5039  suballocItem != m_Suballocations.cend();
5040  ++suballocItem)
5041  {
5042  const VmaSuballocation& suballoc = *suballocItem;
5043  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5044  {
5045  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5046  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5047  }
5048  else
5049  {
5050  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5051  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5052  }
5053  }
5054 }
5055 
5056 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5057 {
5058  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5059 
5060  inoutStats.size += m_Size;
5061  inoutStats.unusedSize += m_SumFreeSize;
5062  inoutStats.allocationCount += rangeCount - m_FreeCount;
5063  inoutStats.unusedRangeCount += m_FreeCount;
5064  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5065 }
5066 
5067 #if VMA_STATS_STRING_ENABLED
5068 
5069 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5070 {
5071  json.BeginObject();
5072 
5073  json.WriteString("TotalBytes");
5074  json.WriteNumber(m_Size);
5075 
5076  json.WriteString("UnusedBytes");
5077  json.WriteNumber(m_SumFreeSize);
5078 
5079  json.WriteString("Allocations");
5080  json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5081 
5082  json.WriteString("UnusedRanges");
5083  json.WriteNumber(m_FreeCount);
5084 
5085  json.WriteString("Suballocations");
5086  json.BeginArray();
5087  size_t i = 0;
5088  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5089  suballocItem != m_Suballocations.cend();
5090  ++suballocItem, ++i)
5091  {
5092  json.BeginObject(true);
5093 
5094  json.WriteString("Type");
5095  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5096 
5097  json.WriteString("Size");
5098  json.WriteNumber(suballocItem->size);
5099 
5100  json.WriteString("Offset");
5101  json.WriteNumber(suballocItem->offset);
5102 
5103  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5104  {
5105  const void* pUserData = suballocItem->hAllocation->GetUserData();
5106  if(pUserData != VMA_NULL)
5107  {
5108  json.WriteString("UserData");
5109  if(suballocItem->hAllocation->IsUserDataString())
5110  {
5111  json.WriteString((const char*)pUserData);
5112  }
5113  else
5114  {
5115  json.BeginString();
5116  json.ContinueString_Pointer(pUserData);
5117  json.EndString();
5118  }
5119  }
5120  }
5121 
5122  json.EndObject();
5123  }
5124  json.EndArray();
5125 
5126  json.EndObject();
5127 }
5128 
5129 #endif // #if VMA_STATS_STRING_ENABLED
5130 
5131 /*
5132 How many suitable free suballocations to analyze before choosing best one.
5133 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5134  be chosen.
5135 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5136  suballocations will be analized and best one will be chosen.
5137 - Any other value is also acceptable.
5138 */
5139 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5140 
5141 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5142 {
5143  VMA_ASSERT(IsEmpty());
5144  pAllocationRequest->offset = 0;
5145  pAllocationRequest->sumFreeSize = m_SumFreeSize;
5146  pAllocationRequest->sumItemSize = 0;
5147  pAllocationRequest->item = m_Suballocations.begin();
5148  pAllocationRequest->itemsToMakeLostCount = 0;
5149 }
5150 
5151 bool VmaBlockMetadata::CreateAllocationRequest(
5152  uint32_t currentFrameIndex,
5153  uint32_t frameInUseCount,
5154  VkDeviceSize bufferImageGranularity,
5155  VkDeviceSize allocSize,
5156  VkDeviceSize allocAlignment,
5157  VmaSuballocationType allocType,
5158  bool canMakeOtherLost,
5159  VmaAllocationRequest* pAllocationRequest)
5160 {
5161  VMA_ASSERT(allocSize > 0);
5162  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5163  VMA_ASSERT(pAllocationRequest != VMA_NULL);
5164  VMA_HEAVY_ASSERT(Validate());
5165 
5166  // There is not enough total free space in this block to fullfill the request: Early return.
5167  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5168  {
5169  return false;
5170  }
5171 
5172  // New algorithm, efficiently searching freeSuballocationsBySize.
5173  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5174  if(freeSuballocCount > 0)
5175  {
5176  if(VMA_BEST_FIT)
5177  {
5178  // Find first free suballocation with size not less than allocSize.
5179  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5180  m_FreeSuballocationsBySize.data(),
5181  m_FreeSuballocationsBySize.data() + freeSuballocCount,
5182  allocSize,
5183  VmaSuballocationItemSizeLess());
5184  size_t index = it - m_FreeSuballocationsBySize.data();
5185  for(; index < freeSuballocCount; ++index)
5186  {
5187  if(CheckAllocation(
5188  currentFrameIndex,
5189  frameInUseCount,
5190  bufferImageGranularity,
5191  allocSize,
5192  allocAlignment,
5193  allocType,
5194  m_FreeSuballocationsBySize[index],
5195  false, // canMakeOtherLost
5196  &pAllocationRequest->offset,
5197  &pAllocationRequest->itemsToMakeLostCount,
5198  &pAllocationRequest->sumFreeSize,
5199  &pAllocationRequest->sumItemSize))
5200  {
5201  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5202  return true;
5203  }
5204  }
5205  }
5206  else
5207  {
5208  // Search staring from biggest suballocations.
5209  for(size_t index = freeSuballocCount; index--; )
5210  {
5211  if(CheckAllocation(
5212  currentFrameIndex,
5213  frameInUseCount,
5214  bufferImageGranularity,
5215  allocSize,
5216  allocAlignment,
5217  allocType,
5218  m_FreeSuballocationsBySize[index],
5219  false, // canMakeOtherLost
5220  &pAllocationRequest->offset,
5221  &pAllocationRequest->itemsToMakeLostCount,
5222  &pAllocationRequest->sumFreeSize,
5223  &pAllocationRequest->sumItemSize))
5224  {
5225  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5226  return true;
5227  }
5228  }
5229  }
5230  }
5231 
5232  if(canMakeOtherLost)
5233  {
5234  // Brute-force algorithm. TODO: Come up with something better.
5235 
5236  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5237  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5238 
5239  VmaAllocationRequest tmpAllocRequest = {};
5240  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5241  suballocIt != m_Suballocations.end();
5242  ++suballocIt)
5243  {
5244  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5245  suballocIt->hAllocation->CanBecomeLost())
5246  {
5247  if(CheckAllocation(
5248  currentFrameIndex,
5249  frameInUseCount,
5250  bufferImageGranularity,
5251  allocSize,
5252  allocAlignment,
5253  allocType,
5254  suballocIt,
5255  canMakeOtherLost,
5256  &tmpAllocRequest.offset,
5257  &tmpAllocRequest.itemsToMakeLostCount,
5258  &tmpAllocRequest.sumFreeSize,
5259  &tmpAllocRequest.sumItemSize))
5260  {
5261  tmpAllocRequest.item = suballocIt;
5262 
5263  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5264  {
5265  *pAllocationRequest = tmpAllocRequest;
5266  }
5267  }
5268  }
5269  }
5270 
5271  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5272  {
5273  return true;
5274  }
5275  }
5276 
5277  return false;
5278 }
5279 
5280 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5281  uint32_t currentFrameIndex,
5282  uint32_t frameInUseCount,
5283  VmaAllocationRequest* pAllocationRequest)
5284 {
5285  while(pAllocationRequest->itemsToMakeLostCount > 0)
5286  {
5287  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5288  {
5289  ++pAllocationRequest->item;
5290  }
5291  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5292  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5293  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5294  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5295  {
5296  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5297  --pAllocationRequest->itemsToMakeLostCount;
5298  }
5299  else
5300  {
5301  return false;
5302  }
5303  }
5304 
5305  VMA_HEAVY_ASSERT(Validate());
5306  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5307  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5308 
5309  return true;
5310 }
5311 
5312 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5313 {
5314  uint32_t lostAllocationCount = 0;
5315  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5316  it != m_Suballocations.end();
5317  ++it)
5318  {
5319  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5320  it->hAllocation->CanBecomeLost() &&
5321  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5322  {
5323  it = FreeSuballocation(it);
5324  ++lostAllocationCount;
5325  }
5326  }
5327  return lostAllocationCount;
5328 }
5329 
5330 void VmaBlockMetadata::Alloc(
5331  const VmaAllocationRequest& request,
5332  VmaSuballocationType type,
5333  VkDeviceSize allocSize,
5334  VmaAllocation hAllocation)
5335 {
5336  VMA_ASSERT(request.item != m_Suballocations.end());
5337  VmaSuballocation& suballoc = *request.item;
5338  // Given suballocation is a free block.
5339  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5340  // Given offset is inside this suballocation.
5341  VMA_ASSERT(request.offset >= suballoc.offset);
5342  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5343  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5344  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5345 
5346  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5347  // it to become used.
5348  UnregisterFreeSuballocation(request.item);
5349 
5350  suballoc.offset = request.offset;
5351  suballoc.size = allocSize;
5352  suballoc.type = type;
5353  suballoc.hAllocation = hAllocation;
5354 
5355  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5356  if(paddingEnd)
5357  {
5358  VmaSuballocation paddingSuballoc = {};
5359  paddingSuballoc.offset = request.offset + allocSize;
5360  paddingSuballoc.size = paddingEnd;
5361  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5362  VmaSuballocationList::iterator next = request.item;
5363  ++next;
5364  const VmaSuballocationList::iterator paddingEndItem =
5365  m_Suballocations.insert(next, paddingSuballoc);
5366  RegisterFreeSuballocation(paddingEndItem);
5367  }
5368 
5369  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5370  if(paddingBegin)
5371  {
5372  VmaSuballocation paddingSuballoc = {};
5373  paddingSuballoc.offset = request.offset - paddingBegin;
5374  paddingSuballoc.size = paddingBegin;
5375  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5376  const VmaSuballocationList::iterator paddingBeginItem =
5377  m_Suballocations.insert(request.item, paddingSuballoc);
5378  RegisterFreeSuballocation(paddingBeginItem);
5379  }
5380 
5381  // Update totals.
5382  m_FreeCount = m_FreeCount - 1;
5383  if(paddingBegin > 0)
5384  {
5385  ++m_FreeCount;
5386  }
5387  if(paddingEnd > 0)
5388  {
5389  ++m_FreeCount;
5390  }
5391  m_SumFreeSize -= allocSize;
5392 }
5393 
5394 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5395 {
5396  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5397  suballocItem != m_Suballocations.end();
5398  ++suballocItem)
5399  {
5400  VmaSuballocation& suballoc = *suballocItem;
5401  if(suballoc.hAllocation == allocation)
5402  {
5403  FreeSuballocation(suballocItem);
5404  VMA_HEAVY_ASSERT(Validate());
5405  return;
5406  }
5407  }
5408  VMA_ASSERT(0 && "Not found!");
5409 }
5410 
5411 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5412 {
5413  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5414  suballocItem != m_Suballocations.end();
5415  ++suballocItem)
5416  {
5417  VmaSuballocation& suballoc = *suballocItem;
5418  if(suballoc.offset == offset)
5419  {
5420  FreeSuballocation(suballocItem);
5421  return;
5422  }
5423  }
5424  VMA_ASSERT(0 && "Not found!");
5425 }
5426 
5427 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5428 {
5429  VkDeviceSize lastSize = 0;
5430  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5431  {
5432  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5433 
5434  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5435  {
5436  VMA_ASSERT(0);
5437  return false;
5438  }
5439  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5440  {
5441  VMA_ASSERT(0);
5442  return false;
5443  }
5444  if(it->size < lastSize)
5445  {
5446  VMA_ASSERT(0);
5447  return false;
5448  }
5449 
5450  lastSize = it->size;
5451  }
5452  return true;
5453 }
5454 
5455 bool VmaBlockMetadata::CheckAllocation(
5456  uint32_t currentFrameIndex,
5457  uint32_t frameInUseCount,
5458  VkDeviceSize bufferImageGranularity,
5459  VkDeviceSize allocSize,
5460  VkDeviceSize allocAlignment,
5461  VmaSuballocationType allocType,
5462  VmaSuballocationList::const_iterator suballocItem,
5463  bool canMakeOtherLost,
5464  VkDeviceSize* pOffset,
5465  size_t* itemsToMakeLostCount,
5466  VkDeviceSize* pSumFreeSize,
5467  VkDeviceSize* pSumItemSize) const
5468 {
5469  VMA_ASSERT(allocSize > 0);
5470  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5471  VMA_ASSERT(suballocItem != m_Suballocations.cend());
5472  VMA_ASSERT(pOffset != VMA_NULL);
5473 
5474  *itemsToMakeLostCount = 0;
5475  *pSumFreeSize = 0;
5476  *pSumItemSize = 0;
5477 
5478  if(canMakeOtherLost)
5479  {
5480  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5481  {
5482  *pSumFreeSize = suballocItem->size;
5483  }
5484  else
5485  {
5486  if(suballocItem->hAllocation->CanBecomeLost() &&
5487  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5488  {
5489  ++*itemsToMakeLostCount;
5490  *pSumItemSize = suballocItem->size;
5491  }
5492  else
5493  {
5494  return false;
5495  }
5496  }
5497 
5498  // Remaining size is too small for this request: Early return.
5499  if(m_Size - suballocItem->offset < allocSize)
5500  {
5501  return false;
5502  }
5503 
5504  // Start from offset equal to beginning of this suballocation.
5505  *pOffset = suballocItem->offset;
5506 
5507  // Apply VMA_DEBUG_MARGIN at the beginning.
5508  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5509  {
5510  *pOffset += VMA_DEBUG_MARGIN;
5511  }
5512 
5513  // Apply alignment.
5514  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5515  *pOffset = VmaAlignUp(*pOffset, alignment);
5516 
5517  // Check previous suballocations for BufferImageGranularity conflicts.
5518  // Make bigger alignment if necessary.
5519  if(bufferImageGranularity > 1)
5520  {
5521  bool bufferImageGranularityConflict = false;
5522  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5523  while(prevSuballocItem != m_Suballocations.cbegin())
5524  {
5525  --prevSuballocItem;
5526  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5527  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5528  {
5529  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5530  {
5531  bufferImageGranularityConflict = true;
5532  break;
5533  }
5534  }
5535  else
5536  // Already on previous page.
5537  break;
5538  }
5539  if(bufferImageGranularityConflict)
5540  {
5541  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5542  }
5543  }
5544 
5545  // Now that we have final *pOffset, check if we are past suballocItem.
5546  // If yes, return false - this function should be called for another suballocItem as starting point.
5547  if(*pOffset >= suballocItem->offset + suballocItem->size)
5548  {
5549  return false;
5550  }
5551 
5552  // Calculate padding at the beginning based on current offset.
5553  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5554 
5555  // Calculate required margin at the end if this is not last suballocation.
5556  VmaSuballocationList::const_iterator next = suballocItem;
5557  ++next;
5558  const VkDeviceSize requiredEndMargin =
5559  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5560 
5561  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5562  // Another early return check.
5563  if(suballocItem->offset + totalSize > m_Size)
5564  {
5565  return false;
5566  }
5567 
5568  // Advance lastSuballocItem until desired size is reached.
5569  // Update itemsToMakeLostCount.
5570  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5571  if(totalSize > suballocItem->size)
5572  {
5573  VkDeviceSize remainingSize = totalSize - suballocItem->size;
5574  while(remainingSize > 0)
5575  {
5576  ++lastSuballocItem;
5577  if(lastSuballocItem == m_Suballocations.cend())
5578  {
5579  return false;
5580  }
5581  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5582  {
5583  *pSumFreeSize += lastSuballocItem->size;
5584  }
5585  else
5586  {
5587  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5588  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5589  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5590  {
5591  ++*itemsToMakeLostCount;
5592  *pSumItemSize += lastSuballocItem->size;
5593  }
5594  else
5595  {
5596  return false;
5597  }
5598  }
5599  remainingSize = (lastSuballocItem->size < remainingSize) ?
5600  remainingSize - lastSuballocItem->size : 0;
5601  }
5602  }
5603 
5604  // Check next suballocations for BufferImageGranularity conflicts.
5605  // If conflict exists, we must mark more allocations lost or fail.
5606  if(bufferImageGranularity > 1)
5607  {
5608  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
5609  ++nextSuballocItem;
5610  while(nextSuballocItem != m_Suballocations.cend())
5611  {
5612  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5613  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5614  {
5615  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5616  {
5617  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
5618  if(nextSuballoc.hAllocation->CanBecomeLost() &&
5619  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5620  {
5621  ++*itemsToMakeLostCount;
5622  }
5623  else
5624  {
5625  return false;
5626  }
5627  }
5628  }
5629  else
5630  {
5631  // Already on next page.
5632  break;
5633  }
5634  ++nextSuballocItem;
5635  }
5636  }
5637  }
5638  else
5639  {
5640  const VmaSuballocation& suballoc = *suballocItem;
5641  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5642 
5643  *pSumFreeSize = suballoc.size;
5644 
5645  // Size of this suballocation is too small for this request: Early return.
5646  if(suballoc.size < allocSize)
5647  {
5648  return false;
5649  }
5650 
5651  // Start from offset equal to beginning of this suballocation.
5652  *pOffset = suballoc.offset;
5653 
5654  // Apply VMA_DEBUG_MARGIN at the beginning.
5655  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5656  {
5657  *pOffset += VMA_DEBUG_MARGIN;
5658  }
5659 
5660  // Apply alignment.
5661  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5662  *pOffset = VmaAlignUp(*pOffset, alignment);
5663 
5664  // Check previous suballocations for BufferImageGranularity conflicts.
5665  // Make bigger alignment if necessary.
5666  if(bufferImageGranularity > 1)
5667  {
5668  bool bufferImageGranularityConflict = false;
5669  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5670  while(prevSuballocItem != m_Suballocations.cbegin())
5671  {
5672  --prevSuballocItem;
5673  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5674  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5675  {
5676  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5677  {
5678  bufferImageGranularityConflict = true;
5679  break;
5680  }
5681  }
5682  else
5683  // Already on previous page.
5684  break;
5685  }
5686  if(bufferImageGranularityConflict)
5687  {
5688  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5689  }
5690  }
5691 
5692  // Calculate padding at the beginning based on current offset.
5693  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
5694 
5695  // Calculate required margin at the end if this is not last suballocation.
5696  VmaSuballocationList::const_iterator next = suballocItem;
5697  ++next;
5698  const VkDeviceSize requiredEndMargin =
5699  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5700 
5701  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
5702  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
5703  {
5704  return false;
5705  }
5706 
5707  // Check next suballocations for BufferImageGranularity conflicts.
5708  // If conflict exists, allocation cannot be made here.
5709  if(bufferImageGranularity > 1)
5710  {
5711  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
5712  ++nextSuballocItem;
5713  while(nextSuballocItem != m_Suballocations.cend())
5714  {
5715  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
5716  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
5717  {
5718  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
5719  {
5720  return false;
5721  }
5722  }
5723  else
5724  {
5725  // Already on next page.
5726  break;
5727  }
5728  ++nextSuballocItem;
5729  }
5730  }
5731  }
5732 
5733  // All tests passed: Success. pOffset is already filled.
5734  return true;
5735 }
5736 
5737 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5738 {
5739  VMA_ASSERT(item != m_Suballocations.end());
5740  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5741 
5742  VmaSuballocationList::iterator nextItem = item;
5743  ++nextItem;
5744  VMA_ASSERT(nextItem != m_Suballocations.end());
5745  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5746 
5747  item->size += nextItem->size;
5748  --m_FreeCount;
5749  m_Suballocations.erase(nextItem);
5750 }
5751 
5752 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5753 {
5754  // Change this suballocation to be marked as free.
5755  VmaSuballocation& suballoc = *suballocItem;
5756  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5757  suballoc.hAllocation = VK_NULL_HANDLE;
5758 
5759  // Update totals.
5760  ++m_FreeCount;
5761  m_SumFreeSize += suballoc.size;
5762 
5763  // Merge with previous and/or next suballocation if it's also free.
5764  bool mergeWithNext = false;
5765  bool mergeWithPrev = false;
5766 
5767  VmaSuballocationList::iterator nextItem = suballocItem;
5768  ++nextItem;
5769  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5770  {
5771  mergeWithNext = true;
5772  }
5773 
5774  VmaSuballocationList::iterator prevItem = suballocItem;
5775  if(suballocItem != m_Suballocations.begin())
5776  {
5777  --prevItem;
5778  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5779  {
5780  mergeWithPrev = true;
5781  }
5782  }
5783 
5784  if(mergeWithNext)
5785  {
5786  UnregisterFreeSuballocation(nextItem);
5787  MergeFreeWithNext(suballocItem);
5788  }
5789 
5790  if(mergeWithPrev)
5791  {
5792  UnregisterFreeSuballocation(prevItem);
5793  MergeFreeWithNext(prevItem);
5794  RegisterFreeSuballocation(prevItem);
5795  return prevItem;
5796  }
5797  else
5798  {
5799  RegisterFreeSuballocation(suballocItem);
5800  return suballocItem;
5801  }
5802 }
5803 
5804 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5805 {
5806  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5807  VMA_ASSERT(item->size > 0);
5808 
5809  // You may want to enable this validation at the beginning or at the end of
5810  // this function, depending on what do you want to check.
5811  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5812 
5813  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5814  {
5815  if(m_FreeSuballocationsBySize.empty())
5816  {
5817  m_FreeSuballocationsBySize.push_back(item);
5818  }
5819  else
5820  {
5821  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5822  }
5823  }
5824 
5825  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5826 }
5827 
5828 
5829 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5830 {
5831  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5832  VMA_ASSERT(item->size > 0);
5833 
5834  // You may want to enable this validation at the beginning or at the end of
5835  // this function, depending on what do you want to check.
5836  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5837 
5838  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5839  {
5840  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5841  m_FreeSuballocationsBySize.data(),
5842  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5843  item,
5844  VmaSuballocationItemSizeLess());
5845  for(size_t index = it - m_FreeSuballocationsBySize.data();
5846  index < m_FreeSuballocationsBySize.size();
5847  ++index)
5848  {
5849  if(m_FreeSuballocationsBySize[index] == item)
5850  {
5851  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5852  return;
5853  }
5854  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5855  }
5856  VMA_ASSERT(0 && "Not found.");
5857  }
5858 
5859  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5860 }
5861 
5863 // class VmaDeviceMemoryMapping
5864 
5865 VmaDeviceMemoryMapping::VmaDeviceMemoryMapping() :
5866  m_MapCount(0),
5867  m_pMappedData(VMA_NULL)
5868 {
5869 }
5870 
5871 VmaDeviceMemoryMapping::~VmaDeviceMemoryMapping()
5872 {
5873  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
5874 }
5875 
5876 VkResult VmaDeviceMemoryMapping::Map(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count, void **ppData)
5877 {
5878  if(count == 0)
5879  {
5880  return VK_SUCCESS;
5881  }
5882 
5883  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5884  if(m_MapCount != 0)
5885  {
5886  m_MapCount += count;
5887  VMA_ASSERT(m_pMappedData != VMA_NULL);
5888  if(ppData != VMA_NULL)
5889  {
5890  *ppData = m_pMappedData;
5891  }
5892  return VK_SUCCESS;
5893  }
5894  else
5895  {
5896  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5897  hAllocator->m_hDevice,
5898  hMemory,
5899  0, // offset
5900  VK_WHOLE_SIZE,
5901  0, // flags
5902  &m_pMappedData);
5903  if(result == VK_SUCCESS)
5904  {
5905  if(ppData != VMA_NULL)
5906  {
5907  *ppData = m_pMappedData;
5908  }
5909  m_MapCount = count;
5910  }
5911  return result;
5912  }
5913 }
5914 
5915 void VmaDeviceMemoryMapping::Unmap(VmaAllocator hAllocator, VkDeviceMemory hMemory, uint32_t count)
5916 {
5917  if(count == 0)
5918  {
5919  return;
5920  }
5921 
5922  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
5923  if(m_MapCount >= count)
5924  {
5925  m_MapCount -= count;
5926  if(m_MapCount == 0)
5927  {
5928  m_pMappedData = VMA_NULL;
5929  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, hMemory);
5930  }
5931  }
5932  else
5933  {
5934  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
5935  }
5936 }
5937 
5939 // class VmaDeviceMemoryBlock
5940 
5941 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5942  m_MemoryTypeIndex(UINT32_MAX),
5943  m_hMemory(VK_NULL_HANDLE),
5944  m_Metadata(hAllocator)
5945 {
5946 }
5947 
5948 void VmaDeviceMemoryBlock::Init(
5949  uint32_t newMemoryTypeIndex,
5950  VkDeviceMemory newMemory,
5951  VkDeviceSize newSize)
5952 {
5953  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5954 
5955  m_MemoryTypeIndex = newMemoryTypeIndex;
5956  m_hMemory = newMemory;
5957 
5958  m_Metadata.Init(newSize);
5959 }
5960 
5961 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5962 {
5963  // This is the most important assert in the entire library.
5964  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5965  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5966 
5967  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5968  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5969  m_hMemory = VK_NULL_HANDLE;
5970 }
5971 
5972 bool VmaDeviceMemoryBlock::Validate() const
5973 {
5974  if((m_hMemory == VK_NULL_HANDLE) ||
5975  (m_Metadata.GetSize() == 0))
5976  {
5977  return false;
5978  }
5979 
5980  return m_Metadata.Validate();
5981 }
5982 
5983 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
5984 {
5985  return m_Mapping.Map(hAllocator, m_hMemory, count, ppData);
5986 }
5987 
5988 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
5989 {
5990  m_Mapping.Unmap(hAllocator, m_hMemory, count);
5991 }
5992 
5993 static void InitStatInfo(VmaStatInfo& outInfo)
5994 {
5995  memset(&outInfo, 0, sizeof(outInfo));
5996  outInfo.allocationSizeMin = UINT64_MAX;
5997  outInfo.unusedRangeSizeMin = UINT64_MAX;
5998 }
5999 
6000 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
6001 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
6002 {
6003  inoutInfo.blockCount += srcInfo.blockCount;
6004  inoutInfo.allocationCount += srcInfo.allocationCount;
6005  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6006  inoutInfo.usedBytes += srcInfo.usedBytes;
6007  inoutInfo.unusedBytes += srcInfo.unusedBytes;
6008  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6009  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6010  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6011  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6012 }
6013 
6014 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6015 {
6016  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6017  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6018  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6019  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6020 }
6021 
6022 VmaPool_T::VmaPool_T(
6023  VmaAllocator hAllocator,
6024  const VmaPoolCreateInfo& createInfo) :
6025  m_BlockVector(
6026  hAllocator,
6027  createInfo.memoryTypeIndex,
6028  createInfo.blockSize,
6029  createInfo.minBlockCount,
6030  createInfo.maxBlockCount,
6031  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6032  createInfo.frameInUseCount,
6033  true) // isCustomPool
6034 {
6035 }
6036 
6037 VmaPool_T::~VmaPool_T()
6038 {
6039 }
6040 
6041 #if VMA_STATS_STRING_ENABLED
6042 
6043 #endif // #if VMA_STATS_STRING_ENABLED
6044 
6045 VmaBlockVector::VmaBlockVector(
6046  VmaAllocator hAllocator,
6047  uint32_t memoryTypeIndex,
6048  VkDeviceSize preferredBlockSize,
6049  size_t minBlockCount,
6050  size_t maxBlockCount,
6051  VkDeviceSize bufferImageGranularity,
6052  uint32_t frameInUseCount,
6053  bool isCustomPool) :
6054  m_hAllocator(hAllocator),
6055  m_MemoryTypeIndex(memoryTypeIndex),
6056  m_PreferredBlockSize(preferredBlockSize),
6057  m_MinBlockCount(minBlockCount),
6058  m_MaxBlockCount(maxBlockCount),
6059  m_BufferImageGranularity(bufferImageGranularity),
6060  m_FrameInUseCount(frameInUseCount),
6061  m_IsCustomPool(isCustomPool),
6062  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6063  m_HasEmptyBlock(false),
6064  m_pDefragmentator(VMA_NULL)
6065 {
6066 }
6067 
6068 VmaBlockVector::~VmaBlockVector()
6069 {
6070  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6071 
6072  for(size_t i = m_Blocks.size(); i--; )
6073  {
6074  m_Blocks[i]->Destroy(m_hAllocator);
6075  vma_delete(m_hAllocator, m_Blocks[i]);
6076  }
6077 }
6078 
6079 VkResult VmaBlockVector::CreateMinBlocks()
6080 {
6081  for(size_t i = 0; i < m_MinBlockCount; ++i)
6082  {
6083  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6084  if(res != VK_SUCCESS)
6085  {
6086  return res;
6087  }
6088  }
6089  return VK_SUCCESS;
6090 }
6091 
6092 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6093 {
6094  pStats->size = 0;
6095  pStats->unusedSize = 0;
6096  pStats->allocationCount = 0;
6097  pStats->unusedRangeCount = 0;
6098  pStats->unusedRangeSizeMax = 0;
6099 
6100  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6101 
6102  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6103  {
6104  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6105  VMA_ASSERT(pBlock);
6106  VMA_HEAVY_ASSERT(pBlock->Validate());
6107  pBlock->m_Metadata.AddPoolStats(*pStats);
6108  }
6109 }
6110 
6111 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6112 
6113 VkResult VmaBlockVector::Allocate(
6114  VmaPool hCurrentPool,
6115  uint32_t currentFrameIndex,
6116  const VkMemoryRequirements& vkMemReq,
6117  const VmaAllocationCreateInfo& createInfo,
6118  VmaSuballocationType suballocType,
6119  VmaAllocation* pAllocation)
6120 {
6121  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6122  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6123 
6124  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6125 
6126  // 1. Search existing allocations. Try to allocate without making other allocations lost.
6127  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6128  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6129  {
6130  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6131  VMA_ASSERT(pCurrBlock);
6132  VmaAllocationRequest currRequest = {};
6133  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6134  currentFrameIndex,
6135  m_FrameInUseCount,
6136  m_BufferImageGranularity,
6137  vkMemReq.size,
6138  vkMemReq.alignment,
6139  suballocType,
6140  false, // canMakeOtherLost
6141  &currRequest))
6142  {
6143  // Allocate from pCurrBlock.
6144  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6145 
6146  if(mapped)
6147  {
6148  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6149  if(res != VK_SUCCESS)
6150  {
6151  return res;
6152  }
6153  }
6154 
6155  // We no longer have an empty Allocation.
6156  if(pCurrBlock->m_Metadata.IsEmpty())
6157  {
6158  m_HasEmptyBlock = false;
6159  }
6160 
6161  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6162  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6163  (*pAllocation)->InitBlockAllocation(
6164  hCurrentPool,
6165  pCurrBlock,
6166  currRequest.offset,
6167  vkMemReq.alignment,
6168  vkMemReq.size,
6169  suballocType,
6170  mapped,
6171  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6172  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6173  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6174  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6175  return VK_SUCCESS;
6176  }
6177  }
6178 
6179  const bool canCreateNewBlock =
6180  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6181  (m_Blocks.size() < m_MaxBlockCount);
6182 
6183  // 2. Try to create new block.
6184  if(canCreateNewBlock)
6185  {
6186  // Calculate optimal size for new block.
6187  VkDeviceSize newBlockSize = m_PreferredBlockSize;
6188  uint32_t newBlockSizeShift = 0;
6189  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6190 
6191  // Allocating blocks of other sizes is allowed only in default pools.
6192  // In custom pools block size is fixed.
6193  if(m_IsCustomPool == false)
6194  {
6195  // Allocate 1/8, 1/4, 1/2 as first blocks.
6196  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6197  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6198  {
6199  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6200  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6201  {
6202  newBlockSize = smallerNewBlockSize;
6203  ++newBlockSizeShift;
6204  }
6205  else
6206  {
6207  break;
6208  }
6209  }
6210  }
6211 
6212  size_t newBlockIndex = 0;
6213  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6214  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6215  if(m_IsCustomPool == false)
6216  {
6217  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6218  {
6219  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6220  if(smallerNewBlockSize >= vkMemReq.size)
6221  {
6222  newBlockSize = smallerNewBlockSize;
6223  ++newBlockSizeShift;
6224  res = CreateBlock(newBlockSize, &newBlockIndex);
6225  }
6226  else
6227  {
6228  break;
6229  }
6230  }
6231  }
6232 
6233  if(res == VK_SUCCESS)
6234  {
6235  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6236  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6237 
6238  if(mapped)
6239  {
6240  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6241  if(res != VK_SUCCESS)
6242  {
6243  return res;
6244  }
6245  }
6246 
6247  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6248  VmaAllocationRequest allocRequest;
6249  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6250  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6251  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6252  (*pAllocation)->InitBlockAllocation(
6253  hCurrentPool,
6254  pBlock,
6255  allocRequest.offset,
6256  vkMemReq.alignment,
6257  vkMemReq.size,
6258  suballocType,
6259  mapped,
6260  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6261  VMA_HEAVY_ASSERT(pBlock->Validate());
6262  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6263  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6264  return VK_SUCCESS;
6265  }
6266  }
6267 
6268  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6269 
6270  // 3. Try to allocate from existing blocks with making other allocations lost.
6271  if(canMakeOtherLost)
6272  {
6273  uint32_t tryIndex = 0;
6274  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6275  {
6276  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6277  VmaAllocationRequest bestRequest = {};
6278  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6279 
6280  // 1. Search existing allocations.
6281  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6282  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6283  {
6284  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6285  VMA_ASSERT(pCurrBlock);
6286  VmaAllocationRequest currRequest = {};
6287  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6288  currentFrameIndex,
6289  m_FrameInUseCount,
6290  m_BufferImageGranularity,
6291  vkMemReq.size,
6292  vkMemReq.alignment,
6293  suballocType,
6294  canMakeOtherLost,
6295  &currRequest))
6296  {
6297  const VkDeviceSize currRequestCost = currRequest.CalcCost();
6298  if(pBestRequestBlock == VMA_NULL ||
6299  currRequestCost < bestRequestCost)
6300  {
6301  pBestRequestBlock = pCurrBlock;
6302  bestRequest = currRequest;
6303  bestRequestCost = currRequestCost;
6304 
6305  if(bestRequestCost == 0)
6306  {
6307  break;
6308  }
6309  }
6310  }
6311  }
6312 
6313  if(pBestRequestBlock != VMA_NULL)
6314  {
6315  if(mapped)
6316  {
6317  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6318  if(res != VK_SUCCESS)
6319  {
6320  return res;
6321  }
6322  }
6323 
6324  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6325  currentFrameIndex,
6326  m_FrameInUseCount,
6327  &bestRequest))
6328  {
6329  // We no longer have an empty Allocation.
6330  if(pBestRequestBlock->m_Metadata.IsEmpty())
6331  {
6332  m_HasEmptyBlock = false;
6333  }
6334  // Allocate from this pBlock.
6335  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6336  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6337  (*pAllocation)->InitBlockAllocation(
6338  hCurrentPool,
6339  pBestRequestBlock,
6340  bestRequest.offset,
6341  vkMemReq.alignment,
6342  vkMemReq.size,
6343  suballocType,
6344  mapped,
6345  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6346  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6347  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6348  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6349  return VK_SUCCESS;
6350  }
6351  // else: Some allocations must have been touched while we are here. Next try.
6352  }
6353  else
6354  {
6355  // Could not find place in any of the blocks - break outer loop.
6356  break;
6357  }
6358  }
6359  /* Maximum number of tries exceeded - a very unlike event when many other
6360  threads are simultaneously touching allocations making it impossible to make
6361  lost at the same time as we try to allocate. */
6362  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6363  {
6364  return VK_ERROR_TOO_MANY_OBJECTS;
6365  }
6366  }
6367 
6368  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6369 }
6370 
6371 void VmaBlockVector::Free(
6372  VmaAllocation hAllocation)
6373 {
6374  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6375 
6376  // Scope for lock.
6377  {
6378  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6379 
6380  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6381 
6382  if(hAllocation->IsPersistentMap())
6383  {
6384  pBlock->m_Mapping.Unmap(m_hAllocator, pBlock->m_hMemory, 1);
6385  }
6386 
6387  pBlock->m_Metadata.Free(hAllocation);
6388  VMA_HEAVY_ASSERT(pBlock->Validate());
6389 
6390  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6391 
6392  // pBlock became empty after this deallocation.
6393  if(pBlock->m_Metadata.IsEmpty())
6394  {
6395  // Already has empty Allocation. We don't want to have two, so delete this one.
6396  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6397  {
6398  pBlockToDelete = pBlock;
6399  Remove(pBlock);
6400  }
6401  // We now have first empty Allocation.
6402  else
6403  {
6404  m_HasEmptyBlock = true;
6405  }
6406  }
6407  // pBlock didn't become empty, but we have another empty block - find and free that one.
6408  // (This is optional, heuristics.)
6409  else if(m_HasEmptyBlock)
6410  {
6411  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6412  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6413  {
6414  pBlockToDelete = pLastBlock;
6415  m_Blocks.pop_back();
6416  m_HasEmptyBlock = false;
6417  }
6418  }
6419 
6420  IncrementallySortBlocks();
6421  }
6422 
6423  // Destruction of a free Allocation. Deferred until this point, outside of mutex
6424  // lock, for performance reason.
6425  if(pBlockToDelete != VMA_NULL)
6426  {
6427  VMA_DEBUG_LOG(" Deleted empty allocation");
6428  pBlockToDelete->Destroy(m_hAllocator);
6429  vma_delete(m_hAllocator, pBlockToDelete);
6430  }
6431 }
6432 
6433 size_t VmaBlockVector::CalcMaxBlockSize() const
6434 {
6435  size_t result = 0;
6436  for(size_t i = m_Blocks.size(); i--; )
6437  {
6438  result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6439  if(result >= m_PreferredBlockSize)
6440  {
6441  break;
6442  }
6443  }
6444  return result;
6445 }
6446 
6447 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6448 {
6449  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6450  {
6451  if(m_Blocks[blockIndex] == pBlock)
6452  {
6453  VmaVectorRemove(m_Blocks, blockIndex);
6454  return;
6455  }
6456  }
6457  VMA_ASSERT(0);
6458 }
6459 
6460 void VmaBlockVector::IncrementallySortBlocks()
6461 {
6462  // Bubble sort only until first swap.
6463  for(size_t i = 1; i < m_Blocks.size(); ++i)
6464  {
6465  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6466  {
6467  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6468  return;
6469  }
6470  }
6471 }
6472 
6473 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6474 {
6475  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6476  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6477  allocInfo.allocationSize = blockSize;
6478  VkDeviceMemory mem = VK_NULL_HANDLE;
6479  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6480  if(res < 0)
6481  {
6482  return res;
6483  }
6484 
6485  // New VkDeviceMemory successfully created.
6486 
6487  // Create new Allocation for it.
6488  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6489  pBlock->Init(
6490  m_MemoryTypeIndex,
6491  mem,
6492  allocInfo.allocationSize);
6493 
6494  m_Blocks.push_back(pBlock);
6495  if(pNewBlockIndex != VMA_NULL)
6496  {
6497  *pNewBlockIndex = m_Blocks.size() - 1;
6498  }
6499 
6500  return VK_SUCCESS;
6501 }
6502 
6503 #if VMA_STATS_STRING_ENABLED
6504 
6505 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6506 {
6507  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6508 
6509  json.BeginObject();
6510 
6511  if(m_IsCustomPool)
6512  {
6513  json.WriteString("MemoryTypeIndex");
6514  json.WriteNumber(m_MemoryTypeIndex);
6515 
6516  json.WriteString("BlockSize");
6517  json.WriteNumber(m_PreferredBlockSize);
6518 
6519  json.WriteString("BlockCount");
6520  json.BeginObject(true);
6521  if(m_MinBlockCount > 0)
6522  {
6523  json.WriteString("Min");
6524  json.WriteNumber((uint64_t)m_MinBlockCount);
6525  }
6526  if(m_MaxBlockCount < SIZE_MAX)
6527  {
6528  json.WriteString("Max");
6529  json.WriteNumber((uint64_t)m_MaxBlockCount);
6530  }
6531  json.WriteString("Cur");
6532  json.WriteNumber((uint64_t)m_Blocks.size());
6533  json.EndObject();
6534 
6535  if(m_FrameInUseCount > 0)
6536  {
6537  json.WriteString("FrameInUseCount");
6538  json.WriteNumber(m_FrameInUseCount);
6539  }
6540  }
6541  else
6542  {
6543  json.WriteString("PreferredBlockSize");
6544  json.WriteNumber(m_PreferredBlockSize);
6545  }
6546 
6547  json.WriteString("Blocks");
6548  json.BeginArray();
6549  for(size_t i = 0; i < m_Blocks.size(); ++i)
6550  {
6551  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6552  }
6553  json.EndArray();
6554 
6555  json.EndObject();
6556 }
6557 
6558 #endif // #if VMA_STATS_STRING_ENABLED
6559 
6560 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6561  VmaAllocator hAllocator,
6562  uint32_t currentFrameIndex)
6563 {
6564  if(m_pDefragmentator == VMA_NULL)
6565  {
6566  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6567  hAllocator,
6568  this,
6569  currentFrameIndex);
6570  }
6571 
6572  return m_pDefragmentator;
6573 }
6574 
6575 VkResult VmaBlockVector::Defragment(
6576  VmaDefragmentationStats* pDefragmentationStats,
6577  VkDeviceSize& maxBytesToMove,
6578  uint32_t& maxAllocationsToMove)
6579 {
6580  if(m_pDefragmentator == VMA_NULL)
6581  {
6582  return VK_SUCCESS;
6583  }
6584 
6585  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6586 
6587  // Defragment.
6588  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
6589 
6590  // Accumulate statistics.
6591  if(pDefragmentationStats != VMA_NULL)
6592  {
6593  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
6594  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
6595  pDefragmentationStats->bytesMoved += bytesMoved;
6596  pDefragmentationStats->allocationsMoved += allocationsMoved;
6597  VMA_ASSERT(bytesMoved <= maxBytesToMove);
6598  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
6599  maxBytesToMove -= bytesMoved;
6600  maxAllocationsToMove -= allocationsMoved;
6601  }
6602 
6603  // Free empty blocks.
6604  m_HasEmptyBlock = false;
6605  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
6606  {
6607  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
6608  if(pBlock->m_Metadata.IsEmpty())
6609  {
6610  if(m_Blocks.size() > m_MinBlockCount)
6611  {
6612  if(pDefragmentationStats != VMA_NULL)
6613  {
6614  ++pDefragmentationStats->deviceMemoryBlocksFreed;
6615  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
6616  }
6617 
6618  VmaVectorRemove(m_Blocks, blockIndex);
6619  pBlock->Destroy(m_hAllocator);
6620  vma_delete(m_hAllocator, pBlock);
6621  }
6622  else
6623  {
6624  m_HasEmptyBlock = true;
6625  }
6626  }
6627  }
6628 
6629  return result;
6630 }
6631 
6632 void VmaBlockVector::DestroyDefragmentator()
6633 {
6634  if(m_pDefragmentator != VMA_NULL)
6635  {
6636  vma_delete(m_hAllocator, m_pDefragmentator);
6637  m_pDefragmentator = VMA_NULL;
6638  }
6639 }
6640 
6641 void VmaBlockVector::MakePoolAllocationsLost(
6642  uint32_t currentFrameIndex,
6643  size_t* pLostAllocationCount)
6644 {
6645  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6646  size_t lostAllocationCount = 0;
6647  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6648  {
6649  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6650  VMA_ASSERT(pBlock);
6651  lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
6652  }
6653  if(pLostAllocationCount != VMA_NULL)
6654  {
6655  *pLostAllocationCount = lostAllocationCount;
6656  }
6657 }
6658 
6659 void VmaBlockVector::AddStats(VmaStats* pStats)
6660 {
6661  const uint32_t memTypeIndex = m_MemoryTypeIndex;
6662  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
6663 
6664  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6665 
6666  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6667  {
6668  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6669  VMA_ASSERT(pBlock);
6670  VMA_HEAVY_ASSERT(pBlock->Validate());
6671  VmaStatInfo allocationStatInfo;
6672  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
6673  VmaAddStatInfo(pStats->total, allocationStatInfo);
6674  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6675  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6676  }
6677 }
6678 
6680 // VmaDefragmentator members definition
6681 
6682 VmaDefragmentator::VmaDefragmentator(
6683  VmaAllocator hAllocator,
6684  VmaBlockVector* pBlockVector,
6685  uint32_t currentFrameIndex) :
6686  m_hAllocator(hAllocator),
6687  m_pBlockVector(pBlockVector),
6688  m_CurrentFrameIndex(currentFrameIndex),
6689  m_BytesMoved(0),
6690  m_AllocationsMoved(0),
6691  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
6692  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
6693 {
6694 }
6695 
6696 VmaDefragmentator::~VmaDefragmentator()
6697 {
6698  for(size_t i = m_Blocks.size(); i--; )
6699  {
6700  vma_delete(m_hAllocator, m_Blocks[i]);
6701  }
6702 }
6703 
6704 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
6705 {
6706  AllocationInfo allocInfo;
6707  allocInfo.m_hAllocation = hAlloc;
6708  allocInfo.m_pChanged = pChanged;
6709  m_Allocations.push_back(allocInfo);
6710 }
6711 
6712 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
6713 {
6714  // It has already been mapped for defragmentation.
6715  if(m_pMappedDataForDefragmentation)
6716  {
6717  *ppMappedData = m_pMappedDataForDefragmentation;
6718  return VK_SUCCESS;
6719  }
6720 
6721  // It is originally mapped.
6722  if(m_pBlock->m_Mapping.GetMappedData())
6723  {
6724  *ppMappedData = m_pBlock->m_Mapping.GetMappedData();
6725  return VK_SUCCESS;
6726  }
6727 
6728  // Map on first usage.
6729  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
6730  *ppMappedData = m_pMappedDataForDefragmentation;
6731  return res;
6732 }
6733 
6734 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
6735 {
6736  if(m_pMappedDataForDefragmentation != VMA_NULL)
6737  {
6738  m_pBlock->Unmap(hAllocator, 1);
6739  }
6740 }
6741 
6742 VkResult VmaDefragmentator::DefragmentRound(
6743  VkDeviceSize maxBytesToMove,
6744  uint32_t maxAllocationsToMove)
6745 {
6746  if(m_Blocks.empty())
6747  {
6748  return VK_SUCCESS;
6749  }
6750 
6751  size_t srcBlockIndex = m_Blocks.size() - 1;
6752  size_t srcAllocIndex = SIZE_MAX;
6753  for(;;)
6754  {
6755  // 1. Find next allocation to move.
6756  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
6757  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
6758  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
6759  {
6760  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
6761  {
6762  // Finished: no more allocations to process.
6763  if(srcBlockIndex == 0)
6764  {
6765  return VK_SUCCESS;
6766  }
6767  else
6768  {
6769  --srcBlockIndex;
6770  srcAllocIndex = SIZE_MAX;
6771  }
6772  }
6773  else
6774  {
6775  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
6776  }
6777  }
6778 
6779  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
6780  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
6781 
6782  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
6783  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
6784  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
6785  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
6786 
6787  // 2. Try to find new place for this allocation in preceding or current block.
6788  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6789  {
6790  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6791  VmaAllocationRequest dstAllocRequest;
6792  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6793  m_CurrentFrameIndex,
6794  m_pBlockVector->GetFrameInUseCount(),
6795  m_pBlockVector->GetBufferImageGranularity(),
6796  size,
6797  alignment,
6798  suballocType,
6799  false, // canMakeOtherLost
6800  &dstAllocRequest) &&
6801  MoveMakesSense(
6802  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6803  {
6804  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6805 
6806  // Reached limit on number of allocations or bytes to move.
6807  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6808  (m_BytesMoved + size > maxBytesToMove))
6809  {
6810  return VK_INCOMPLETE;
6811  }
6812 
6813  void* pDstMappedData = VMA_NULL;
6814  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6815  if(res != VK_SUCCESS)
6816  {
6817  return res;
6818  }
6819 
6820  void* pSrcMappedData = VMA_NULL;
6821  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6822  if(res != VK_SUCCESS)
6823  {
6824  return res;
6825  }
6826 
6827  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6828  memcpy(
6829  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6830  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6831  static_cast<size_t>(size));
6832 
6833  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6834  pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
6835 
6836  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6837 
6838  if(allocInfo.m_pChanged != VMA_NULL)
6839  {
6840  *allocInfo.m_pChanged = VK_TRUE;
6841  }
6842 
6843  ++m_AllocationsMoved;
6844  m_BytesMoved += size;
6845 
6846  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6847 
6848  break;
6849  }
6850  }
6851 
6852  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6853 
6854  if(srcAllocIndex > 0)
6855  {
6856  --srcAllocIndex;
6857  }
6858  else
6859  {
6860  if(srcBlockIndex > 0)
6861  {
6862  --srcBlockIndex;
6863  srcAllocIndex = SIZE_MAX;
6864  }
6865  else
6866  {
6867  return VK_SUCCESS;
6868  }
6869  }
6870  }
6871 }
6872 
6873 VkResult VmaDefragmentator::Defragment(
6874  VkDeviceSize maxBytesToMove,
6875  uint32_t maxAllocationsToMove)
6876 {
6877  if(m_Allocations.empty())
6878  {
6879  return VK_SUCCESS;
6880  }
6881 
6882  // Create block info for each block.
6883  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6884  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6885  {
6886  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6887  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6888  m_Blocks.push_back(pBlockInfo);
6889  }
6890 
6891  // Sort them by m_pBlock pointer value.
6892  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6893 
6894  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6895  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6896  {
6897  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6898  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6899  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6900  {
6901  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6902  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6903  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6904  {
6905  (*it)->m_Allocations.push_back(allocInfo);
6906  }
6907  else
6908  {
6909  VMA_ASSERT(0);
6910  }
6911  }
6912  }
6913  m_Allocations.clear();
6914 
6915  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6916  {
6917  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6918  pBlockInfo->CalcHasNonMovableAllocations();
6919  pBlockInfo->SortAllocationsBySizeDescecnding();
6920  }
6921 
6922  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6923  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6924 
6925  // Execute defragmentation rounds (the main part).
6926  VkResult result = VK_SUCCESS;
6927  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6928  {
6929  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6930  }
6931 
6932  // Unmap blocks that were mapped for defragmentation.
6933  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6934  {
6935  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6936  }
6937 
6938  return result;
6939 }
6940 
6941 bool VmaDefragmentator::MoveMakesSense(
6942  size_t dstBlockIndex, VkDeviceSize dstOffset,
6943  size_t srcBlockIndex, VkDeviceSize srcOffset)
6944 {
6945  if(dstBlockIndex < srcBlockIndex)
6946  {
6947  return true;
6948  }
6949  if(dstBlockIndex > srcBlockIndex)
6950  {
6951  return false;
6952  }
6953  if(dstOffset < srcOffset)
6954  {
6955  return true;
6956  }
6957  return false;
6958 }
6959 
6961 // VmaAllocator_T
6962 
6963 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6964  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6965  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
6966  m_hDevice(pCreateInfo->device),
6967  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6968  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6969  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6970  m_PreferredLargeHeapBlockSize(0),
6971  m_PhysicalDevice(pCreateInfo->physicalDevice),
6972  m_CurrentFrameIndex(0),
6973  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6974 {
6975  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6976 
6977  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6978  memset(&m_MemProps, 0, sizeof(m_MemProps));
6979  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6980 
6981  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6982  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
6983 
6984  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6985  {
6986  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6987  }
6988 
6989  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6990  {
6991  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6992  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6993  }
6994 
6995  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6996 
6997  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6998  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6999 
7000  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
7001  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
7002 
7003  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7004  {
7005  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7006  {
7007  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7008  if(limit != VK_WHOLE_SIZE)
7009  {
7010  m_HeapSizeLimit[heapIndex] = limit;
7011  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7012  {
7013  m_MemProps.memoryHeaps[heapIndex].size = limit;
7014  }
7015  }
7016  }
7017  }
7018 
7019  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7020  {
7021  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7022 
7023  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7024  this,
7025  memTypeIndex,
7026  preferredBlockSize,
7027  0,
7028  SIZE_MAX,
7029  GetBufferImageGranularity(),
7030  pCreateInfo->frameInUseCount,
7031  false); // isCustomPool
7032  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7033  // becase minBlockCount is 0.
7034  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7035  }
7036 }
7037 
7038 VmaAllocator_T::~VmaAllocator_T()
7039 {
7040  VMA_ASSERT(m_Pools.empty());
7041 
7042  for(size_t i = GetMemoryTypeCount(); i--; )
7043  {
7044  vma_delete(this, m_pDedicatedAllocations[i]);
7045  vma_delete(this, m_pBlockVectors[i]);
7046  }
7047 }
7048 
7049 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7050 {
7051 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7052  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7053  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7054  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7055  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7056  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7057  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7058  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7059  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7060  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7061  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7062  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7063  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7064  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7065  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7066  if(m_UseKhrDedicatedAllocation)
7067  {
7068  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7069  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7070  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7071  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7072  }
7073 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7074 
7075 #define VMA_COPY_IF_NOT_NULL(funcName) \
7076  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7077 
7078  if(pVulkanFunctions != VMA_NULL)
7079  {
7080  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7081  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7082  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7083  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7084  VMA_COPY_IF_NOT_NULL(vkMapMemory);
7085  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7086  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7087  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7088  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7089  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7090  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7091  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7092  VMA_COPY_IF_NOT_NULL(vkCreateImage);
7093  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7094  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7095  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7096  }
7097 
7098 #undef VMA_COPY_IF_NOT_NULL
7099 
7100  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7101  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7102  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7103  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7104  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7105  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7106  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7107  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7108  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7109  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7110  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7111  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7112  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7113  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7114  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7115  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7116  if(m_UseKhrDedicatedAllocation)
7117  {
7118  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7119  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7120  }
7121 }
7122 
7123 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7124 {
7125  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7126  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7127  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7128  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7129 }
7130 
7131 VkResult VmaAllocator_T::AllocateMemoryOfType(
7132  const VkMemoryRequirements& vkMemReq,
7133  bool dedicatedAllocation,
7134  VkBuffer dedicatedBuffer,
7135  VkImage dedicatedImage,
7136  const VmaAllocationCreateInfo& createInfo,
7137  uint32_t memTypeIndex,
7138  VmaSuballocationType suballocType,
7139  VmaAllocation* pAllocation)
7140 {
7141  VMA_ASSERT(pAllocation != VMA_NULL);
7142  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7143 
7144  VmaAllocationCreateInfo finalCreateInfo = createInfo;
7145 
7146  // If memory type is not HOST_VISIBLE, disable MAPPED.
7147  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7148  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7149  {
7150  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7151  }
7152 
7153  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7154  VMA_ASSERT(blockVector);
7155 
7156  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7157  bool preferDedicatedMemory =
7158  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7159  dedicatedAllocation ||
7160  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7161  vkMemReq.size > preferredBlockSize / 2;
7162 
7163  if(preferDedicatedMemory &&
7164  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7165  finalCreateInfo.pool == VK_NULL_HANDLE)
7166  {
7167  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7168  }
7169 
7170  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7171  {
7172  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7173  {
7174  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7175  }
7176  else
7177  {
7178  return AllocateDedicatedMemory(
7179  vkMemReq.size,
7180  suballocType,
7181  memTypeIndex,
7182  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7183  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7184  finalCreateInfo.pUserData,
7185  dedicatedBuffer,
7186  dedicatedImage,
7187  pAllocation);
7188  }
7189  }
7190  else
7191  {
7192  VkResult res = blockVector->Allocate(
7193  VK_NULL_HANDLE, // hCurrentPool
7194  m_CurrentFrameIndex.load(),
7195  vkMemReq,
7196  finalCreateInfo,
7197  suballocType,
7198  pAllocation);
7199  if(res == VK_SUCCESS)
7200  {
7201  return res;
7202  }
7203 
7204  // 5. Try dedicated memory.
7205  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7206  {
7207  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7208  }
7209  else
7210  {
7211  res = AllocateDedicatedMemory(
7212  vkMemReq.size,
7213  suballocType,
7214  memTypeIndex,
7215  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7216  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7217  finalCreateInfo.pUserData,
7218  dedicatedBuffer,
7219  dedicatedImage,
7220  pAllocation);
7221  if(res == VK_SUCCESS)
7222  {
7223  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7224  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7225  return VK_SUCCESS;
7226  }
7227  else
7228  {
7229  // Everything failed: Return error code.
7230  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7231  return res;
7232  }
7233  }
7234  }
7235 }
7236 
7237 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7238  VkDeviceSize size,
7239  VmaSuballocationType suballocType,
7240  uint32_t memTypeIndex,
7241  bool map,
7242  bool isUserDataString,
7243  void* pUserData,
7244  VkBuffer dedicatedBuffer,
7245  VkImage dedicatedImage,
7246  VmaAllocation* pAllocation)
7247 {
7248  VMA_ASSERT(pAllocation);
7249 
7250  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7251  allocInfo.memoryTypeIndex = memTypeIndex;
7252  allocInfo.allocationSize = size;
7253 
7254  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7255  if(m_UseKhrDedicatedAllocation)
7256  {
7257  if(dedicatedBuffer != VK_NULL_HANDLE)
7258  {
7259  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7260  dedicatedAllocInfo.buffer = dedicatedBuffer;
7261  allocInfo.pNext = &dedicatedAllocInfo;
7262  }
7263  else if(dedicatedImage != VK_NULL_HANDLE)
7264  {
7265  dedicatedAllocInfo.image = dedicatedImage;
7266  allocInfo.pNext = &dedicatedAllocInfo;
7267  }
7268  }
7269 
7270  // Allocate VkDeviceMemory.
7271  VkDeviceMemory hMemory = VK_NULL_HANDLE;
7272  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7273  if(res < 0)
7274  {
7275  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7276  return res;
7277  }
7278 
7279  void* pMappedData = VMA_NULL;
7280  if(map)
7281  {
7282  res = (*m_VulkanFunctions.vkMapMemory)(
7283  m_hDevice,
7284  hMemory,
7285  0,
7286  VK_WHOLE_SIZE,
7287  0,
7288  &pMappedData);
7289  if(res < 0)
7290  {
7291  VMA_DEBUG_LOG(" vkMapMemory FAILED");
7292  FreeVulkanMemory(memTypeIndex, size, hMemory);
7293  return res;
7294  }
7295  }
7296 
7297  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7298  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7299  (*pAllocation)->SetUserData(this, pUserData);
7300 
7301  // Register it in m_pDedicatedAllocations.
7302  {
7303  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7304  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7305  VMA_ASSERT(pDedicatedAllocations);
7306  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7307  }
7308 
7309  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7310 
7311  return VK_SUCCESS;
7312 }
7313 
7314 void VmaAllocator_T::GetBufferMemoryRequirements(
7315  VkBuffer hBuffer,
7316  VkMemoryRequirements& memReq,
7317  bool& requiresDedicatedAllocation,
7318  bool& prefersDedicatedAllocation) const
7319 {
7320  if(m_UseKhrDedicatedAllocation)
7321  {
7322  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7323  memReqInfo.buffer = hBuffer;
7324 
7325  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7326 
7327  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7328  memReq2.pNext = &memDedicatedReq;
7329 
7330  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7331 
7332  memReq = memReq2.memoryRequirements;
7333  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7334  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7335  }
7336  else
7337  {
7338  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7339  requiresDedicatedAllocation = false;
7340  prefersDedicatedAllocation = false;
7341  }
7342 }
7343 
7344 void VmaAllocator_T::GetImageMemoryRequirements(
7345  VkImage hImage,
7346  VkMemoryRequirements& memReq,
7347  bool& requiresDedicatedAllocation,
7348  bool& prefersDedicatedAllocation) const
7349 {
7350  if(m_UseKhrDedicatedAllocation)
7351  {
7352  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7353  memReqInfo.image = hImage;
7354 
7355  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7356 
7357  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7358  memReq2.pNext = &memDedicatedReq;
7359 
7360  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7361 
7362  memReq = memReq2.memoryRequirements;
7363  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7364  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7365  }
7366  else
7367  {
7368  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7369  requiresDedicatedAllocation = false;
7370  prefersDedicatedAllocation = false;
7371  }
7372 }
7373 
7374 VkResult VmaAllocator_T::AllocateMemory(
7375  const VkMemoryRequirements& vkMemReq,
7376  bool requiresDedicatedAllocation,
7377  bool prefersDedicatedAllocation,
7378  VkBuffer dedicatedBuffer,
7379  VkImage dedicatedImage,
7380  const VmaAllocationCreateInfo& createInfo,
7381  VmaSuballocationType suballocType,
7382  VmaAllocation* pAllocation)
7383 {
7384  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7385  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7386  {
7387  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7388  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7389  }
7390  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7391  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7392  {
7393  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7394  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7395  }
7396  if(requiresDedicatedAllocation)
7397  {
7398  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7399  {
7400  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7401  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7402  }
7403  if(createInfo.pool != VK_NULL_HANDLE)
7404  {
7405  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7406  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7407  }
7408  }
7409  if((createInfo.pool != VK_NULL_HANDLE) &&
7410  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7411  {
7412  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7413  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7414  }
7415 
7416  if(createInfo.pool != VK_NULL_HANDLE)
7417  {
7418  return createInfo.pool->m_BlockVector.Allocate(
7419  createInfo.pool,
7420  m_CurrentFrameIndex.load(),
7421  vkMemReq,
7422  createInfo,
7423  suballocType,
7424  pAllocation);
7425  }
7426  else
7427  {
7428  // Bit mask of memory Vulkan types acceptable for this allocation.
7429  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7430  uint32_t memTypeIndex = UINT32_MAX;
7431  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7432  if(res == VK_SUCCESS)
7433  {
7434  res = AllocateMemoryOfType(
7435  vkMemReq,
7436  requiresDedicatedAllocation || prefersDedicatedAllocation,
7437  dedicatedBuffer,
7438  dedicatedImage,
7439  createInfo,
7440  memTypeIndex,
7441  suballocType,
7442  pAllocation);
7443  // Succeeded on first try.
7444  if(res == VK_SUCCESS)
7445  {
7446  return res;
7447  }
7448  // Allocation from this memory type failed. Try other compatible memory types.
7449  else
7450  {
7451  for(;;)
7452  {
7453  // Remove old memTypeIndex from list of possibilities.
7454  memoryTypeBits &= ~(1u << memTypeIndex);
7455  // Find alternative memTypeIndex.
7456  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7457  if(res == VK_SUCCESS)
7458  {
7459  res = AllocateMemoryOfType(
7460  vkMemReq,
7461  requiresDedicatedAllocation || prefersDedicatedAllocation,
7462  dedicatedBuffer,
7463  dedicatedImage,
7464  createInfo,
7465  memTypeIndex,
7466  suballocType,
7467  pAllocation);
7468  // Allocation from this alternative memory type succeeded.
7469  if(res == VK_SUCCESS)
7470  {
7471  return res;
7472  }
7473  // else: Allocation from this memory type failed. Try next one - next loop iteration.
7474  }
7475  // No other matching memory type index could be found.
7476  else
7477  {
7478  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7479  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7480  }
7481  }
7482  }
7483  }
7484  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7485  else
7486  return res;
7487  }
7488 }
7489 
7490 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7491 {
7492  VMA_ASSERT(allocation);
7493 
7494  if(allocation->CanBecomeLost() == false ||
7495  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7496  {
7497  switch(allocation->GetType())
7498  {
7499  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7500  {
7501  VmaBlockVector* pBlockVector = VMA_NULL;
7502  VmaPool hPool = allocation->GetPool();
7503  if(hPool != VK_NULL_HANDLE)
7504  {
7505  pBlockVector = &hPool->m_BlockVector;
7506  }
7507  else
7508  {
7509  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7510  pBlockVector = m_pBlockVectors[memTypeIndex];
7511  }
7512  pBlockVector->Free(allocation);
7513  }
7514  break;
7515  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7516  FreeDedicatedMemory(allocation);
7517  break;
7518  default:
7519  VMA_ASSERT(0);
7520  }
7521  }
7522 
7523  allocation->SetUserData(this, VMA_NULL);
7524  vma_delete(this, allocation);
7525 }
7526 
7527 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7528 {
7529  // Initialize.
7530  InitStatInfo(pStats->total);
7531  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7532  InitStatInfo(pStats->memoryType[i]);
7533  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7534  InitStatInfo(pStats->memoryHeap[i]);
7535 
7536  // Process default pools.
7537  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7538  {
7539  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7540  VMA_ASSERT(pBlockVector);
7541  pBlockVector->AddStats(pStats);
7542  }
7543 
7544  // Process custom pools.
7545  {
7546  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7547  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7548  {
7549  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7550  }
7551  }
7552 
7553  // Process dedicated allocations.
7554  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7555  {
7556  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7557  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7558  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7559  VMA_ASSERT(pDedicatedAllocVector);
7560  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7561  {
7562  VmaStatInfo allocationStatInfo;
7563  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7564  VmaAddStatInfo(pStats->total, allocationStatInfo);
7565  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7566  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7567  }
7568  }
7569 
7570  // Postprocess.
7571  VmaPostprocessCalcStatInfo(pStats->total);
7572  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7573  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7574  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7575  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7576 }
7577 
7578 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7579 
7580 VkResult VmaAllocator_T::Defragment(
7581  VmaAllocation* pAllocations,
7582  size_t allocationCount,
7583  VkBool32* pAllocationsChanged,
7584  const VmaDefragmentationInfo* pDefragmentationInfo,
7585  VmaDefragmentationStats* pDefragmentationStats)
7586 {
7587  if(pAllocationsChanged != VMA_NULL)
7588  {
7589  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
7590  }
7591  if(pDefragmentationStats != VMA_NULL)
7592  {
7593  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
7594  }
7595 
7596  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
7597 
7598  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
7599 
7600  const size_t poolCount = m_Pools.size();
7601 
7602  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
7603  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
7604  {
7605  VmaAllocation hAlloc = pAllocations[allocIndex];
7606  VMA_ASSERT(hAlloc);
7607  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
7608  // DedicatedAlloc cannot be defragmented.
7609  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
7610  // Only HOST_VISIBLE memory types can be defragmented.
7611  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
7612  // Lost allocation cannot be defragmented.
7613  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
7614  {
7615  VmaBlockVector* pAllocBlockVector = VMA_NULL;
7616 
7617  const VmaPool hAllocPool = hAlloc->GetPool();
7618  // This allocation belongs to custom pool.
7619  if(hAllocPool != VK_NULL_HANDLE)
7620  {
7621  pAllocBlockVector = &hAllocPool->GetBlockVector();
7622  }
7623  // This allocation belongs to general pool.
7624  else
7625  {
7626  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
7627  }
7628 
7629  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
7630 
7631  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
7632  &pAllocationsChanged[allocIndex] : VMA_NULL;
7633  pDefragmentator->AddAllocation(hAlloc, pChanged);
7634  }
7635  }
7636 
7637  VkResult result = VK_SUCCESS;
7638 
7639  // ======== Main processing.
7640 
7641  VkDeviceSize maxBytesToMove = SIZE_MAX;
7642  uint32_t maxAllocationsToMove = UINT32_MAX;
7643  if(pDefragmentationInfo != VMA_NULL)
7644  {
7645  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
7646  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
7647  }
7648 
7649  // Process standard memory.
7650  for(uint32_t memTypeIndex = 0;
7651  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
7652  ++memTypeIndex)
7653  {
7654  // Only HOST_VISIBLE memory types can be defragmented.
7655  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7656  {
7657  result = m_pBlockVectors[memTypeIndex]->Defragment(
7658  pDefragmentationStats,
7659  maxBytesToMove,
7660  maxAllocationsToMove);
7661  }
7662  }
7663 
7664  // Process custom pools.
7665  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
7666  {
7667  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
7668  pDefragmentationStats,
7669  maxBytesToMove,
7670  maxAllocationsToMove);
7671  }
7672 
7673  // ======== Destroy defragmentators.
7674 
7675  // Process custom pools.
7676  for(size_t poolIndex = poolCount; poolIndex--; )
7677  {
7678  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
7679  }
7680 
7681  // Process standard memory.
7682  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
7683  {
7684  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7685  {
7686  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
7687  }
7688  }
7689 
7690  return result;
7691 }
7692 
7693 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
7694 {
7695  if(hAllocation->CanBecomeLost())
7696  {
7697  /*
7698  Warning: This is a carefully designed algorithm.
7699  Do not modify unless you really know what you're doing :)
7700  */
7701  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
7702  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
7703  for(;;)
7704  {
7705  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
7706  {
7707  pAllocationInfo->memoryType = UINT32_MAX;
7708  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
7709  pAllocationInfo->offset = 0;
7710  pAllocationInfo->size = hAllocation->GetSize();
7711  pAllocationInfo->pMappedData = VMA_NULL;
7712  pAllocationInfo->pUserData = hAllocation->GetUserData();
7713  return;
7714  }
7715  else if(localLastUseFrameIndex == localCurrFrameIndex)
7716  {
7717  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7718  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7719  pAllocationInfo->offset = hAllocation->GetOffset();
7720  pAllocationInfo->size = hAllocation->GetSize();
7721  pAllocationInfo->pMappedData = VMA_NULL;
7722  pAllocationInfo->pUserData = hAllocation->GetUserData();
7723  return;
7724  }
7725  else // Last use time earlier than current time.
7726  {
7727  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
7728  {
7729  localLastUseFrameIndex = localCurrFrameIndex;
7730  }
7731  }
7732  }
7733  }
7734  else
7735  {
7736  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
7737  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
7738  pAllocationInfo->offset = hAllocation->GetOffset();
7739  pAllocationInfo->size = hAllocation->GetSize();
7740  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
7741  pAllocationInfo->pUserData = hAllocation->GetUserData();
7742  }
7743 }
7744 
7745 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
7746 {
7747  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
7748 
7749  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
7750 
7751  if(newCreateInfo.maxBlockCount == 0)
7752  {
7753  newCreateInfo.maxBlockCount = SIZE_MAX;
7754  }
7755  if(newCreateInfo.blockSize == 0)
7756  {
7757  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
7758  }
7759 
7760  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
7761 
7762  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
7763  if(res != VK_SUCCESS)
7764  {
7765  vma_delete(this, *pPool);
7766  *pPool = VMA_NULL;
7767  return res;
7768  }
7769 
7770  // Add to m_Pools.
7771  {
7772  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7773  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
7774  }
7775 
7776  return VK_SUCCESS;
7777 }
7778 
7779 void VmaAllocator_T::DestroyPool(VmaPool pool)
7780 {
7781  // Remove from m_Pools.
7782  {
7783  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7784  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
7785  VMA_ASSERT(success && "Pool not found in Allocator.");
7786  }
7787 
7788  vma_delete(this, pool);
7789 }
7790 
7791 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
7792 {
7793  pool->m_BlockVector.GetPoolStats(pPoolStats);
7794 }
7795 
7796 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
7797 {
7798  m_CurrentFrameIndex.store(frameIndex);
7799 }
7800 
7801 void VmaAllocator_T::MakePoolAllocationsLost(
7802  VmaPool hPool,
7803  size_t* pLostAllocationCount)
7804 {
7805  hPool->m_BlockVector.MakePoolAllocationsLost(
7806  m_CurrentFrameIndex.load(),
7807  pLostAllocationCount);
7808 }
7809 
7810 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
7811 {
7812  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
7813  (*pAllocation)->InitLost();
7814 }
7815 
7816 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
7817 {
7818  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
7819 
7820  VkResult res;
7821  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7822  {
7823  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7824  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
7825  {
7826  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7827  if(res == VK_SUCCESS)
7828  {
7829  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
7830  }
7831  }
7832  else
7833  {
7834  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
7835  }
7836  }
7837  else
7838  {
7839  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7840  }
7841 
7842  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
7843  {
7844  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
7845  }
7846 
7847  return res;
7848 }
7849 
7850 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
7851 {
7852  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
7853  {
7854  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
7855  }
7856 
7857  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7858 
7859  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7860  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7861  {
7862  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7863  m_HeapSizeLimit[heapIndex] += size;
7864  }
7865 }
7866 
7867 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
7868 {
7869  if(hAllocation->CanBecomeLost())
7870  {
7871  return VK_ERROR_MEMORY_MAP_FAILED;
7872  }
7873 
7874  switch(hAllocation->GetType())
7875  {
7876  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7877  {
7878  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7879  char *pBytes = VMA_NULL;
7880  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
7881  if(res == VK_SUCCESS)
7882  {
7883  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
7884  hAllocation->BlockAllocMap();
7885  }
7886  return res;
7887  }
7888  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7889  return hAllocation->DedicatedAllocMap(this, ppData);
7890  default:
7891  VMA_ASSERT(0);
7892  return VK_ERROR_MEMORY_MAP_FAILED;
7893  }
7894 }
7895 
7896 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
7897 {
7898  switch(hAllocation->GetType())
7899  {
7900  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7901  {
7902  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
7903  hAllocation->BlockAllocUnmap();
7904  pBlock->Unmap(this, 1);
7905  }
7906  break;
7907  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7908  hAllocation->DedicatedAllocUnmap(this);
7909  break;
7910  default:
7911  VMA_ASSERT(0);
7912  }
7913 }
7914 
7915 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
7916 {
7917  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
7918 
7919  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7920  {
7921  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7922  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7923  VMA_ASSERT(pDedicatedAllocations);
7924  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
7925  VMA_ASSERT(success);
7926  }
7927 
7928  VkDeviceMemory hMemory = allocation->GetMemory();
7929 
7930  if(allocation->GetMappedData() != VMA_NULL)
7931  {
7932  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7933  }
7934 
7935  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7936 
7937  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
7938 }
7939 
7940 #if VMA_STATS_STRING_ENABLED
7941 
7942 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7943 {
7944  bool dedicatedAllocationsStarted = false;
7945  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7946  {
7947  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7948  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7949  VMA_ASSERT(pDedicatedAllocVector);
7950  if(pDedicatedAllocVector->empty() == false)
7951  {
7952  if(dedicatedAllocationsStarted == false)
7953  {
7954  dedicatedAllocationsStarted = true;
7955  json.WriteString("DedicatedAllocations");
7956  json.BeginObject();
7957  }
7958 
7959  json.BeginString("Type ");
7960  json.ContinueString(memTypeIndex);
7961  json.EndString();
7962 
7963  json.BeginArray();
7964 
7965  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
7966  {
7967  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
7968  json.BeginObject(true);
7969 
7970  json.WriteString("Type");
7971  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7972 
7973  json.WriteString("Size");
7974  json.WriteNumber(hAlloc->GetSize());
7975 
7976  const void* pUserData = hAlloc->GetUserData();
7977  if(pUserData != VMA_NULL)
7978  {
7979  json.WriteString("UserData");
7980  if(hAlloc->IsUserDataString())
7981  {
7982  json.WriteString((const char*)pUserData);
7983  }
7984  else
7985  {
7986  json.BeginString();
7987  json.ContinueString_Pointer(pUserData);
7988  json.EndString();
7989  }
7990  }
7991 
7992  json.EndObject();
7993  }
7994 
7995  json.EndArray();
7996  }
7997  }
7998  if(dedicatedAllocationsStarted)
7999  {
8000  json.EndObject();
8001  }
8002 
8003  {
8004  bool allocationsStarted = false;
8005  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8006  {
8007  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8008  {
8009  if(allocationsStarted == false)
8010  {
8011  allocationsStarted = true;
8012  json.WriteString("DefaultPools");
8013  json.BeginObject();
8014  }
8015 
8016  json.BeginString("Type ");
8017  json.ContinueString(memTypeIndex);
8018  json.EndString();
8019 
8020  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8021  }
8022  }
8023  if(allocationsStarted)
8024  {
8025  json.EndObject();
8026  }
8027  }
8028 
8029  {
8030  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8031  const size_t poolCount = m_Pools.size();
8032  if(poolCount > 0)
8033  {
8034  json.WriteString("Pools");
8035  json.BeginArray();
8036  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8037  {
8038  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8039  }
8040  json.EndArray();
8041  }
8042  }
8043 }
8044 
8045 #endif // #if VMA_STATS_STRING_ENABLED
8046 
8047 static VkResult AllocateMemoryForImage(
8048  VmaAllocator allocator,
8049  VkImage image,
8050  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8051  VmaSuballocationType suballocType,
8052  VmaAllocation* pAllocation)
8053 {
8054  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8055 
8056  VkMemoryRequirements vkMemReq = {};
8057  bool requiresDedicatedAllocation = false;
8058  bool prefersDedicatedAllocation = false;
8059  allocator->GetImageMemoryRequirements(image, vkMemReq,
8060  requiresDedicatedAllocation, prefersDedicatedAllocation);
8061 
8062  return allocator->AllocateMemory(
8063  vkMemReq,
8064  requiresDedicatedAllocation,
8065  prefersDedicatedAllocation,
8066  VK_NULL_HANDLE, // dedicatedBuffer
8067  image, // dedicatedImage
8068  *pAllocationCreateInfo,
8069  suballocType,
8070  pAllocation);
8071 }
8072 
8074 // Public interface
8075 
8076 VkResult vmaCreateAllocator(
8077  const VmaAllocatorCreateInfo* pCreateInfo,
8078  VmaAllocator* pAllocator)
8079 {
8080  VMA_ASSERT(pCreateInfo && pAllocator);
8081  VMA_DEBUG_LOG("vmaCreateAllocator");
8082  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8083  return VK_SUCCESS;
8084 }
8085 
8086 void vmaDestroyAllocator(
8087  VmaAllocator allocator)
8088 {
8089  if(allocator != VK_NULL_HANDLE)
8090  {
8091  VMA_DEBUG_LOG("vmaDestroyAllocator");
8092  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8093  vma_delete(&allocationCallbacks, allocator);
8094  }
8095 }
8096 
8097 void vmaGetPhysicalDeviceProperties(
8098  VmaAllocator allocator,
8099  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8100 {
8101  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8102  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8103 }
8104 
8105 void vmaGetMemoryProperties(
8106  VmaAllocator allocator,
8107  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8108 {
8109  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8110  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8111 }
8112 
8113 void vmaGetMemoryTypeProperties(
8114  VmaAllocator allocator,
8115  uint32_t memoryTypeIndex,
8116  VkMemoryPropertyFlags* pFlags)
8117 {
8118  VMA_ASSERT(allocator && pFlags);
8119  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8120  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8121 }
8122 
8123 void vmaSetCurrentFrameIndex(
8124  VmaAllocator allocator,
8125  uint32_t frameIndex)
8126 {
8127  VMA_ASSERT(allocator);
8128  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8129 
8130  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8131 
8132  allocator->SetCurrentFrameIndex(frameIndex);
8133 }
8134 
8135 void vmaCalculateStats(
8136  VmaAllocator allocator,
8137  VmaStats* pStats)
8138 {
8139  VMA_ASSERT(allocator && pStats);
8140  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8141  allocator->CalculateStats(pStats);
8142 }
8143 
8144 #if VMA_STATS_STRING_ENABLED
8145 
8146 void vmaBuildStatsString(
8147  VmaAllocator allocator,
8148  char** ppStatsString,
8149  VkBool32 detailedMap)
8150 {
8151  VMA_ASSERT(allocator && ppStatsString);
8152  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8153 
8154  VmaStringBuilder sb(allocator);
8155  {
8156  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8157  json.BeginObject();
8158 
8159  VmaStats stats;
8160  allocator->CalculateStats(&stats);
8161 
8162  json.WriteString("Total");
8163  VmaPrintStatInfo(json, stats.total);
8164 
8165  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8166  {
8167  json.BeginString("Heap ");
8168  json.ContinueString(heapIndex);
8169  json.EndString();
8170  json.BeginObject();
8171 
8172  json.WriteString("Size");
8173  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8174 
8175  json.WriteString("Flags");
8176  json.BeginArray(true);
8177  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8178  {
8179  json.WriteString("DEVICE_LOCAL");
8180  }
8181  json.EndArray();
8182 
8183  if(stats.memoryHeap[heapIndex].blockCount > 0)
8184  {
8185  json.WriteString("Stats");
8186  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8187  }
8188 
8189  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8190  {
8191  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8192  {
8193  json.BeginString("Type ");
8194  json.ContinueString(typeIndex);
8195  json.EndString();
8196 
8197  json.BeginObject();
8198 
8199  json.WriteString("Flags");
8200  json.BeginArray(true);
8201  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8202  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8203  {
8204  json.WriteString("DEVICE_LOCAL");
8205  }
8206  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8207  {
8208  json.WriteString("HOST_VISIBLE");
8209  }
8210  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8211  {
8212  json.WriteString("HOST_COHERENT");
8213  }
8214  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8215  {
8216  json.WriteString("HOST_CACHED");
8217  }
8218  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8219  {
8220  json.WriteString("LAZILY_ALLOCATED");
8221  }
8222  json.EndArray();
8223 
8224  if(stats.memoryType[typeIndex].blockCount > 0)
8225  {
8226  json.WriteString("Stats");
8227  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8228  }
8229 
8230  json.EndObject();
8231  }
8232  }
8233 
8234  json.EndObject();
8235  }
8236  if(detailedMap == VK_TRUE)
8237  {
8238  allocator->PrintDetailedMap(json);
8239  }
8240 
8241  json.EndObject();
8242  }
8243 
8244  const size_t len = sb.GetLength();
8245  char* const pChars = vma_new_array(allocator, char, len + 1);
8246  if(len > 0)
8247  {
8248  memcpy(pChars, sb.GetData(), len);
8249  }
8250  pChars[len] = '\0';
8251  *ppStatsString = pChars;
8252 }
8253 
8254 void vmaFreeStatsString(
8255  VmaAllocator allocator,
8256  char* pStatsString)
8257 {
8258  if(pStatsString != VMA_NULL)
8259  {
8260  VMA_ASSERT(allocator);
8261  size_t len = strlen(pStatsString);
8262  vma_delete_array(allocator, pStatsString, len + 1);
8263  }
8264 }
8265 
8266 #endif // #if VMA_STATS_STRING_ENABLED
8267 
8268 /*
8269 This function is not protected by any mutex because it just reads immutable data.
8270 */
8271 VkResult vmaFindMemoryTypeIndex(
8272  VmaAllocator allocator,
8273  uint32_t memoryTypeBits,
8274  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8275  uint32_t* pMemoryTypeIndex)
8276 {
8277  VMA_ASSERT(allocator != VK_NULL_HANDLE);
8278  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8279  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8280 
8281  if(pAllocationCreateInfo->memoryTypeBits != 0)
8282  {
8283  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8284  }
8285 
8286  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8287  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8288 
8289  // Convert usage to requiredFlags and preferredFlags.
8290  switch(pAllocationCreateInfo->usage)
8291  {
8292  case VMA_MEMORY_USAGE_UNKNOWN:
8293  break;
8294  case VMA_MEMORY_USAGE_GPU_ONLY:
8295  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8296  break;
8297  case VMA_MEMORY_USAGE_CPU_ONLY:
8298  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8299  break;
8300  case VMA_MEMORY_USAGE_CPU_TO_GPU:
8301  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8302  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8303  break;
8304  case VMA_MEMORY_USAGE_GPU_TO_CPU:
8305  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8306  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8307  break;
8308  default:
8309  break;
8310  }
8311 
8312  *pMemoryTypeIndex = UINT32_MAX;
8313  uint32_t minCost = UINT32_MAX;
8314  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8315  memTypeIndex < allocator->GetMemoryTypeCount();
8316  ++memTypeIndex, memTypeBit <<= 1)
8317  {
8318  // This memory type is acceptable according to memoryTypeBits bitmask.
8319  if((memTypeBit & memoryTypeBits) != 0)
8320  {
8321  const VkMemoryPropertyFlags currFlags =
8322  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8323  // This memory type contains requiredFlags.
8324  if((requiredFlags & ~currFlags) == 0)
8325  {
8326  // Calculate cost as number of bits from preferredFlags not present in this memory type.
8327  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8328  // Remember memory type with lowest cost.
8329  if(currCost < minCost)
8330  {
8331  *pMemoryTypeIndex = memTypeIndex;
8332  if(currCost == 0)
8333  {
8334  return VK_SUCCESS;
8335  }
8336  minCost = currCost;
8337  }
8338  }
8339  }
8340  }
8341  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8342 }
8343 
8344 VkResult vmaCreatePool(
8345  VmaAllocator allocator,
8346  const VmaPoolCreateInfo* pCreateInfo,
8347  VmaPool* pPool)
8348 {
8349  VMA_ASSERT(allocator && pCreateInfo && pPool);
8350 
8351  VMA_DEBUG_LOG("vmaCreatePool");
8352 
8353  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8354 
8355  return allocator->CreatePool(pCreateInfo, pPool);
8356 }
8357 
8358 void vmaDestroyPool(
8359  VmaAllocator allocator,
8360  VmaPool pool)
8361 {
8362  VMA_ASSERT(allocator);
8363 
8364  if(pool == VK_NULL_HANDLE)
8365  {
8366  return;
8367  }
8368 
8369  VMA_DEBUG_LOG("vmaDestroyPool");
8370 
8371  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8372 
8373  allocator->DestroyPool(pool);
8374 }
8375 
8376 void vmaGetPoolStats(
8377  VmaAllocator allocator,
8378  VmaPool pool,
8379  VmaPoolStats* pPoolStats)
8380 {
8381  VMA_ASSERT(allocator && pool && pPoolStats);
8382 
8383  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8384 
8385  allocator->GetPoolStats(pool, pPoolStats);
8386 }
8387 
8388 void vmaMakePoolAllocationsLost(
8389  VmaAllocator allocator,
8390  VmaPool pool,
8391  size_t* pLostAllocationCount)
8392 {
8393  VMA_ASSERT(allocator && pool);
8394 
8395  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8396 
8397  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8398 }
8399 
8400 VkResult vmaAllocateMemory(
8401  VmaAllocator allocator,
8402  const VkMemoryRequirements* pVkMemoryRequirements,
8403  const VmaAllocationCreateInfo* pCreateInfo,
8404  VmaAllocation* pAllocation,
8405  VmaAllocationInfo* pAllocationInfo)
8406 {
8407  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8408 
8409  VMA_DEBUG_LOG("vmaAllocateMemory");
8410 
8411  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8412 
8413  VkResult result = allocator->AllocateMemory(
8414  *pVkMemoryRequirements,
8415  false, // requiresDedicatedAllocation
8416  false, // prefersDedicatedAllocation
8417  VK_NULL_HANDLE, // dedicatedBuffer
8418  VK_NULL_HANDLE, // dedicatedImage
8419  *pCreateInfo,
8420  VMA_SUBALLOCATION_TYPE_UNKNOWN,
8421  pAllocation);
8422 
8423  if(pAllocationInfo && result == VK_SUCCESS)
8424  {
8425  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8426  }
8427 
8428  return result;
8429 }
8430 
8431 VkResult vmaAllocateMemoryForBuffer(
8432  VmaAllocator allocator,
8433  VkBuffer buffer,
8434  const VmaAllocationCreateInfo* pCreateInfo,
8435  VmaAllocation* pAllocation,
8436  VmaAllocationInfo* pAllocationInfo)
8437 {
8438  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8439 
8440  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
8441 
8442  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8443 
8444  VkMemoryRequirements vkMemReq = {};
8445  bool requiresDedicatedAllocation = false;
8446  bool prefersDedicatedAllocation = false;
8447  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
8448  requiresDedicatedAllocation,
8449  prefersDedicatedAllocation);
8450 
8451  VkResult result = allocator->AllocateMemory(
8452  vkMemReq,
8453  requiresDedicatedAllocation,
8454  prefersDedicatedAllocation,
8455  buffer, // dedicatedBuffer
8456  VK_NULL_HANDLE, // dedicatedImage
8457  *pCreateInfo,
8458  VMA_SUBALLOCATION_TYPE_BUFFER,
8459  pAllocation);
8460 
8461  if(pAllocationInfo && result == VK_SUCCESS)
8462  {
8463  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8464  }
8465 
8466  return result;
8467 }
8468 
8469 VkResult vmaAllocateMemoryForImage(
8470  VmaAllocator allocator,
8471  VkImage image,
8472  const VmaAllocationCreateInfo* pCreateInfo,
8473  VmaAllocation* pAllocation,
8474  VmaAllocationInfo* pAllocationInfo)
8475 {
8476  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
8477 
8478  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
8479 
8480  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8481 
8482  VkResult result = AllocateMemoryForImage(
8483  allocator,
8484  image,
8485  pCreateInfo,
8486  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
8487  pAllocation);
8488 
8489  if(pAllocationInfo && result == VK_SUCCESS)
8490  {
8491  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8492  }
8493 
8494  return result;
8495 }
8496 
8497 void vmaFreeMemory(
8498  VmaAllocator allocator,
8499  VmaAllocation allocation)
8500 {
8501  VMA_ASSERT(allocator && allocation);
8502 
8503  VMA_DEBUG_LOG("vmaFreeMemory");
8504 
8505  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8506 
8507  allocator->FreeMemory(allocation);
8508 }
8509 
8510 void vmaGetAllocationInfo(
8511  VmaAllocator allocator,
8512  VmaAllocation allocation,
8513  VmaAllocationInfo* pAllocationInfo)
8514 {
8515  VMA_ASSERT(allocator && allocation && pAllocationInfo);
8516 
8517  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8518 
8519  allocator->GetAllocationInfo(allocation, pAllocationInfo);
8520 }
8521 
8522 void vmaSetAllocationUserData(
8523  VmaAllocator allocator,
8524  VmaAllocation allocation,
8525  void* pUserData)
8526 {
8527  VMA_ASSERT(allocator && allocation);
8528 
8529  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8530 
8531  allocation->SetUserData(allocator, pUserData);
8532 }
8533 
8534 void vmaCreateLostAllocation(
8535  VmaAllocator allocator,
8536  VmaAllocation* pAllocation)
8537 {
8538  VMA_ASSERT(allocator && pAllocation);
8539 
8540  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
8541 
8542  allocator->CreateLostAllocation(pAllocation);
8543 }
8544 
8545 VkResult vmaMapMemory(
8546  VmaAllocator allocator,
8547  VmaAllocation allocation,
8548  void** ppData)
8549 {
8550  VMA_ASSERT(allocator && allocation && ppData);
8551 
8552  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8553 
8554  return allocator->Map(allocation, ppData);
8555 }
8556 
8557 void vmaUnmapMemory(
8558  VmaAllocator allocator,
8559  VmaAllocation allocation)
8560 {
8561  VMA_ASSERT(allocator && allocation);
8562 
8563  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8564 
8565  allocator->Unmap(allocation);
8566 }
8567 
8568 VkResult vmaDefragment(
8569  VmaAllocator allocator,
8570  VmaAllocation* pAllocations,
8571  size_t allocationCount,
8572  VkBool32* pAllocationsChanged,
8573  const VmaDefragmentationInfo *pDefragmentationInfo,
8574  VmaDefragmentationStats* pDefragmentationStats)
8575 {
8576  VMA_ASSERT(allocator && pAllocations);
8577 
8578  VMA_DEBUG_LOG("vmaDefragment");
8579 
8580  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8581 
8582  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
8583 }
8584 
8585 VkResult vmaCreateBuffer(
8586  VmaAllocator allocator,
8587  const VkBufferCreateInfo* pBufferCreateInfo,
8588  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8589  VkBuffer* pBuffer,
8590  VmaAllocation* pAllocation,
8591  VmaAllocationInfo* pAllocationInfo)
8592 {
8593  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
8594 
8595  VMA_DEBUG_LOG("vmaCreateBuffer");
8596 
8597  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8598 
8599  *pBuffer = VK_NULL_HANDLE;
8600  *pAllocation = VK_NULL_HANDLE;
8601 
8602  // 1. Create VkBuffer.
8603  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
8604  allocator->m_hDevice,
8605  pBufferCreateInfo,
8606  allocator->GetAllocationCallbacks(),
8607  pBuffer);
8608  if(res >= 0)
8609  {
8610  // 2. vkGetBufferMemoryRequirements.
8611  VkMemoryRequirements vkMemReq = {};
8612  bool requiresDedicatedAllocation = false;
8613  bool prefersDedicatedAllocation = false;
8614  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
8615  requiresDedicatedAllocation, prefersDedicatedAllocation);
8616 
8617  // Make sure alignment requirements for specific buffer usages reported
8618  // in Physical Device Properties are included in alignment reported by memory requirements.
8619  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
8620  {
8621  VMA_ASSERT(vkMemReq.alignment %
8622  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
8623  }
8624  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
8625  {
8626  VMA_ASSERT(vkMemReq.alignment %
8627  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
8628  }
8629  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
8630  {
8631  VMA_ASSERT(vkMemReq.alignment %
8632  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
8633  }
8634 
8635  // 3. Allocate memory using allocator.
8636  res = allocator->AllocateMemory(
8637  vkMemReq,
8638  requiresDedicatedAllocation,
8639  prefersDedicatedAllocation,
8640  *pBuffer, // dedicatedBuffer
8641  VK_NULL_HANDLE, // dedicatedImage
8642  *pAllocationCreateInfo,
8643  VMA_SUBALLOCATION_TYPE_BUFFER,
8644  pAllocation);
8645  if(res >= 0)
8646  {
8647  // 3. Bind buffer with memory.
8648  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
8649  allocator->m_hDevice,
8650  *pBuffer,
8651  (*pAllocation)->GetMemory(),
8652  (*pAllocation)->GetOffset());
8653  if(res >= 0)
8654  {
8655  // All steps succeeded.
8656  if(pAllocationInfo != VMA_NULL)
8657  {
8658  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8659  }
8660  return VK_SUCCESS;
8661  }
8662  allocator->FreeMemory(*pAllocation);
8663  *pAllocation = VK_NULL_HANDLE;
8664  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8665  *pBuffer = VK_NULL_HANDLE;
8666  return res;
8667  }
8668  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
8669  *pBuffer = VK_NULL_HANDLE;
8670  return res;
8671  }
8672  return res;
8673 }
8674 
8675 void vmaDestroyBuffer(
8676  VmaAllocator allocator,
8677  VkBuffer buffer,
8678  VmaAllocation allocation)
8679 {
8680  if(buffer != VK_NULL_HANDLE)
8681  {
8682  VMA_ASSERT(allocator);
8683 
8684  VMA_DEBUG_LOG("vmaDestroyBuffer");
8685 
8686  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8687 
8688  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
8689 
8690  allocator->FreeMemory(allocation);
8691  }
8692 }
8693 
8694 VkResult vmaCreateImage(
8695  VmaAllocator allocator,
8696  const VkImageCreateInfo* pImageCreateInfo,
8697  const VmaAllocationCreateInfo* pAllocationCreateInfo,
8698  VkImage* pImage,
8699  VmaAllocation* pAllocation,
8700  VmaAllocationInfo* pAllocationInfo)
8701 {
8702  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
8703 
8704  VMA_DEBUG_LOG("vmaCreateImage");
8705 
8706  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8707 
8708  *pImage = VK_NULL_HANDLE;
8709  *pAllocation = VK_NULL_HANDLE;
8710 
8711  // 1. Create VkImage.
8712  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
8713  allocator->m_hDevice,
8714  pImageCreateInfo,
8715  allocator->GetAllocationCallbacks(),
8716  pImage);
8717  if(res >= 0)
8718  {
8719  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
8720  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
8721  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
8722 
8723  // 2. Allocate memory using allocator.
8724  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
8725  if(res >= 0)
8726  {
8727  // 3. Bind image with memory.
8728  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
8729  allocator->m_hDevice,
8730  *pImage,
8731  (*pAllocation)->GetMemory(),
8732  (*pAllocation)->GetOffset());
8733  if(res >= 0)
8734  {
8735  // All steps succeeded.
8736  if(pAllocationInfo != VMA_NULL)
8737  {
8738  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8739  }
8740  return VK_SUCCESS;
8741  }
8742  allocator->FreeMemory(*pAllocation);
8743  *pAllocation = VK_NULL_HANDLE;
8744  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8745  *pImage = VK_NULL_HANDLE;
8746  return res;
8747  }
8748  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
8749  *pImage = VK_NULL_HANDLE;
8750  return res;
8751  }
8752  return res;
8753 }
8754 
8755 void vmaDestroyImage(
8756  VmaAllocator allocator,
8757  VkImage image,
8758  VmaAllocation allocation)
8759 {
8760  if(image != VK_NULL_HANDLE)
8761  {
8762  VMA_ASSERT(allocator);
8763 
8764  VMA_DEBUG_LOG("vmaDestroyImage");
8765 
8766  VMA_DEBUG_GLOBAL_MUTEX_LOCK
8767 
8768  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
8769 
8770  allocator->FreeMemory(allocation);
8771  }
8772 }
8773 
8774 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:896
+
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1150
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
-
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:918
+
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:921
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
-
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:903
+
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:906
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
VmaStats
struct VmaStats VmaStats
General statistics from current state of Allocator.
-
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1104
-
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:897
-
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1415
-
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:915
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1581
-
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1285
+
VMA_MEMORY_USAGE_GPU_ONLY
Definition: vk_mem_alloc.h:1107
+
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:900
+
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:1418
+
VmaAllocatorCreateInfo::flags
VmaAllocatorCreateFlags flags
Flags for created allocator. Use VmaAllocatorCreateFlagBits enum.
Definition: vk_mem_alloc.h:918
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1584
+
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:1288
vmaMakePoolAllocationsLost
void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
-
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1339
-
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1184
-
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:886
-
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1222
-
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1131
-
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:927
+
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:1342
+
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:1187
+
VmaAllocatorCreateFlags
VkFlags VmaAllocatorCreateFlags
Definition: vk_mem_alloc.h:889
+
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a memory type chosen for an allocation.
Definition: vk_mem_alloc.h:1225
+
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:1134
+
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:930
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
-
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:980
-
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:912
+
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:983
+
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:915
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
-
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1135
+
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:1138
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation.
-
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:1045
-
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:900
-
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:1044
-
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:908
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1585
+
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:1048
+
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:903
+
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:1047
+
VmaVulkanFunctions::vkGetImageMemoryRequirements2KHR
PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR
Definition: vk_mem_alloc.h:911
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1588
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
-
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:944
-
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1054
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1593
-
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1206
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1576
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:901
-
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:828
-
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:921
-
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1293
-
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1287
-
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1425
+
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:947
+
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:1057
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1596
+
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:1209
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1579
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:904
+
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:831
+
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:924
+
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:1296
+
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1290
+
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:1428
vmaGetMemoryTypeProperties
void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
Given Memory Type Index, returns Property Flags of this memory type.
-
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:898
-
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1243
-
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1309
-
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1345
+
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:901
+
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:1246
+
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:1312
+
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:1348
VmaVulkanFunctions
struct VmaVulkanFunctions VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
-
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:884
-
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1296
+
VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:887
+
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:1299
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
-
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1082
+
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:1085
VmaAllocationInfo
struct VmaAllocationInfo VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1571
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1574
VmaPoolCreateInfo
struct VmaPoolCreateInfo VmaPoolCreateInfo
Describes parameter of created VmaPool.
vmaDestroyPool
void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
Destroys VmaPool object and frees Vulkan device memory.
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1589
-
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1121
-
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1230
-
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:899
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1592
+
VMA_MEMORY_USAGE_CPU_TO_GPU
Definition: vk_mem_alloc.h:1124
+
VmaAllocationCreateInfo::memoryTypeBits
uint32_t memoryTypeBits
Bitmask containing one bit set for every memory type acceptable for this allocation.
Definition: vk_mem_alloc.h:1233
+
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:902
vmaGetPoolStats
void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
Retrieves statistics of existing VmaPool object.
VmaDefragmentationInfo
struct VmaDefragmentationInfo VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
-
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1050
-
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:834
+
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:1053
+
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:837
vmaSetAllocationUserData
void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
Sets pUserData in given allocation to new value.
vmaCreatePool
VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
Allocates Vulkan device memory and creates VmaPool object.
-
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:855
+
VmaAllocatorCreateFlagBits
VmaAllocatorCreateFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:858
VmaStatInfo
struct VmaStatInfo VmaStatInfo
Calculated statistics of memory usage in entire allocator.
-
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:860
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1591
+
VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:863
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1594
vmaCreateLostAllocation
void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
Creates new allocation that is in lost state from the beginning.
-
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1217
-
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1355
+
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:1220
+
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:1358
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
-
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:894
-
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:1033
-
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1304
-
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:847
+
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:897
+
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:1036
+
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:1307
+
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:850
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1191
-
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1046
-
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:851
-
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1299
-
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1130
+
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:1194
+
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:1049
+
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:854
+
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:1302
+
VMA_MEMORY_USAGE_GPU_TO_CPU
Definition: vk_mem_alloc.h:1133
VmaPoolStats
struct VmaPoolStats VmaPoolStats
Describes parameter of existing VmaPool.
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaCreateBuffer().
-
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1212
-
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1203
-
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:1036
-
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:896
-
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:1317
-
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:930
-
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1348
-
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1201
-
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1236
+
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:1215
+
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:1206
+
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:1039
+
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:899
+
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:1320
+
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:933
+
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1351
+
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:1204
+
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:1239
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
-
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:968
-
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1052
-
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1171
-
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:1045
-
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:905
-
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:849
-
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:904
+
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:971
+
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:1055
+
VMA_ALLOCATION_CREATE_MAPPED_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:1174
+
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:1048
+
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:908
+
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:852
+
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:907
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
Maps memory represented by given allocation and returns pointer to it.
-
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1331
+
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:1334
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaAllocateMemoryForBuffer().
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
-
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1439
-
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:924
-
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:1045
-
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:1042
+
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:1442
+
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps > 1 GiB...
Definition: vk_mem_alloc.h:927
+
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:1048
+
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:1045
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
-
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1336
-
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1420
-
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1199
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1587
-
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:892
+
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:1339
+
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:1423
+
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:1202
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1590
+
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:895
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:907
-
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:1040
-
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1087
-
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1289
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements2KHR
PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR
Definition: vk_mem_alloc.h:910
+
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:1043
+
VMA_MEMORY_USAGE_UNKNOWN
Definition: vk_mem_alloc.h:1090
+
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:1292
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:1038
-
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:902
-
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:906
-
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1158
-
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1114
-
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1434
+
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:1041
+
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:905
+
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:909
+
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:1161
+
VMA_MEMORY_USAGE_CPU_ONLY
Definition: vk_mem_alloc.h:1117
+
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data.
Definition: vk_mem_alloc.h:1437
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
Destroys Vulkan image and frees allocated memory.
-
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:882
+
VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT
Enables usage of VK_KHR_dedicated_allocation extension.
Definition: vk_mem_alloc.h:885
VmaDefragmentationStats
struct VmaDefragmentationStats VmaDefragmentationStats
Statistics returned by function vmaDefragment().
-
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:895
-
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1401
+
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:898
+
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:1404
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
General purpose memory allocation.
vmaSetCurrentFrameIndex
void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
Sets index of the current frame.
VmaAllocationCreateInfo
struct VmaAllocationCreateInfo VmaAllocationCreateInfo
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1267
-
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1046
-
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1197
-
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1053
+
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:1270
+
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:1049
+
VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT
Definition: vk_mem_alloc.h:1200
+
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:1056
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
Destroys Vulkan buffer and frees allocated memory.
-
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1342
-
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1046
-
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1406
+
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:1345
+
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:1049
+
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:1409
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 1205376..c7e6c45 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -362,6 +362,9 @@ allocation from video memory might have failed, so the library chose system memo You can detect this case and map such allocation to access its memory on CPU directly, instead of launching a transfer operation. +In order to do that: inspect `allocInfo.memoryType`, call vmaGetMemoryTypeProperties(), +and look for `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag in properties of that memory type. + You can even use `VMA_ALLOCATION_CREATE_MAPPED_BIT` flag while creating allocations that are not necessarily `HOST_VISIBLE` (e.g. using `VMA_MEMORY_USAGE_GPU_ONLY`). If the allocation ends up in memory type that is `HOST_VISIBLE`, it will be persistently mapped and you can use it directly.