From 1bb85fa719d9cb8d056fa24aa258272a323192a0 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Mon, 2 Oct 2017 14:28:51 +0200 Subject: [PATCH] Added support for VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT without HOST_VISIBLE. Improved empty block heuristics. --- docs/html/group__layer1.html | 3 +- docs/html/vk__mem__alloc_8h_source.html | 92 ++++++++++----------- src/vk_mem_alloc.h | 102 ++++++++++++++++-------- 3 files changed, 116 insertions(+), 81 deletions(-) diff --git a/docs/html/group__layer1.html b/docs/html/group__layer1.html index c58e442..6559315 100644 --- a/docs/html/group__layer1.html +++ b/docs/html/group__layer1.html @@ -203,7 +203,8 @@ Functions VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT 

Set this flag to use a memory that will be persistently mapped and retrieve pointer to it.

Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. You cannot map the memory on your own as multiple mappings of a single VkDeviceMemory are illegal.

-

If VmaAllocationCreateInfo::pool is not null, usage of this flag must match usage of flag VMA_POOL_CREATE_PERSISTENT_MAP_BIT used during pool creation.

+

If VmaAllocationCreateInfo::pool is not null, usage of this flag must match usage of flag VMA_POOL_CREATE_PERSISTENT_MAP_BIT used during pool creation.

+

Is it valid to use this flag for allocation made from memory type that is not HOST_VISIBLE. This flag is then ignored and memory is not mapped. This is useful if you need an allocation that is efficient to use on GPU (DEVICE_LOCAL) and still want to map it directly if possible on platforms that support it (e.g. Intel GPU).

VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT 

Allocation created with this flag can become lost as a result of another allocation with VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag, so you must check it before use.

To check if allocation is not lost, call vmaGetAllocationInfo() and check if VmaAllocationInfo::deviceMemory is not VK_NULL_HANDLE.

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index b189ea8..41633d7 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,28 +62,28 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
393 #include <vulkan/vulkan.h>
394 
396 
400 VK_DEFINE_HANDLE(VmaAllocator)
401 
402 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
404  VmaAllocator allocator,
405  uint32_t memoryType,
406  VkDeviceMemory memory,
407  VkDeviceSize size);
409 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
410  VmaAllocator allocator,
411  uint32_t memoryType,
412  VkDeviceMemory memory,
413  VkDeviceSize size);
414 
422 typedef struct VmaDeviceMemoryCallbacks {
424  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
426  PFN_vmaFreeDeviceMemoryFunction pfnFree;
427 } VmaDeviceMemoryCallbacks;
428 
430 typedef enum VmaAllocatorFlagBits {
435  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
436 
437  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
438 } VmaAllocatorFlagBits;
439 typedef VkFlags VmaAllocatorFlags;
440 
445 typedef struct VmaVulkanFunctions {
446  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
447  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
448  PFN_vkAllocateMemory vkAllocateMemory;
449  PFN_vkFreeMemory vkFreeMemory;
450  PFN_vkMapMemory vkMapMemory;
451  PFN_vkUnmapMemory vkUnmapMemory;
452  PFN_vkBindBufferMemory vkBindBufferMemory;
453  PFN_vkBindImageMemory vkBindImageMemory;
454  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
455  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
456  PFN_vkCreateBuffer vkCreateBuffer;
457  PFN_vkDestroyBuffer vkDestroyBuffer;
458  PFN_vkCreateImage vkCreateImage;
459  PFN_vkDestroyImage vkDestroyImage;
460 } VmaVulkanFunctions;
461 
463 typedef struct VmaAllocatorCreateInfo
464 {
466  VmaAllocatorFlags flags;
468 
469  VkPhysicalDevice physicalDevice;
471 
472  VkDevice device;
474 
475  VkDeviceSize preferredLargeHeapBlockSize;
477 
478  VkDeviceSize preferredSmallHeapBlockSize;
480 
481  const VkAllocationCallbacks* pAllocationCallbacks;
483 
484  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
498  uint32_t frameInUseCount;
516  const VkDeviceSize* pHeapSizeLimit;
528  const VmaVulkanFunctions* pVulkanFunctions;
529 } VmaAllocatorCreateInfo;
530 
532 VkResult vmaCreateAllocator(
533  const VmaAllocatorCreateInfo* pCreateInfo,
534  VmaAllocator* pAllocator);
535 
537 void vmaDestroyAllocator(
538  VmaAllocator allocator);
539 
544 void vmaGetPhysicalDeviceProperties(
545  VmaAllocator allocator,
546  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
547 
552 void vmaGetMemoryProperties(
553  VmaAllocator allocator,
554  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
555 
562 void vmaGetMemoryTypeProperties(
563  VmaAllocator allocator,
564  uint32_t memoryTypeIndex,
565  VkMemoryPropertyFlags* pFlags);
566 
575 void vmaSetCurrentFrameIndex(
576  VmaAllocator allocator,
577  uint32_t frameIndex);
578 
581 typedef struct VmaStatInfo
582 {
584  uint32_t blockCount;
586  uint32_t allocationCount;
588  uint32_t unusedRangeCount;
590  VkDeviceSize usedBytes;
592  VkDeviceSize unusedBytes;
593  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
594  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
595 } VmaStatInfo;
596 
598 typedef struct VmaStats
599 {
600  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
601  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
602  VmaStatInfo total;
603 } VmaStats;
604 
606 void vmaCalculateStats(
607  VmaAllocator allocator,
608  VmaStats* pStats);
609 
610 #define VMA_STATS_STRING_ENABLED 1
611 
612 #if VMA_STATS_STRING_ENABLED
613 
615 
617 void vmaBuildStatsString(
618  VmaAllocator allocator,
619  char** ppStatsString,
620  VkBool32 detailedMap);
621 
622 void vmaFreeStatsString(
623  VmaAllocator allocator,
624  char* pStatsString);
625 
626 #endif // #if VMA_STATS_STRING_ENABLED
627 
630 
635 VK_DEFINE_HANDLE(VmaPool)
636 
637 typedef enum VmaMemoryUsage
638 {
640  VMA_MEMORY_USAGE_UNKNOWN = 0,
642  VMA_MEMORY_USAGE_GPU_ONLY = 1,
644 
645  VMA_MEMORY_USAGE_CPU_ONLY = 2,
647 
648  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
650 
651  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
652  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
653 } VmaMemoryUsage;
654 
656 typedef enum VmaAllocationCreateFlagBits {
668  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001,
669 
679  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
689  VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004,
700  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
707  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
708 
709  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
710 } VmaAllocationCreateFlagBits;
711 typedef VkFlags VmaAllocationCreateFlags;
712 
713 typedef struct VmaAllocationCreateInfo
714 {
716  VmaAllocationCreateFlags flags;
722  VmaMemoryUsage usage;
727  VkMemoryPropertyFlags requiredFlags;
733  VkMemoryPropertyFlags preferredFlags;
735  void* pUserData;
740  VmaPool pool;
741 } VmaAllocationCreateInfo;
742 
757 VkResult vmaFindMemoryTypeIndex(
758  VmaAllocator allocator,
759  uint32_t memoryTypeBits,
760  const VmaAllocationCreateInfo* pAllocationCreateInfo,
761  uint32_t* pMemoryTypeIndex);
762 
765 
770 typedef enum VmaPoolCreateFlagBits {
780  VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001,
798  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
799 
800  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
801 } VmaPoolCreateFlagBits;
802 typedef VkFlags VmaPoolCreateFlags;
803 
806 typedef struct VmaPoolCreateInfo {
809  uint32_t memoryTypeIndex;
812  VmaPoolCreateFlags flags;
817  VkDeviceSize blockSize;
822  size_t minBlockCount;
830  size_t maxBlockCount;
844  uint32_t frameInUseCount;
845 } VmaPoolCreateInfo;
846 
849 typedef struct VmaPoolStats {
852  VkDeviceSize size;
855  VkDeviceSize unusedSize;
858  size_t allocationCount;
861  size_t unusedRangeCount;
868  VkDeviceSize unusedRangeSizeMax;
869 } VmaPoolStats;
870 
877 VkResult vmaCreatePool(
878  VmaAllocator allocator,
879  const VmaPoolCreateInfo* pCreateInfo,
880  VmaPool* pPool);
881 
884 void vmaDestroyPool(
885  VmaAllocator allocator,
886  VmaPool pool);
887 
894 void vmaGetPoolStats(
895  VmaAllocator allocator,
896  VmaPool pool,
897  VmaPoolStats* pPoolStats);
898 
905 void vmaMakePoolAllocationsLost(
906  VmaAllocator allocator,
907  VmaPool pool,
908  size_t* pLostAllocationCount);
909 
910 VK_DEFINE_HANDLE(VmaAllocation)
911 
912 
914 typedef struct VmaAllocationInfo {
919  uint32_t memoryType;
928  VkDeviceMemory deviceMemory;
933  VkDeviceSize offset;
938  VkDeviceSize size;
944  void* pMappedData;
949  void* pUserData;
950 } VmaAllocationInfo;
951 
962 VkResult vmaAllocateMemory(
963  VmaAllocator allocator,
964  const VkMemoryRequirements* pVkMemoryRequirements,
965  const VmaAllocationCreateInfo* pCreateInfo,
966  VmaAllocation* pAllocation,
967  VmaAllocationInfo* pAllocationInfo);
968 
975 VkResult vmaAllocateMemoryForBuffer(
976  VmaAllocator allocator,
977  VkBuffer buffer,
978  const VmaAllocationCreateInfo* pCreateInfo,
979  VmaAllocation* pAllocation,
980  VmaAllocationInfo* pAllocationInfo);
981 
983 VkResult vmaAllocateMemoryForImage(
984  VmaAllocator allocator,
985  VkImage image,
986  const VmaAllocationCreateInfo* pCreateInfo,
987  VmaAllocation* pAllocation,
988  VmaAllocationInfo* pAllocationInfo);
989 
991 void vmaFreeMemory(
992  VmaAllocator allocator,
993  VmaAllocation allocation);
994 
996 void vmaGetAllocationInfo(
997  VmaAllocator allocator,
998  VmaAllocation allocation,
999  VmaAllocationInfo* pAllocationInfo);
1000 
1002 void vmaSetAllocationUserData(
1003  VmaAllocator allocator,
1004  VmaAllocation allocation,
1005  void* pUserData);
1006 
1017 void vmaCreateLostAllocation(
1018  VmaAllocator allocator,
1019  VmaAllocation* pAllocation);
1020 
1029 VkResult vmaMapMemory(
1030  VmaAllocator allocator,
1031  VmaAllocation allocation,
1032  void** ppData);
1033 
1034 void vmaUnmapMemory(
1035  VmaAllocator allocator,
1036  VmaAllocation allocation);
1037 
1059 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
1060 
1068 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
1069 
1071 typedef struct VmaDefragmentationInfo {
1076  VkDeviceSize maxBytesToMove;
1081  uint32_t maxAllocationsToMove;
1082 } VmaDefragmentationInfo;
1083 
1085 typedef struct VmaDefragmentationStats {
1087  VkDeviceSize bytesMoved;
1089  VkDeviceSize bytesFreed;
1091  uint32_t allocationsMoved;
1093  uint32_t deviceMemoryBlocksFreed;
1094 } VmaDefragmentationStats;
1095 
1166 VkResult vmaDefragment(
1167  VmaAllocator allocator,
1168  VmaAllocation* pAllocations,
1169  size_t allocationCount,
1170  VkBool32* pAllocationsChanged,
1171  const VmaDefragmentationInfo *pDefragmentationInfo,
1172  VmaDefragmentationStats* pDefragmentationStats);
1173 
1176 
1199 VkResult vmaCreateBuffer(
1200  VmaAllocator allocator,
1201  const VkBufferCreateInfo* pBufferCreateInfo,
1202  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1203  VkBuffer* pBuffer,
1204  VmaAllocation* pAllocation,
1205  VmaAllocationInfo* pAllocationInfo);
1206 
1215 void vmaDestroyBuffer(
1216  VmaAllocator allocator,
1217  VkBuffer buffer,
1218  VmaAllocation allocation);
1219 
1221 VkResult vmaCreateImage(
1222  VmaAllocator allocator,
1223  const VkImageCreateInfo* pImageCreateInfo,
1224  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1225  VkImage* pImage,
1226  VmaAllocation* pAllocation,
1227  VmaAllocationInfo* pAllocationInfo);
1228 
1237 void vmaDestroyImage(
1238  VmaAllocator allocator,
1239  VkImage image,
1240  VmaAllocation allocation);
1241 
1244 #ifdef __cplusplus
1245 }
1246 #endif
1247 
1248 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1249 
1250 // For Visual Studio IntelliSense.
1251 #ifdef __INTELLISENSE__
1252 #define VMA_IMPLEMENTATION
1253 #endif
1254 
1255 #ifdef VMA_IMPLEMENTATION
1256 #undef VMA_IMPLEMENTATION
1257 
1258 #include <cstdint>
1259 #include <cstdlib>
1260 #include <cstring>
1261 
1262 /*******************************************************************************
1263 CONFIGURATION SECTION
1264 
1265 Define some of these macros before each #include of this header or change them
1266 here if you need other then default behavior depending on your environment.
1267 */
1268 
1269 /*
1270 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1271 internally, like:
1272 
1273  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1274 
1275 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1276 VmaAllocatorCreateInfo::pVulkanFunctions.
1277 */
1278 #ifndef VMA_STATIC_VULKAN_FUNCTIONS
1279 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1280 #endif
1281 
1282 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1283 //#define VMA_USE_STL_CONTAINERS 1
1284 
1285 /* Set this macro to 1 to make the library including and using STL containers:
1286 std::pair, std::vector, std::list, std::unordered_map.
1287 
1288 Set it to 0 or undefined to make the library using its own implementation of
1289 the containers.
1290 */
1291 #if VMA_USE_STL_CONTAINERS
1292  #define VMA_USE_STL_VECTOR 1
1293  #define VMA_USE_STL_UNORDERED_MAP 1
1294  #define VMA_USE_STL_LIST 1
1295 #endif
1296 
1297 #if VMA_USE_STL_VECTOR
1298  #include <vector>
1299 #endif
1300 
1301 #if VMA_USE_STL_UNORDERED_MAP
1302  #include <unordered_map>
1303 #endif
1304 
1305 #if VMA_USE_STL_LIST
1306  #include <list>
1307 #endif
1308 
1309 /*
1310 Following headers are used in this CONFIGURATION section only, so feel free to
1311 remove them if not needed.
1312 */
1313 #include <cassert> // for assert
1314 #include <algorithm> // for min, max
1315 #include <mutex> // for std::mutex
1316 #include <atomic> // for std::atomic
1317 
1318 #if !defined(_WIN32)
1319  #include <malloc.h> // for aligned_alloc()
1320 #endif
1321 
1322 // Normal assert to check for programmer's errors, especially in Debug configuration.
1323 #ifndef VMA_ASSERT
1324  #ifdef _DEBUG
1325  #define VMA_ASSERT(expr) assert(expr)
1326  #else
1327  #define VMA_ASSERT(expr)
1328  #endif
1329 #endif
1330 
1331 // Assert that will be called very often, like inside data structures e.g. operator[].
1332 // Making it non-empty can make program slow.
1333 #ifndef VMA_HEAVY_ASSERT
1334  #ifdef _DEBUG
1335  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1336  #else
1337  #define VMA_HEAVY_ASSERT(expr)
1338  #endif
1339 #endif
1340 
1341 #ifndef VMA_NULL
1342  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1343  #define VMA_NULL nullptr
1344 #endif
1345 
1346 #ifndef VMA_ALIGN_OF
1347  #define VMA_ALIGN_OF(type) (__alignof(type))
1348 #endif
1349 
1350 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1351  #if defined(_WIN32)
1352  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1353  #else
1354  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1355  #endif
1356 #endif
1357 
1358 #ifndef VMA_SYSTEM_FREE
1359  #if defined(_WIN32)
1360  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1361  #else
1362  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1363  #endif
1364 #endif
1365 
1366 #ifndef VMA_MIN
1367  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1368 #endif
1369 
1370 #ifndef VMA_MAX
1371  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1372 #endif
1373 
1374 #ifndef VMA_SWAP
1375  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1376 #endif
1377 
1378 #ifndef VMA_SORT
1379  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1380 #endif
1381 
1382 #ifndef VMA_DEBUG_LOG
1383  #define VMA_DEBUG_LOG(format, ...)
1384  /*
1385  #define VMA_DEBUG_LOG(format, ...) do { \
1386  printf(format, __VA_ARGS__); \
1387  printf("\n"); \
1388  } while(false)
1389  */
1390 #endif
1391 
1392 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1393 #if VMA_STATS_STRING_ENABLED
1394  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1395  {
1396  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1397  }
1398  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1399  {
1400  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1401  }
1402  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1403  {
1404  snprintf(outStr, strLen, "%p", ptr);
1405  }
1406 #endif
1407 
1408 #ifndef VMA_MUTEX
1409  class VmaMutex
1410  {
1411  public:
1412  VmaMutex() { }
1413  ~VmaMutex() { }
1414  void Lock() { m_Mutex.lock(); }
1415  void Unlock() { m_Mutex.unlock(); }
1416  private:
1417  std::mutex m_Mutex;
1418  };
1419  #define VMA_MUTEX VmaMutex
1420 #endif
1421 
1422 /*
1423 If providing your own implementation, you need to implement a subset of std::atomic:
1424 
1425 - Constructor(uint32_t desired)
1426 - uint32_t load() const
1427 - void store(uint32_t desired)
1428 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1429 */
1430 #ifndef VMA_ATOMIC_UINT32
1431  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1432 #endif
1433 
1434 #ifndef VMA_BEST_FIT
1435 
1447  #define VMA_BEST_FIT (1)
1448 #endif
1449 
1450 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
1451 
1455  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
1456 #endif
1457 
1458 #ifndef VMA_DEBUG_ALIGNMENT
1459 
1463  #define VMA_DEBUG_ALIGNMENT (1)
1464 #endif
1465 
1466 #ifndef VMA_DEBUG_MARGIN
1467 
1471  #define VMA_DEBUG_MARGIN (0)
1472 #endif
1473 
1474 #ifndef VMA_DEBUG_GLOBAL_MUTEX
1475 
1479  #define VMA_DEBUG_GLOBAL_MUTEX (0)
1480 #endif
1481 
1482 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
1483 
1487  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
1488 #endif
1489 
1490 #ifndef VMA_SMALL_HEAP_MAX_SIZE
1491  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
1493 #endif
1494 
1495 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
1496  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
1498 #endif
1499 
1500 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
1501  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
1503 #endif
1504 
1505 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
1506 
1507 /*******************************************************************************
1508 END OF CONFIGURATION
1509 */
1510 
1511 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
1512  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
1513 
1514 // Returns number of bits set to 1 in (v).
1515 static inline uint32_t CountBitsSet(uint32_t v)
1516 {
1517  uint32_t c = v - ((v >> 1) & 0x55555555);
1518  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
1519  c = ((c >> 4) + c) & 0x0F0F0F0F;
1520  c = ((c >> 8) + c) & 0x00FF00FF;
1521  c = ((c >> 16) + c) & 0x0000FFFF;
1522  return c;
1523 }
1524 
1525 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
1526 // Use types like uint32_t, uint64_t as T.
1527 template <typename T>
1528 static inline T VmaAlignUp(T val, T align)
1529 {
1530  return (val + align - 1) / align * align;
1531 }
1532 
1533 // Division with mathematical rounding to nearest number.
1534 template <typename T>
1535 inline T VmaRoundDiv(T x, T y)
1536 {
1537  return (x + (y / (T)2)) / y;
1538 }
1539 
1540 #ifndef VMA_SORT
1541 
1542 template<typename Iterator, typename Compare>
1543 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
1544 {
1545  Iterator centerValue = end; --centerValue;
1546  Iterator insertIndex = beg;
1547  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
1548  {
1549  if(cmp(*memTypeIndex, *centerValue))
1550  {
1551  if(insertIndex != memTypeIndex)
1552  {
1553  VMA_SWAP(*memTypeIndex, *insertIndex);
1554  }
1555  ++insertIndex;
1556  }
1557  }
1558  if(insertIndex != centerValue)
1559  {
1560  VMA_SWAP(*insertIndex, *centerValue);
1561  }
1562  return insertIndex;
1563 }
1564 
1565 template<typename Iterator, typename Compare>
1566 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
1567 {
1568  if(beg < end)
1569  {
1570  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
1571  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
1572  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
1573  }
1574 }
1575 
1576 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
1577 
1578 #endif // #ifndef VMA_SORT
1579 
1580 /*
1581 Returns true if two memory blocks occupy overlapping pages.
1582 ResourceA must be in less memory offset than ResourceB.
1583 
1584 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1585 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1586 */
1587 static inline bool VmaBlocksOnSamePage(
1588  VkDeviceSize resourceAOffset,
1589  VkDeviceSize resourceASize,
1590  VkDeviceSize resourceBOffset,
1591  VkDeviceSize pageSize)
1592 {
1593  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1594  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1595  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1596  VkDeviceSize resourceBStart = resourceBOffset;
1597  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1598  return resourceAEndPage == resourceBStartPage;
1599 }
1600 
1601 enum VmaSuballocationType
1602 {
1603  VMA_SUBALLOCATION_TYPE_FREE = 0,
1604  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1605  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1606  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1607  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1608  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1609  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1610 };
1611 
1612 /*
1613 Returns true if given suballocation types could conflict and must respect
1614 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1615 or linear image and another one is optimal image. If type is unknown, behave
1616 conservatively.
1617 */
1618 static inline bool VmaIsBufferImageGranularityConflict(
1619  VmaSuballocationType suballocType1,
1620  VmaSuballocationType suballocType2)
1621 {
1622  if(suballocType1 > suballocType2)
1623  {
1624  VMA_SWAP(suballocType1, suballocType2);
1625  }
1626 
1627  switch(suballocType1)
1628  {
1629  case VMA_SUBALLOCATION_TYPE_FREE:
1630  return false;
1631  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1632  return true;
1633  case VMA_SUBALLOCATION_TYPE_BUFFER:
1634  return
1635  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1636  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1637  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1638  return
1639  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1640  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1641  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1642  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1643  return
1644  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1645  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1646  return false;
1647  default:
1648  VMA_ASSERT(0);
1649  return true;
1650  }
1651 }
1652 
1653 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1654 struct VmaMutexLock
1655 {
1656 public:
1657  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1658  m_pMutex(useMutex ? &mutex : VMA_NULL)
1659  {
1660  if(m_pMutex)
1661  {
1662  m_pMutex->Lock();
1663  }
1664  }
1665 
1666  ~VmaMutexLock()
1667  {
1668  if(m_pMutex)
1669  {
1670  m_pMutex->Unlock();
1671  }
1672  }
1673 
1674 private:
1675  VMA_MUTEX* m_pMutex;
1676 };
1677 
1678 #if VMA_DEBUG_GLOBAL_MUTEX
1679  static VMA_MUTEX gDebugGlobalMutex;
1680  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
1681 #else
1682  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1683 #endif
1684 
1685 // Minimum size of a free suballocation to register it in the free suballocation collection.
1686 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1687 
1688 /*
1689 Performs binary search and returns iterator to first element that is greater or
1690 equal to (key), according to comparison (cmp).
1691 
1692 Cmp should return true if first argument is less than second argument.
1693 
1694 Returned value is the found element, if present in the collection or place where
1695 new element with value (key) should be inserted.
1696 */
1697 template <typename IterT, typename KeyT, typename CmpT>
1698 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1699 {
1700  size_t down = 0, up = (end - beg);
1701  while(down < up)
1702  {
1703  const size_t mid = (down + up) / 2;
1704  if(cmp(*(beg+mid), key))
1705  {
1706  down = mid + 1;
1707  }
1708  else
1709  {
1710  up = mid;
1711  }
1712  }
1713  return beg + down;
1714 }
1715 
1717 // Memory allocation
1718 
1719 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1720 {
1721  if((pAllocationCallbacks != VMA_NULL) &&
1722  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1723  {
1724  return (*pAllocationCallbacks->pfnAllocation)(
1725  pAllocationCallbacks->pUserData,
1726  size,
1727  alignment,
1728  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1729  }
1730  else
1731  {
1732  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1733  }
1734 }
1735 
1736 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1737 {
1738  if((pAllocationCallbacks != VMA_NULL) &&
1739  (pAllocationCallbacks->pfnFree != VMA_NULL))
1740  {
1741  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1742  }
1743  else
1744  {
1745  VMA_SYSTEM_FREE(ptr);
1746  }
1747 }
1748 
1749 template<typename T>
1750 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1751 {
1752  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1753 }
1754 
1755 template<typename T>
1756 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1757 {
1758  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1759 }
1760 
1761 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1762 
1763 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1764 
1765 template<typename T>
1766 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1767 {
1768  ptr->~T();
1769  VmaFree(pAllocationCallbacks, ptr);
1770 }
1771 
1772 template<typename T>
1773 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1774 {
1775  if(ptr != VMA_NULL)
1776  {
1777  for(size_t i = count; i--; )
1778  {
1779  ptr[i].~T();
1780  }
1781  VmaFree(pAllocationCallbacks, ptr);
1782  }
1783 }
1784 
1785 // STL-compatible allocator.
1786 template<typename T>
1787 class VmaStlAllocator
1788 {
1789 public:
1790  const VkAllocationCallbacks* const m_pCallbacks;
1791  typedef T value_type;
1792 
1793  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1794  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1795 
1796  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1797  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1798 
1799  template<typename U>
1800  bool operator==(const VmaStlAllocator<U>& rhs) const
1801  {
1802  return m_pCallbacks == rhs.m_pCallbacks;
1803  }
1804  template<typename U>
1805  bool operator!=(const VmaStlAllocator<U>& rhs) const
1806  {
1807  return m_pCallbacks != rhs.m_pCallbacks;
1808  }
1809 
1810  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1811 };
1812 
1813 #if VMA_USE_STL_VECTOR
1814 
1815 #define VmaVector std::vector
1816 
1817 template<typename T, typename allocatorT>
1818 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1819 {
1820  vec.insert(vec.begin() + index, item);
1821 }
1822 
1823 template<typename T, typename allocatorT>
1824 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1825 {
1826  vec.erase(vec.begin() + index);
1827 }
1828 
1829 #else // #if VMA_USE_STL_VECTOR
1830 
1831 /* Class with interface compatible with subset of std::vector.
1832 T must be POD because constructors and destructors are not called and memcpy is
1833 used for these objects. */
1834 template<typename T, typename AllocatorT>
1835 class VmaVector
1836 {
1837 public:
1838  typedef T value_type;
1839 
1840  VmaVector(const AllocatorT& allocator) :
1841  m_Allocator(allocator),
1842  m_pArray(VMA_NULL),
1843  m_Count(0),
1844  m_Capacity(0)
1845  {
1846  }
1847 
1848  VmaVector(size_t count, const AllocatorT& allocator) :
1849  m_Allocator(allocator),
1850  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
1851  m_Count(count),
1852  m_Capacity(count)
1853  {
1854  }
1855 
1856  VmaVector(const VmaVector<T, AllocatorT>& src) :
1857  m_Allocator(src.m_Allocator),
1858  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
1859  m_Count(src.m_Count),
1860  m_Capacity(src.m_Count)
1861  {
1862  if(m_Count != 0)
1863  {
1864  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1865  }
1866  }
1867 
1868  ~VmaVector()
1869  {
1870  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1871  }
1872 
1873  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1874  {
1875  if(&rhs != this)
1876  {
1877  resize(rhs.m_Count);
1878  if(m_Count != 0)
1879  {
1880  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1881  }
1882  }
1883  return *this;
1884  }
1885 
1886  bool empty() const { return m_Count == 0; }
1887  size_t size() const { return m_Count; }
1888  T* data() { return m_pArray; }
1889  const T* data() const { return m_pArray; }
1890 
1891  T& operator[](size_t index)
1892  {
1893  VMA_HEAVY_ASSERT(index < m_Count);
1894  return m_pArray[index];
1895  }
1896  const T& operator[](size_t index) const
1897  {
1898  VMA_HEAVY_ASSERT(index < m_Count);
1899  return m_pArray[index];
1900  }
1901 
1902  T& front()
1903  {
1904  VMA_HEAVY_ASSERT(m_Count > 0);
1905  return m_pArray[0];
1906  }
1907  const T& front() const
1908  {
1909  VMA_HEAVY_ASSERT(m_Count > 0);
1910  return m_pArray[0];
1911  }
1912  T& back()
1913  {
1914  VMA_HEAVY_ASSERT(m_Count > 0);
1915  return m_pArray[m_Count - 1];
1916  }
1917  const T& back() const
1918  {
1919  VMA_HEAVY_ASSERT(m_Count > 0);
1920  return m_pArray[m_Count - 1];
1921  }
1922 
1923  void reserve(size_t newCapacity, bool freeMemory = false)
1924  {
1925  newCapacity = VMA_MAX(newCapacity, m_Count);
1926 
1927  if((newCapacity < m_Capacity) && !freeMemory)
1928  {
1929  newCapacity = m_Capacity;
1930  }
1931 
1932  if(newCapacity != m_Capacity)
1933  {
1934  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1935  if(m_Count != 0)
1936  {
1937  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1938  }
1939  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1940  m_Capacity = newCapacity;
1941  m_pArray = newArray;
1942  }
1943  }
1944 
1945  void resize(size_t newCount, bool freeMemory = false)
1946  {
1947  size_t newCapacity = m_Capacity;
1948  if(newCount > m_Capacity)
1949  {
1950  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1951  }
1952  else if(freeMemory)
1953  {
1954  newCapacity = newCount;
1955  }
1956 
1957  if(newCapacity != m_Capacity)
1958  {
1959  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1960  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1961  if(elementsToCopy != 0)
1962  {
1963  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1964  }
1965  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1966  m_Capacity = newCapacity;
1967  m_pArray = newArray;
1968  }
1969 
1970  m_Count = newCount;
1971  }
1972 
1973  void clear(bool freeMemory = false)
1974  {
1975  resize(0, freeMemory);
1976  }
1977 
1978  void insert(size_t index, const T& src)
1979  {
1980  VMA_HEAVY_ASSERT(index <= m_Count);
1981  const size_t oldCount = size();
1982  resize(oldCount + 1);
1983  if(index < oldCount)
1984  {
1985  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1986  }
1987  m_pArray[index] = src;
1988  }
1989 
1990  void remove(size_t index)
1991  {
1992  VMA_HEAVY_ASSERT(index < m_Count);
1993  const size_t oldCount = size();
1994  if(index < oldCount - 1)
1995  {
1996  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1997  }
1998  resize(oldCount - 1);
1999  }
2000 
2001  void push_back(const T& src)
2002  {
2003  const size_t newIndex = size();
2004  resize(newIndex + 1);
2005  m_pArray[newIndex] = src;
2006  }
2007 
2008  void pop_back()
2009  {
2010  VMA_HEAVY_ASSERT(m_Count > 0);
2011  resize(size() - 1);
2012  }
2013 
2014  void push_front(const T& src)
2015  {
2016  insert(0, src);
2017  }
2018 
2019  void pop_front()
2020  {
2021  VMA_HEAVY_ASSERT(m_Count > 0);
2022  remove(0);
2023  }
2024 
2025  typedef T* iterator;
2026 
2027  iterator begin() { return m_pArray; }
2028  iterator end() { return m_pArray + m_Count; }
2029 
2030 private:
2031  AllocatorT m_Allocator;
2032  T* m_pArray;
2033  size_t m_Count;
2034  size_t m_Capacity;
2035 };
2036 
2037 template<typename T, typename allocatorT>
2038 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2039 {
2040  vec.insert(index, item);
2041 }
2042 
2043 template<typename T, typename allocatorT>
2044 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2045 {
2046  vec.remove(index);
2047 }
2048 
2049 #endif // #if VMA_USE_STL_VECTOR
2050 
2051 template<typename CmpLess, typename VectorT>
2052 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2053 {
2054  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2055  vector.data(),
2056  vector.data() + vector.size(),
2057  value,
2058  CmpLess()) - vector.data();
2059  VmaVectorInsert(vector, indexToInsert, value);
2060  return indexToInsert;
2061 }
2062 
2063 template<typename CmpLess, typename VectorT>
2064 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2065 {
2066  CmpLess comparator;
2067  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2068  vector.begin(),
2069  vector.end(),
2070  value,
2071  comparator);
2072  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2073  {
2074  size_t indexToRemove = it - vector.begin();
2075  VmaVectorRemove(vector, indexToRemove);
2076  return true;
2077  }
2078  return false;
2079 }
2080 
2081 template<typename CmpLess, typename VectorT>
2082 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2083 {
2084  CmpLess comparator;
2085  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2086  vector.data(),
2087  vector.data() + vector.size(),
2088  value,
2089  comparator);
2090  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2091  {
2092  return it - vector.begin();
2093  }
2094  else
2095  {
2096  return vector.size();
2097  }
2098 }
2099 
2101 // class VmaPoolAllocator
2102 
2103 /*
2104 Allocator for objects of type T using a list of arrays (pools) to speed up
2105 allocation. Number of elements that can be allocated is not bounded because
2106 allocator can create multiple blocks.
2107 */
2108 template<typename T>
2109 class VmaPoolAllocator
2110 {
2111 public:
2112  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2113  ~VmaPoolAllocator();
2114  void Clear();
2115  T* Alloc();
2116  void Free(T* ptr);
2117 
2118 private:
2119  union Item
2120  {
2121  uint32_t NextFreeIndex;
2122  T Value;
2123  };
2124 
2125  struct ItemBlock
2126  {
2127  Item* pItems;
2128  uint32_t FirstFreeIndex;
2129  };
2130 
2131  const VkAllocationCallbacks* m_pAllocationCallbacks;
2132  size_t m_ItemsPerBlock;
2133  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2134 
2135  ItemBlock& CreateNewBlock();
2136 };
2137 
2138 template<typename T>
2139 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2140  m_pAllocationCallbacks(pAllocationCallbacks),
2141  m_ItemsPerBlock(itemsPerBlock),
2142  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2143 {
2144  VMA_ASSERT(itemsPerBlock > 0);
2145 }
2146 
2147 template<typename T>
2148 VmaPoolAllocator<T>::~VmaPoolAllocator()
2149 {
2150  Clear();
2151 }
2152 
2153 template<typename T>
2154 void VmaPoolAllocator<T>::Clear()
2155 {
2156  for(size_t i = m_ItemBlocks.size(); i--; )
2157  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2158  m_ItemBlocks.clear();
2159 }
2160 
2161 template<typename T>
2162 T* VmaPoolAllocator<T>::Alloc()
2163 {
2164  for(size_t i = m_ItemBlocks.size(); i--; )
2165  {
2166  ItemBlock& block = m_ItemBlocks[i];
2167  // This block has some free items: Use first one.
2168  if(block.FirstFreeIndex != UINT32_MAX)
2169  {
2170  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2171  block.FirstFreeIndex = pItem->NextFreeIndex;
2172  return &pItem->Value;
2173  }
2174  }
2175 
2176  // No block has free item: Create new one and use it.
2177  ItemBlock& newBlock = CreateNewBlock();
2178  Item* const pItem = &newBlock.pItems[0];
2179  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2180  return &pItem->Value;
2181 }
2182 
2183 template<typename T>
2184 void VmaPoolAllocator<T>::Free(T* ptr)
2185 {
2186  // Search all memory blocks to find ptr.
2187  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2188  {
2189  ItemBlock& block = m_ItemBlocks[i];
2190 
2191  // Casting to union.
2192  Item* pItemPtr;
2193  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2194 
2195  // Check if pItemPtr is in address range of this block.
2196  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2197  {
2198  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2199  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2200  block.FirstFreeIndex = index;
2201  return;
2202  }
2203  }
2204  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2205 }
2206 
2207 template<typename T>
2208 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2209 {
2210  ItemBlock newBlock = {
2211  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2212 
2213  m_ItemBlocks.push_back(newBlock);
2214 
2215  // Setup singly-linked list of all free items in this block.
2216  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2217  newBlock.pItems[i].NextFreeIndex = i + 1;
2218  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2219  return m_ItemBlocks.back();
2220 }
2221 
2223 // class VmaRawList, VmaList
2224 
2225 #if VMA_USE_STL_LIST
2226 
2227 #define VmaList std::list
2228 
2229 #else // #if VMA_USE_STL_LIST
2230 
2231 template<typename T>
2232 struct VmaListItem
2233 {
2234  VmaListItem* pPrev;
2235  VmaListItem* pNext;
2236  T Value;
2237 };
2238 
2239 // Doubly linked list.
2240 template<typename T>
2241 class VmaRawList
2242 {
2243 public:
2244  typedef VmaListItem<T> ItemType;
2245 
2246  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2247  ~VmaRawList();
2248  void Clear();
2249 
2250  size_t GetCount() const { return m_Count; }
2251  bool IsEmpty() const { return m_Count == 0; }
2252 
2253  ItemType* Front() { return m_pFront; }
2254  const ItemType* Front() const { return m_pFront; }
2255  ItemType* Back() { return m_pBack; }
2256  const ItemType* Back() const { return m_pBack; }
2257 
2258  ItemType* PushBack();
2259  ItemType* PushFront();
2260  ItemType* PushBack(const T& value);
2261  ItemType* PushFront(const T& value);
2262  void PopBack();
2263  void PopFront();
2264 
2265  // Item can be null - it means PushBack.
2266  ItemType* InsertBefore(ItemType* pItem);
2267  // Item can be null - it means PushFront.
2268  ItemType* InsertAfter(ItemType* pItem);
2269 
2270  ItemType* InsertBefore(ItemType* pItem, const T& value);
2271  ItemType* InsertAfter(ItemType* pItem, const T& value);
2272 
2273  void Remove(ItemType* pItem);
2274 
2275 private:
2276  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2277  VmaPoolAllocator<ItemType> m_ItemAllocator;
2278  ItemType* m_pFront;
2279  ItemType* m_pBack;
2280  size_t m_Count;
2281 
2282  // Declared not defined, to block copy constructor and assignment operator.
2283  VmaRawList(const VmaRawList<T>& src);
2284  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2285 };
2286 
2287 template<typename T>
2288 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2289  m_pAllocationCallbacks(pAllocationCallbacks),
2290  m_ItemAllocator(pAllocationCallbacks, 128),
2291  m_pFront(VMA_NULL),
2292  m_pBack(VMA_NULL),
2293  m_Count(0)
2294 {
2295 }
2296 
2297 template<typename T>
2298 VmaRawList<T>::~VmaRawList()
2299 {
2300  // Intentionally not calling Clear, because that would be unnecessary
2301  // computations to return all items to m_ItemAllocator as free.
2302 }
2303 
2304 template<typename T>
2305 void VmaRawList<T>::Clear()
2306 {
2307  if(IsEmpty() == false)
2308  {
2309  ItemType* pItem = m_pBack;
2310  while(pItem != VMA_NULL)
2311  {
2312  ItemType* const pPrevItem = pItem->pPrev;
2313  m_ItemAllocator.Free(pItem);
2314  pItem = pPrevItem;
2315  }
2316  m_pFront = VMA_NULL;
2317  m_pBack = VMA_NULL;
2318  m_Count = 0;
2319  }
2320 }
2321 
2322 template<typename T>
2323 VmaListItem<T>* VmaRawList<T>::PushBack()
2324 {
2325  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2326  pNewItem->pNext = VMA_NULL;
2327  if(IsEmpty())
2328  {
2329  pNewItem->pPrev = VMA_NULL;
2330  m_pFront = pNewItem;
2331  m_pBack = pNewItem;
2332  m_Count = 1;
2333  }
2334  else
2335  {
2336  pNewItem->pPrev = m_pBack;
2337  m_pBack->pNext = pNewItem;
2338  m_pBack = pNewItem;
2339  ++m_Count;
2340  }
2341  return pNewItem;
2342 }
2343 
2344 template<typename T>
2345 VmaListItem<T>* VmaRawList<T>::PushFront()
2346 {
2347  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2348  pNewItem->pPrev = VMA_NULL;
2349  if(IsEmpty())
2350  {
2351  pNewItem->pNext = VMA_NULL;
2352  m_pFront = pNewItem;
2353  m_pBack = pNewItem;
2354  m_Count = 1;
2355  }
2356  else
2357  {
2358  pNewItem->pNext = m_pFront;
2359  m_pFront->pPrev = pNewItem;
2360  m_pFront = pNewItem;
2361  ++m_Count;
2362  }
2363  return pNewItem;
2364 }
2365 
2366 template<typename T>
2367 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2368 {
2369  ItemType* const pNewItem = PushBack();
2370  pNewItem->Value = value;
2371  return pNewItem;
2372 }
2373 
2374 template<typename T>
2375 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2376 {
2377  ItemType* const pNewItem = PushFront();
2378  pNewItem->Value = value;
2379  return pNewItem;
2380 }
2381 
2382 template<typename T>
2383 void VmaRawList<T>::PopBack()
2384 {
2385  VMA_HEAVY_ASSERT(m_Count > 0);
2386  ItemType* const pBackItem = m_pBack;
2387  ItemType* const pPrevItem = pBackItem->pPrev;
2388  if(pPrevItem != VMA_NULL)
2389  {
2390  pPrevItem->pNext = VMA_NULL;
2391  }
2392  m_pBack = pPrevItem;
2393  m_ItemAllocator.Free(pBackItem);
2394  --m_Count;
2395 }
2396 
2397 template<typename T>
2398 void VmaRawList<T>::PopFront()
2399 {
2400  VMA_HEAVY_ASSERT(m_Count > 0);
2401  ItemType* const pFrontItem = m_pFront;
2402  ItemType* const pNextItem = pFrontItem->pNext;
2403  if(pNextItem != VMA_NULL)
2404  {
2405  pNextItem->pPrev = VMA_NULL;
2406  }
2407  m_pFront = pNextItem;
2408  m_ItemAllocator.Free(pFrontItem);
2409  --m_Count;
2410 }
2411 
2412 template<typename T>
2413 void VmaRawList<T>::Remove(ItemType* pItem)
2414 {
2415  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2416  VMA_HEAVY_ASSERT(m_Count > 0);
2417 
2418  if(pItem->pPrev != VMA_NULL)
2419  {
2420  pItem->pPrev->pNext = pItem->pNext;
2421  }
2422  else
2423  {
2424  VMA_HEAVY_ASSERT(m_pFront == pItem);
2425  m_pFront = pItem->pNext;
2426  }
2427 
2428  if(pItem->pNext != VMA_NULL)
2429  {
2430  pItem->pNext->pPrev = pItem->pPrev;
2431  }
2432  else
2433  {
2434  VMA_HEAVY_ASSERT(m_pBack == pItem);
2435  m_pBack = pItem->pPrev;
2436  }
2437 
2438  m_ItemAllocator.Free(pItem);
2439  --m_Count;
2440 }
2441 
2442 template<typename T>
2443 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2444 {
2445  if(pItem != VMA_NULL)
2446  {
2447  ItemType* const prevItem = pItem->pPrev;
2448  ItemType* const newItem = m_ItemAllocator.Alloc();
2449  newItem->pPrev = prevItem;
2450  newItem->pNext = pItem;
2451  pItem->pPrev = newItem;
2452  if(prevItem != VMA_NULL)
2453  {
2454  prevItem->pNext = newItem;
2455  }
2456  else
2457  {
2458  VMA_HEAVY_ASSERT(m_pFront == pItem);
2459  m_pFront = newItem;
2460  }
2461  ++m_Count;
2462  return newItem;
2463  }
2464  else
2465  return PushBack();
2466 }
2467 
2468 template<typename T>
2469 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2470 {
2471  if(pItem != VMA_NULL)
2472  {
2473  ItemType* const nextItem = pItem->pNext;
2474  ItemType* const newItem = m_ItemAllocator.Alloc();
2475  newItem->pNext = nextItem;
2476  newItem->pPrev = pItem;
2477  pItem->pNext = newItem;
2478  if(nextItem != VMA_NULL)
2479  {
2480  nextItem->pPrev = newItem;
2481  }
2482  else
2483  {
2484  VMA_HEAVY_ASSERT(m_pBack == pItem);
2485  m_pBack = newItem;
2486  }
2487  ++m_Count;
2488  return newItem;
2489  }
2490  else
2491  return PushFront();
2492 }
2493 
2494 template<typename T>
2495 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
2496 {
2497  ItemType* const newItem = InsertBefore(pItem);
2498  newItem->Value = value;
2499  return newItem;
2500 }
2501 
2502 template<typename T>
2503 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
2504 {
2505  ItemType* const newItem = InsertAfter(pItem);
2506  newItem->Value = value;
2507  return newItem;
2508 }
2509 
2510 template<typename T, typename AllocatorT>
2511 class VmaList
2512 {
2513 public:
2514  class iterator
2515  {
2516  public:
2517  iterator() :
2518  m_pList(VMA_NULL),
2519  m_pItem(VMA_NULL)
2520  {
2521  }
2522 
2523  T& operator*() const
2524  {
2525  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2526  return m_pItem->Value;
2527  }
2528  T* operator->() const
2529  {
2530  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2531  return &m_pItem->Value;
2532  }
2533 
2534  iterator& operator++()
2535  {
2536  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2537  m_pItem = m_pItem->pNext;
2538  return *this;
2539  }
2540  iterator& operator--()
2541  {
2542  if(m_pItem != VMA_NULL)
2543  {
2544  m_pItem = m_pItem->pPrev;
2545  }
2546  else
2547  {
2548  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
2549  m_pItem = m_pList->Back();
2550  }
2551  return *this;
2552  }
2553 
2554  iterator operator++(int)
2555  {
2556  iterator result = *this;
2557  ++*this;
2558  return result;
2559  }
2560  iterator operator--(int)
2561  {
2562  iterator result = *this;
2563  --*this;
2564  return result;
2565  }
2566 
2567  bool operator==(const iterator& rhs) const
2568  {
2569  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2570  return m_pItem == rhs.m_pItem;
2571  }
2572  bool operator!=(const iterator& rhs) const
2573  {
2574  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2575  return m_pItem != rhs.m_pItem;
2576  }
2577 
2578  private:
2579  VmaRawList<T>* m_pList;
2580  VmaListItem<T>* m_pItem;
2581 
2582  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
2583  m_pList(pList),
2584  m_pItem(pItem)
2585  {
2586  }
2587 
2588  friend class VmaList<T, AllocatorT>;
2589  };
2590 
2591  class const_iterator
2592  {
2593  public:
2594  const_iterator() :
2595  m_pList(VMA_NULL),
2596  m_pItem(VMA_NULL)
2597  {
2598  }
2599 
2600  const_iterator(const iterator& src) :
2601  m_pList(src.m_pList),
2602  m_pItem(src.m_pItem)
2603  {
2604  }
2605 
2606  const T& operator*() const
2607  {
2608  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2609  return m_pItem->Value;
2610  }
2611  const T* operator->() const
2612  {
2613  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2614  return &m_pItem->Value;
2615  }
2616 
2617  const_iterator& operator++()
2618  {
2619  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2620  m_pItem = m_pItem->pNext;
2621  return *this;
2622  }
2623  const_iterator& operator--()
2624  {
2625  if(m_pItem != VMA_NULL)
2626  {
2627  m_pItem = m_pItem->pPrev;
2628  }
2629  else
2630  {
2631  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2632  m_pItem = m_pList->Back();
2633  }
2634  return *this;
2635  }
2636 
2637  const_iterator operator++(int)
2638  {
2639  const_iterator result = *this;
2640  ++*this;
2641  return result;
2642  }
2643  const_iterator operator--(int)
2644  {
2645  const_iterator result = *this;
2646  --*this;
2647  return result;
2648  }
2649 
2650  bool operator==(const const_iterator& rhs) const
2651  {
2652  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2653  return m_pItem == rhs.m_pItem;
2654  }
2655  bool operator!=(const const_iterator& rhs) const
2656  {
2657  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2658  return m_pItem != rhs.m_pItem;
2659  }
2660 
2661  private:
2662  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2663  m_pList(pList),
2664  m_pItem(pItem)
2665  {
2666  }
2667 
2668  const VmaRawList<T>* m_pList;
2669  const VmaListItem<T>* m_pItem;
2670 
2671  friend class VmaList<T, AllocatorT>;
2672  };
2673 
2674  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2675 
2676  bool empty() const { return m_RawList.IsEmpty(); }
2677  size_t size() const { return m_RawList.GetCount(); }
2678 
2679  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2680  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2681 
2682  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2683  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2684 
2685  void clear() { m_RawList.Clear(); }
2686  void push_back(const T& value) { m_RawList.PushBack(value); }
2687  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2688  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2689 
2690 private:
2691  VmaRawList<T> m_RawList;
2692 };
2693 
2694 #endif // #if VMA_USE_STL_LIST
2695 
2697 // class VmaMap
2698 
2699 // Unused in this version.
2700 #if 0
2701 
2702 #if VMA_USE_STL_UNORDERED_MAP
2703 
2704 #define VmaPair std::pair
2705 
2706 #define VMA_MAP_TYPE(KeyT, ValueT) \
2707  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2708 
2709 #else // #if VMA_USE_STL_UNORDERED_MAP
2710 
2711 template<typename T1, typename T2>
2712 struct VmaPair
2713 {
2714  T1 first;
2715  T2 second;
2716 
2717  VmaPair() : first(), second() { }
2718  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2719 };
2720 
2721 /* Class compatible with subset of interface of std::unordered_map.
2722 KeyT, ValueT must be POD because they will be stored in VmaVector.
2723 */
2724 template<typename KeyT, typename ValueT>
2725 class VmaMap
2726 {
2727 public:
2728  typedef VmaPair<KeyT, ValueT> PairType;
2729  typedef PairType* iterator;
2730 
2731  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2732 
2733  iterator begin() { return m_Vector.begin(); }
2734  iterator end() { return m_Vector.end(); }
2735 
2736  void insert(const PairType& pair);
2737  iterator find(const KeyT& key);
2738  void erase(iterator it);
2739 
2740 private:
2741  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2742 };
2743 
2744 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2745 
2746 template<typename FirstT, typename SecondT>
2747 struct VmaPairFirstLess
2748 {
2749  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2750  {
2751  return lhs.first < rhs.first;
2752  }
2753  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2754  {
2755  return lhs.first < rhsFirst;
2756  }
2757 };
2758 
2759 template<typename KeyT, typename ValueT>
2760 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2761 {
2762  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2763  m_Vector.data(),
2764  m_Vector.data() + m_Vector.size(),
2765  pair,
2766  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2767  VmaVectorInsert(m_Vector, indexToInsert, pair);
2768 }
2769 
2770 template<typename KeyT, typename ValueT>
2771 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2772 {
2773  PairType* it = VmaBinaryFindFirstNotLess(
2774  m_Vector.data(),
2775  m_Vector.data() + m_Vector.size(),
2776  key,
2777  VmaPairFirstLess<KeyT, ValueT>());
2778  if((it != m_Vector.end()) && (it->first == key))
2779  {
2780  return it;
2781  }
2782  else
2783  {
2784  return m_Vector.end();
2785  }
2786 }
2787 
2788 template<typename KeyT, typename ValueT>
2789 void VmaMap<KeyT, ValueT>::erase(iterator it)
2790 {
2791  VmaVectorRemove(m_Vector, it - m_Vector.begin());
2792 }
2793 
2794 #endif // #if VMA_USE_STL_UNORDERED_MAP
2795 
2796 #endif // #if 0
2797 
2799 
2800 class VmaDeviceMemoryBlock;
2801 
2802 enum VMA_BLOCK_VECTOR_TYPE
2803 {
2804  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2805  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2806  VMA_BLOCK_VECTOR_TYPE_COUNT
2807 };
2808 
2809 static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags)
2810 {
2811  return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ?
2812  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2813  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2814 }
2815 
2816 struct VmaAllocation_T
2817 {
2818 public:
2819  enum ALLOCATION_TYPE
2820  {
2821  ALLOCATION_TYPE_NONE,
2822  ALLOCATION_TYPE_BLOCK,
2823  ALLOCATION_TYPE_OWN,
2824  };
2825 
2826  VmaAllocation_T(uint32_t currentFrameIndex) :
2827  m_Alignment(1),
2828  m_Size(0),
2829  m_pUserData(VMA_NULL),
2830  m_Type(ALLOCATION_TYPE_NONE),
2831  m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN),
2832  m_LastUseFrameIndex(currentFrameIndex)
2833  {
2834  }
2835 
2836  void InitBlockAllocation(
2837  VmaPool hPool,
2838  VmaDeviceMemoryBlock* block,
2839  VkDeviceSize offset,
2840  VkDeviceSize alignment,
2841  VkDeviceSize size,
2842  VmaSuballocationType suballocationType,
2843  void* pUserData,
2844  bool canBecomeLost)
2845  {
2846  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2847  VMA_ASSERT(block != VMA_NULL);
2848  m_Type = ALLOCATION_TYPE_BLOCK;
2849  m_Alignment = alignment;
2850  m_Size = size;
2851  m_pUserData = pUserData;
2852  m_SuballocationType = suballocationType;
2853  m_BlockAllocation.m_hPool = hPool;
2854  m_BlockAllocation.m_Block = block;
2855  m_BlockAllocation.m_Offset = offset;
2856  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
2857  }
2858 
2859  void InitLost()
2860  {
2861  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2862  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
2863  m_Type = ALLOCATION_TYPE_BLOCK;
2864  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
2865  m_BlockAllocation.m_Block = VMA_NULL;
2866  m_BlockAllocation.m_Offset = 0;
2867  m_BlockAllocation.m_CanBecomeLost = true;
2868  }
2869 
2870  void ChangeBlockAllocation(
2871  VmaDeviceMemoryBlock* block,
2872  VkDeviceSize offset)
2873  {
2874  VMA_ASSERT(block != VMA_NULL);
2875  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2876  m_BlockAllocation.m_Block = block;
2877  m_BlockAllocation.m_Offset = offset;
2878  }
2879 
2880  void InitOwnAllocation(
2881  uint32_t memoryTypeIndex,
2882  VkDeviceMemory hMemory,
2883  VmaSuballocationType suballocationType,
2884  bool persistentMap,
2885  void* pMappedData,
2886  VkDeviceSize size,
2887  void* pUserData)
2888  {
2889  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2890  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2891  m_Type = ALLOCATION_TYPE_OWN;
2892  m_Alignment = 0;
2893  m_Size = size;
2894  m_pUserData = pUserData;
2895  m_SuballocationType = suballocationType;
2896  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2897  m_OwnAllocation.m_hMemory = hMemory;
2898  m_OwnAllocation.m_PersistentMap = persistentMap;
2899  m_OwnAllocation.m_pMappedData = pMappedData;
2900  }
2901 
2902  ALLOCATION_TYPE GetType() const { return m_Type; }
2903  VkDeviceSize GetAlignment() const { return m_Alignment; }
2904  VkDeviceSize GetSize() const { return m_Size; }
2905  void* GetUserData() const { return m_pUserData; }
2906  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2907  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2908 
2909  VmaDeviceMemoryBlock* GetBlock() const
2910  {
2911  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2912  return m_BlockAllocation.m_Block;
2913  }
2914  VkDeviceSize GetOffset() const;
2915  VkDeviceMemory GetMemory() const;
2916  uint32_t GetMemoryTypeIndex() const;
2917  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2918  void* GetMappedData() const;
2919  bool CanBecomeLost() const;
2920  VmaPool GetPool() const;
2921 
2922  VkResult OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator);
2923  void OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator);
2924 
2925  uint32_t GetLastUseFrameIndex() const
2926  {
2927  return m_LastUseFrameIndex.load();
2928  }
2929  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
2930  {
2931  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
2932  }
2933  /*
2934  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
2935  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
2936  - Else, returns false.
2937 
2938  If hAllocation is already lost, assert - you should not call it then.
2939  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
2940  */
2941  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
2942 
2943  void OwnAllocCalcStatsInfo(VmaStatInfo& outInfo)
2944  {
2945  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2946  outInfo.blockCount = 1;
2947  outInfo.allocationCount = 1;
2948  outInfo.unusedRangeCount = 0;
2949  outInfo.usedBytes = m_Size;
2950  outInfo.unusedBytes = 0;
2951  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
2952  outInfo.unusedRangeSizeMin = UINT64_MAX;
2953  outInfo.unusedRangeSizeMax = 0;
2954  }
2955 
2956 private:
2957  VkDeviceSize m_Alignment;
2958  VkDeviceSize m_Size;
2959  void* m_pUserData;
2960  ALLOCATION_TYPE m_Type;
2961  VmaSuballocationType m_SuballocationType;
2962  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
2963 
2964  // Allocation out of VmaDeviceMemoryBlock.
2965  struct BlockAllocation
2966  {
2967  VmaPool m_hPool; // Null if belongs to general memory.
2968  VmaDeviceMemoryBlock* m_Block;
2969  VkDeviceSize m_Offset;
2970  bool m_CanBecomeLost;
2971  };
2972 
2973  // Allocation for an object that has its own private VkDeviceMemory.
2974  struct OwnAllocation
2975  {
2976  uint32_t m_MemoryTypeIndex;
2977  VkDeviceMemory m_hMemory;
2978  bool m_PersistentMap;
2979  void* m_pMappedData;
2980  };
2981 
2982  union
2983  {
2984  // Allocation out of VmaDeviceMemoryBlock.
2985  BlockAllocation m_BlockAllocation;
2986  // Allocation for an object that has its own private VkDeviceMemory.
2987  OwnAllocation m_OwnAllocation;
2988  };
2989 };
2990 
2991 /*
2992 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
2993 allocated memory block or free.
2994 */
2995 struct VmaSuballocation
2996 {
2997  VkDeviceSize offset;
2998  VkDeviceSize size;
2999  VmaAllocation hAllocation;
3000  VmaSuballocationType type;
3001 };
3002 
3003 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3004 
3005 // Cost of one additional allocation lost, as equivalent in bytes.
3006 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3007 
3008 /*
3009 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3010 
3011 If canMakeOtherLost was false:
3012 - item points to a FREE suballocation.
3013 - itemsToMakeLostCount is 0.
3014 
3015 If canMakeOtherLost was true:
3016 - item points to first of sequence of suballocations, which are either FREE,
3017  or point to VmaAllocations that can become lost.
3018 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3019  the requested allocation to succeed.
3020 */
3021 struct VmaAllocationRequest
3022 {
3023  VkDeviceSize offset;
3024  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3025  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3026  VmaSuballocationList::iterator item;
3027  size_t itemsToMakeLostCount;
3028 
3029  VkDeviceSize CalcCost() const
3030  {
3031  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3032  }
3033 };
3034 
3035 /*
3036 Data structure used for bookkeeping of allocations and unused ranges of memory
3037 in a single VkDeviceMemory block.
3038 */
3039 class VmaBlockMetadata
3040 {
3041 public:
3042  VmaBlockMetadata(VmaAllocator hAllocator);
3043  ~VmaBlockMetadata();
3044  void Init(VkDeviceSize size);
3045 
3046  // Validates all data structures inside this object. If not valid, returns false.
3047  bool Validate() const;
3048  VkDeviceSize GetSize() const { return m_Size; }
3049  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3050  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3051  VkDeviceSize GetUnusedRangeSizeMax() const;
3052  // Returns true if this block is empty - contains only single free suballocation.
3053  bool IsEmpty() const;
3054 
3055  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3056  void AddPoolStats(VmaPoolStats& inoutStats) const;
3057 
3058 #if VMA_STATS_STRING_ENABLED
3059  void PrintDetailedMap(class VmaJsonWriter& json) const;
3060 #endif
3061 
3062  // Creates trivial request for case when block is empty.
3063  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3064 
3065  // Tries to find a place for suballocation with given parameters inside this block.
3066  // If succeeded, fills pAllocationRequest and returns true.
3067  // If failed, returns false.
3068  bool CreateAllocationRequest(
3069  uint32_t currentFrameIndex,
3070  uint32_t frameInUseCount,
3071  VkDeviceSize bufferImageGranularity,
3072  VkDeviceSize allocSize,
3073  VkDeviceSize allocAlignment,
3074  VmaSuballocationType allocType,
3075  bool canMakeOtherLost,
3076  VmaAllocationRequest* pAllocationRequest);
3077 
3078  bool MakeRequestedAllocationsLost(
3079  uint32_t currentFrameIndex,
3080  uint32_t frameInUseCount,
3081  VmaAllocationRequest* pAllocationRequest);
3082 
3083  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3084 
3085  // Makes actual allocation based on request. Request must already be checked and valid.
3086  void Alloc(
3087  const VmaAllocationRequest& request,
3088  VmaSuballocationType type,
3089  VkDeviceSize allocSize,
3090  VmaAllocation hAllocation);
3091 
3092  // Frees suballocation assigned to given memory region.
3093  void Free(const VmaAllocation allocation);
3094 
3095 private:
3096  VkDeviceSize m_Size;
3097  uint32_t m_FreeCount;
3098  VkDeviceSize m_SumFreeSize;
3099  VmaSuballocationList m_Suballocations;
3100  // Suballocations that are free and have size greater than certain threshold.
3101  // Sorted by size, ascending.
3102  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3103 
3104  bool ValidateFreeSuballocationList() const;
3105 
3106  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3107  // If yes, fills pOffset and returns true. If no, returns false.
3108  bool CheckAllocation(
3109  uint32_t currentFrameIndex,
3110  uint32_t frameInUseCount,
3111  VkDeviceSize bufferImageGranularity,
3112  VkDeviceSize allocSize,
3113  VkDeviceSize allocAlignment,
3114  VmaSuballocationType allocType,
3115  VmaSuballocationList::const_iterator suballocItem,
3116  bool canMakeOtherLost,
3117  VkDeviceSize* pOffset,
3118  size_t* itemsToMakeLostCount,
3119  VkDeviceSize* pSumFreeSize,
3120  VkDeviceSize* pSumItemSize) const;
3121  // Given free suballocation, it merges it with following one, which must also be free.
3122  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3123  // Releases given suballocation, making it free.
3124  // Merges it with adjacent free suballocations if applicable.
3125  // Returns iterator to new free suballocation at this place.
3126  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3127  // Given free suballocation, it inserts it into sorted list of
3128  // m_FreeSuballocationsBySize if it's suitable.
3129  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3130  // Given free suballocation, it removes it from sorted list of
3131  // m_FreeSuballocationsBySize if it's suitable.
3132  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3133 };
3134 
3135 /*
3136 Represents a single block of device memory (`VkDeviceMemory`) with all the
3137 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3138 
3139 Thread-safety: This class must be externally synchronized.
3140 */
3141 class VmaDeviceMemoryBlock
3142 {
3143 public:
3144  uint32_t m_MemoryTypeIndex;
3145  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3146  VkDeviceMemory m_hMemory;
3147  bool m_PersistentMap;
3148  void* m_pMappedData;
3149  VmaBlockMetadata m_Metadata;
3150 
3151  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3152 
3153  ~VmaDeviceMemoryBlock()
3154  {
3155  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3156  }
3157 
3158  // Always call after construction.
3159  void Init(
3160  uint32_t newMemoryTypeIndex,
3161  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
3162  VkDeviceMemory newMemory,
3163  VkDeviceSize newSize,
3164  bool persistentMap,
3165  void* pMappedData);
3166  // Always call before destruction.
3167  void Destroy(VmaAllocator allocator);
3168 
3169  // Validates all data structures inside this object. If not valid, returns false.
3170  bool Validate() const;
3171 };
3172 
3173 struct VmaPointerLess
3174 {
3175  bool operator()(const void* lhs, const void* rhs) const
3176  {
3177  return lhs < rhs;
3178  }
3179 };
3180 
3181 class VmaDefragmentator;
3182 
3183 /*
3184 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3185 Vulkan memory type.
3186 
3187 Synchronized internally with a mutex.
3188 */
3189 struct VmaBlockVector
3190 {
3191  VmaBlockVector(
3192  VmaAllocator hAllocator,
3193  uint32_t memoryTypeIndex,
3194  VMA_BLOCK_VECTOR_TYPE blockVectorType,
3195  VkDeviceSize preferredBlockSize,
3196  size_t minBlockCount,
3197  size_t maxBlockCount,
3198  VkDeviceSize bufferImageGranularity,
3199  uint32_t frameInUseCount,
3200  bool isCustomPool);
3201  ~VmaBlockVector();
3202 
3203  VkResult CreateMinBlocks();
3204 
3205  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3206  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3207  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3208  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3209  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; }
3210 
3211  void GetPoolStats(VmaPoolStats* pStats);
3212 
3213  bool IsEmpty() const { return m_Blocks.empty(); }
3214 
3215  VkResult Allocate(
3216  VmaPool hCurrentPool,
3217  uint32_t currentFrameIndex,
3218  const VkMemoryRequirements& vkMemReq,
3219  const VmaAllocationCreateInfo& createInfo,
3220  VmaSuballocationType suballocType,
3221  VmaAllocation* pAllocation);
3222 
3223  void Free(
3224  VmaAllocation hAllocation);
3225 
3226  // Adds statistics of this BlockVector to pStats.
3227  void AddStats(VmaStats* pStats);
3228 
3229 #if VMA_STATS_STRING_ENABLED
3230  void PrintDetailedMap(class VmaJsonWriter& json);
3231 #endif
3232 
3233  void UnmapPersistentlyMappedMemory();
3234  VkResult MapPersistentlyMappedMemory();
3235 
3236  void MakePoolAllocationsLost(
3237  uint32_t currentFrameIndex,
3238  size_t* pLostAllocationCount);
3239 
3240  VmaDefragmentator* EnsureDefragmentator(
3241  VmaAllocator hAllocator,
3242  uint32_t currentFrameIndex);
3243 
3244  VkResult Defragment(
3245  VmaDefragmentationStats* pDefragmentationStats,
3246  VkDeviceSize& maxBytesToMove,
3247  uint32_t& maxAllocationsToMove);
3248 
3249  void DestroyDefragmentator();
3250 
3251 private:
3252  friend class VmaDefragmentator;
3253 
3254  const VmaAllocator m_hAllocator;
3255  const uint32_t m_MemoryTypeIndex;
3256  const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3257  const VkDeviceSize m_PreferredBlockSize;
3258  const size_t m_MinBlockCount;
3259  const size_t m_MaxBlockCount;
3260  const VkDeviceSize m_BufferImageGranularity;
3261  const uint32_t m_FrameInUseCount;
3262  const bool m_IsCustomPool;
3263  VMA_MUTEX m_Mutex;
3264  // Incrementally sorted by sumFreeSize, ascending.
3265  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3266  /* There can be at most one allocation that is completely empty - a
3267  hysteresis to avoid pessimistic case of alternating creation and destruction
3268  of a VkDeviceMemory. */
3269  bool m_HasEmptyBlock;
3270  VmaDefragmentator* m_pDefragmentator;
3271 
3272  // Finds and removes given block from vector.
3273  void Remove(VmaDeviceMemoryBlock* pBlock);
3274 
3275  // Performs single step in sorting m_Blocks. They may not be fully sorted
3276  // after this call.
3277  void IncrementallySortBlocks();
3278 
3279  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3280 };
3281 
3282 struct VmaPool_T
3283 {
3284 public:
3285  VmaBlockVector m_BlockVector;
3286 
3287  // Takes ownership.
3288  VmaPool_T(
3289  VmaAllocator hAllocator,
3290  const VmaPoolCreateInfo& createInfo);
3291  ~VmaPool_T();
3292 
3293  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3294 
3295 #if VMA_STATS_STRING_ENABLED
3296  //void PrintDetailedMap(class VmaStringBuilder& sb);
3297 #endif
3298 };
3299 
3300 class VmaDefragmentator
3301 {
3302  const VmaAllocator m_hAllocator;
3303  VmaBlockVector* const m_pBlockVector;
3304  uint32_t m_CurrentFrameIndex;
3305  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3306  VkDeviceSize m_BytesMoved;
3307  uint32_t m_AllocationsMoved;
3308 
3309  struct AllocationInfo
3310  {
3311  VmaAllocation m_hAllocation;
3312  VkBool32* m_pChanged;
3313 
3314  AllocationInfo() :
3315  m_hAllocation(VK_NULL_HANDLE),
3316  m_pChanged(VMA_NULL)
3317  {
3318  }
3319  };
3320 
3321  struct AllocationInfoSizeGreater
3322  {
3323  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3324  {
3325  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3326  }
3327  };
3328 
3329  // Used between AddAllocation and Defragment.
3330  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3331 
3332  struct BlockInfo
3333  {
3334  VmaDeviceMemoryBlock* m_pBlock;
3335  bool m_HasNonMovableAllocations;
3336  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3337 
3338  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3339  m_pBlock(VMA_NULL),
3340  m_HasNonMovableAllocations(true),
3341  m_Allocations(pAllocationCallbacks),
3342  m_pMappedDataForDefragmentation(VMA_NULL)
3343  {
3344  }
3345 
3346  void CalcHasNonMovableAllocations()
3347  {
3348  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3349  const size_t defragmentAllocCount = m_Allocations.size();
3350  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3351  }
3352 
3353  void SortAllocationsBySizeDescecnding()
3354  {
3355  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3356  }
3357 
3358  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3359  void Unmap(VmaAllocator hAllocator);
3360 
3361  private:
3362  // Not null if mapped for defragmentation only, not persistently mapped.
3363  void* m_pMappedDataForDefragmentation;
3364  };
3365 
3366  struct BlockPointerLess
3367  {
3368  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3369  {
3370  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3371  }
3372  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3373  {
3374  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3375  }
3376  };
3377 
3378  // 1. Blocks with some non-movable allocations go first.
3379  // 2. Blocks with smaller sumFreeSize go first.
3380  struct BlockInfoCompareMoveDestination
3381  {
3382  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3383  {
3384  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3385  {
3386  return true;
3387  }
3388  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3389  {
3390  return false;
3391  }
3392  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3393  {
3394  return true;
3395  }
3396  return false;
3397  }
3398  };
3399 
3400  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3401  BlockInfoVector m_Blocks;
3402 
3403  VkResult DefragmentRound(
3404  VkDeviceSize maxBytesToMove,
3405  uint32_t maxAllocationsToMove);
3406 
3407  static bool MoveMakesSense(
3408  size_t dstBlockIndex, VkDeviceSize dstOffset,
3409  size_t srcBlockIndex, VkDeviceSize srcOffset);
3410 
3411 public:
3412  VmaDefragmentator(
3413  VmaAllocator hAllocator,
3414  VmaBlockVector* pBlockVector,
3415  uint32_t currentFrameIndex);
3416 
3417  ~VmaDefragmentator();
3418 
3419  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3420  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3421 
3422  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3423 
3424  VkResult Defragment(
3425  VkDeviceSize maxBytesToMove,
3426  uint32_t maxAllocationsToMove);
3427 };
3428 
3429 // Main allocator object.
3430 struct VmaAllocator_T
3431 {
3432  bool m_UseMutex;
3433  VkDevice m_hDevice;
3434  bool m_AllocationCallbacksSpecified;
3435  VkAllocationCallbacks m_AllocationCallbacks;
3436  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3437  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
3438  // Counter to allow nested calls to these functions.
3439  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
3440 
3441  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3442  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3443  VMA_MUTEX m_HeapSizeLimitMutex;
3444 
3445  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3446  VkPhysicalDeviceMemoryProperties m_MemProps;
3447 
3448  // Default pools.
3449  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3450 
3451  // Each vector is sorted by memory (handle value).
3452  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3453  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3454  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
3455 
3456  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
3457  ~VmaAllocator_T();
3458 
3459  const VkAllocationCallbacks* GetAllocationCallbacks() const
3460  {
3461  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
3462  }
3463  const VmaVulkanFunctions& GetVulkanFunctions() const
3464  {
3465  return m_VulkanFunctions;
3466  }
3467 
3468  VkDeviceSize GetBufferImageGranularity() const
3469  {
3470  return VMA_MAX(
3471  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
3472  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
3473  }
3474 
3475  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
3476  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
3477 
3478  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
3479  {
3480  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
3481  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3482  }
3483 
3484  // Main allocation function.
3485  VkResult AllocateMemory(
3486  const VkMemoryRequirements& vkMemReq,
3487  const VmaAllocationCreateInfo& createInfo,
3488  VmaSuballocationType suballocType,
3489  VmaAllocation* pAllocation);
3490 
3491  // Main deallocation function.
3492  void FreeMemory(const VmaAllocation allocation);
3493 
3494  void CalculateStats(VmaStats* pStats);
3495 
3496 #if VMA_STATS_STRING_ENABLED
3497  void PrintDetailedMap(class VmaJsonWriter& json);
3498 #endif
3499 
3500  void UnmapPersistentlyMappedMemory();
3501  VkResult MapPersistentlyMappedMemory();
3502 
3503  VkResult Defragment(
3504  VmaAllocation* pAllocations,
3505  size_t allocationCount,
3506  VkBool32* pAllocationsChanged,
3507  const VmaDefragmentationInfo* pDefragmentationInfo,
3508  VmaDefragmentationStats* pDefragmentationStats);
3509 
3510  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
3511 
3512  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
3513  void DestroyPool(VmaPool pool);
3514  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
3515 
3516  void SetCurrentFrameIndex(uint32_t frameIndex);
3517 
3518  void MakePoolAllocationsLost(
3519  VmaPool hPool,
3520  size_t* pLostAllocationCount);
3521 
3522  void CreateLostAllocation(VmaAllocation* pAllocation);
3523 
3524  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
3525  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
3526 
3527 private:
3528  VkDeviceSize m_PreferredLargeHeapBlockSize;
3529  VkDeviceSize m_PreferredSmallHeapBlockSize;
3530 
3531  VkPhysicalDevice m_PhysicalDevice;
3532  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
3533 
3534  VMA_MUTEX m_PoolsMutex;
3535  // Protected by m_PoolsMutex. Sorted by pointer value.
3536  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
3537 
3538  VmaVulkanFunctions m_VulkanFunctions;
3539 
3540  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
3541 
3542  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
3543 
3544  VkResult AllocateMemoryOfType(
3545  const VkMemoryRequirements& vkMemReq,
3546  const VmaAllocationCreateInfo& createInfo,
3547  uint32_t memTypeIndex,
3548  VmaSuballocationType suballocType,
3549  VmaAllocation* pAllocation);
3550 
3551  // Allocates and registers new VkDeviceMemory specifically for single allocation.
3552  VkResult AllocateOwnMemory(
3553  VkDeviceSize size,
3554  VmaSuballocationType suballocType,
3555  uint32_t memTypeIndex,
3556  bool map,
3557  void* pUserData,
3558  VmaAllocation* pAllocation);
3559 
3560  // Tries to free pMemory as Own Memory. Returns true if found and freed.
3561  void FreeOwnMemory(VmaAllocation allocation);
3562 };
3563 
3565 // Memory allocation #2 after VmaAllocator_T definition
3566 
3567 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
3568 {
3569  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
3570 }
3571 
3572 static void VmaFree(VmaAllocator hAllocator, void* ptr)
3573 {
3574  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
3575 }
3576 
3577 template<typename T>
3578 static T* VmaAllocate(VmaAllocator hAllocator)
3579 {
3580  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
3581 }
3582 
3583 template<typename T>
3584 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
3585 {
3586  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
3587 }
3588 
3589 template<typename T>
3590 static void vma_delete(VmaAllocator hAllocator, T* ptr)
3591 {
3592  if(ptr != VMA_NULL)
3593  {
3594  ptr->~T();
3595  VmaFree(hAllocator, ptr);
3596  }
3597 }
3598 
3599 template<typename T>
3600 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
3601 {
3602  if(ptr != VMA_NULL)
3603  {
3604  for(size_t i = count; i--; )
3605  ptr[i].~T();
3606  VmaFree(hAllocator, ptr);
3607  }
3608 }
3609 
3611 // VmaStringBuilder
3612 
3613 #if VMA_STATS_STRING_ENABLED
3614 
3615 class VmaStringBuilder
3616 {
3617 public:
3618  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
3619  size_t GetLength() const { return m_Data.size(); }
3620  const char* GetData() const { return m_Data.data(); }
3621 
3622  void Add(char ch) { m_Data.push_back(ch); }
3623  void Add(const char* pStr);
3624  void AddNewLine() { Add('\n'); }
3625  void AddNumber(uint32_t num);
3626  void AddNumber(uint64_t num);
3627  void AddPointer(const void* ptr);
3628 
3629 private:
3630  VmaVector< char, VmaStlAllocator<char> > m_Data;
3631 };
3632 
3633 void VmaStringBuilder::Add(const char* pStr)
3634 {
3635  const size_t strLen = strlen(pStr);
3636  if(strLen > 0)
3637  {
3638  const size_t oldCount = m_Data.size();
3639  m_Data.resize(oldCount + strLen);
3640  memcpy(m_Data.data() + oldCount, pStr, strLen);
3641  }
3642 }
3643 
3644 void VmaStringBuilder::AddNumber(uint32_t num)
3645 {
3646  char buf[11];
3647  VmaUint32ToStr(buf, sizeof(buf), num);
3648  Add(buf);
3649 }
3650 
3651 void VmaStringBuilder::AddNumber(uint64_t num)
3652 {
3653  char buf[21];
3654  VmaUint64ToStr(buf, sizeof(buf), num);
3655  Add(buf);
3656 }
3657 
3658 void VmaStringBuilder::AddPointer(const void* ptr)
3659 {
3660  char buf[21];
3661  VmaPtrToStr(buf, sizeof(buf), ptr);
3662  Add(buf);
3663 }
3664 
3665 #endif // #if VMA_STATS_STRING_ENABLED
3666 
3668 // VmaJsonWriter
3669 
3670 #if VMA_STATS_STRING_ENABLED
3671 
3672 class VmaJsonWriter
3673 {
3674 public:
3675  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
3676  ~VmaJsonWriter();
3677 
3678  void BeginObject(bool singleLine = false);
3679  void EndObject();
3680 
3681  void BeginArray(bool singleLine = false);
3682  void EndArray();
3683 
3684  void WriteString(const char* pStr);
3685  void BeginString(const char* pStr = VMA_NULL);
3686  void ContinueString(const char* pStr);
3687  void ContinueString(uint32_t n);
3688  void ContinueString(uint64_t n);
3689  void EndString(const char* pStr = VMA_NULL);
3690 
3691  void WriteNumber(uint32_t n);
3692  void WriteNumber(uint64_t n);
3693  void WriteBool(bool b);
3694  void WriteNull();
3695 
3696 private:
3697  static const char* const INDENT;
3698 
3699  enum COLLECTION_TYPE
3700  {
3701  COLLECTION_TYPE_OBJECT,
3702  COLLECTION_TYPE_ARRAY,
3703  };
3704  struct StackItem
3705  {
3706  COLLECTION_TYPE type;
3707  uint32_t valueCount;
3708  bool singleLineMode;
3709  };
3710 
3711  VmaStringBuilder& m_SB;
3712  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
3713  bool m_InsideString;
3714 
3715  void BeginValue(bool isString);
3716  void WriteIndent(bool oneLess = false);
3717 };
3718 
3719 const char* const VmaJsonWriter::INDENT = " ";
3720 
3721 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
3722  m_SB(sb),
3723  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
3724  m_InsideString(false)
3725 {
3726 }
3727 
3728 VmaJsonWriter::~VmaJsonWriter()
3729 {
3730  VMA_ASSERT(!m_InsideString);
3731  VMA_ASSERT(m_Stack.empty());
3732 }
3733 
3734 void VmaJsonWriter::BeginObject(bool singleLine)
3735 {
3736  VMA_ASSERT(!m_InsideString);
3737 
3738  BeginValue(false);
3739  m_SB.Add('{');
3740 
3741  StackItem item;
3742  item.type = COLLECTION_TYPE_OBJECT;
3743  item.valueCount = 0;
3744  item.singleLineMode = singleLine;
3745  m_Stack.push_back(item);
3746 }
3747 
3748 void VmaJsonWriter::EndObject()
3749 {
3750  VMA_ASSERT(!m_InsideString);
3751 
3752  WriteIndent(true);
3753  m_SB.Add('}');
3754 
3755  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
3756  m_Stack.pop_back();
3757 }
3758 
3759 void VmaJsonWriter::BeginArray(bool singleLine)
3760 {
3761  VMA_ASSERT(!m_InsideString);
3762 
3763  BeginValue(false);
3764  m_SB.Add('[');
3765 
3766  StackItem item;
3767  item.type = COLLECTION_TYPE_ARRAY;
3768  item.valueCount = 0;
3769  item.singleLineMode = singleLine;
3770  m_Stack.push_back(item);
3771 }
3772 
3773 void VmaJsonWriter::EndArray()
3774 {
3775  VMA_ASSERT(!m_InsideString);
3776 
3777  WriteIndent(true);
3778  m_SB.Add(']');
3779 
3780  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
3781  m_Stack.pop_back();
3782 }
3783 
3784 void VmaJsonWriter::WriteString(const char* pStr)
3785 {
3786  BeginString(pStr);
3787  EndString();
3788 }
3789 
3790 void VmaJsonWriter::BeginString(const char* pStr)
3791 {
3792  VMA_ASSERT(!m_InsideString);
3793 
3794  BeginValue(true);
3795  m_SB.Add('"');
3796  m_InsideString = true;
3797  if(pStr != VMA_NULL && pStr[0] != '\0')
3798  {
3799  ContinueString(pStr);
3800  }
3801 }
3802 
3803 void VmaJsonWriter::ContinueString(const char* pStr)
3804 {
3805  VMA_ASSERT(m_InsideString);
3806 
3807  const size_t strLen = strlen(pStr);
3808  for(size_t i = 0; i < strLen; ++i)
3809  {
3810  char ch = pStr[i];
3811  if(ch == '\'')
3812  {
3813  m_SB.Add("\\\\");
3814  }
3815  else if(ch == '"')
3816  {
3817  m_SB.Add("\\\"");
3818  }
3819  else if(ch >= 32)
3820  {
3821  m_SB.Add(ch);
3822  }
3823  else switch(ch)
3824  {
3825  case '\n':
3826  m_SB.Add("\\n");
3827  break;
3828  case '\r':
3829  m_SB.Add("\\r");
3830  break;
3831  case '\t':
3832  m_SB.Add("\\t");
3833  break;
3834  default:
3835  VMA_ASSERT(0 && "Character not currently supported.");
3836  break;
3837  }
3838  }
3839 }
3840 
3841 void VmaJsonWriter::ContinueString(uint32_t n)
3842 {
3843  VMA_ASSERT(m_InsideString);
3844  m_SB.AddNumber(n);
3845 }
3846 
3847 void VmaJsonWriter::ContinueString(uint64_t n)
3848 {
3849  VMA_ASSERT(m_InsideString);
3850  m_SB.AddNumber(n);
3851 }
3852 
3853 void VmaJsonWriter::EndString(const char* pStr)
3854 {
3855  VMA_ASSERT(m_InsideString);
3856  if(pStr != VMA_NULL && pStr[0] != '\0')
3857  {
3858  ContinueString(pStr);
3859  }
3860  m_SB.Add('"');
3861  m_InsideString = false;
3862 }
3863 
3864 void VmaJsonWriter::WriteNumber(uint32_t n)
3865 {
3866  VMA_ASSERT(!m_InsideString);
3867  BeginValue(false);
3868  m_SB.AddNumber(n);
3869 }
3870 
3871 void VmaJsonWriter::WriteNumber(uint64_t n)
3872 {
3873  VMA_ASSERT(!m_InsideString);
3874  BeginValue(false);
3875  m_SB.AddNumber(n);
3876 }
3877 
3878 void VmaJsonWriter::WriteBool(bool b)
3879 {
3880  VMA_ASSERT(!m_InsideString);
3881  BeginValue(false);
3882  m_SB.Add(b ? "true" : "false");
3883 }
3884 
3885 void VmaJsonWriter::WriteNull()
3886 {
3887  VMA_ASSERT(!m_InsideString);
3888  BeginValue(false);
3889  m_SB.Add("null");
3890 }
3891 
3892 void VmaJsonWriter::BeginValue(bool isString)
3893 {
3894  if(!m_Stack.empty())
3895  {
3896  StackItem& currItem = m_Stack.back();
3897  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3898  currItem.valueCount % 2 == 0)
3899  {
3900  VMA_ASSERT(isString);
3901  }
3902 
3903  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3904  currItem.valueCount % 2 != 0)
3905  {
3906  m_SB.Add(": ");
3907  }
3908  else if(currItem.valueCount > 0)
3909  {
3910  m_SB.Add(", ");
3911  WriteIndent();
3912  }
3913  else
3914  {
3915  WriteIndent();
3916  }
3917  ++currItem.valueCount;
3918  }
3919 }
3920 
3921 void VmaJsonWriter::WriteIndent(bool oneLess)
3922 {
3923  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
3924  {
3925  m_SB.AddNewLine();
3926 
3927  size_t count = m_Stack.size();
3928  if(count > 0 && oneLess)
3929  {
3930  --count;
3931  }
3932  for(size_t i = 0; i < count; ++i)
3933  {
3934  m_SB.Add(INDENT);
3935  }
3936  }
3937 }
3938 
3939 #endif // #if VMA_STATS_STRING_ENABLED
3940 
3942 
3943 VkDeviceSize VmaAllocation_T::GetOffset() const
3944 {
3945  switch(m_Type)
3946  {
3947  case ALLOCATION_TYPE_BLOCK:
3948  return m_BlockAllocation.m_Offset;
3949  case ALLOCATION_TYPE_OWN:
3950  return 0;
3951  default:
3952  VMA_ASSERT(0);
3953  return 0;
3954  }
3955 }
3956 
3957 VkDeviceMemory VmaAllocation_T::GetMemory() const
3958 {
3959  switch(m_Type)
3960  {
3961  case ALLOCATION_TYPE_BLOCK:
3962  return m_BlockAllocation.m_Block->m_hMemory;
3963  case ALLOCATION_TYPE_OWN:
3964  return m_OwnAllocation.m_hMemory;
3965  default:
3966  VMA_ASSERT(0);
3967  return VK_NULL_HANDLE;
3968  }
3969 }
3970 
3971 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
3972 {
3973  switch(m_Type)
3974  {
3975  case ALLOCATION_TYPE_BLOCK:
3976  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
3977  case ALLOCATION_TYPE_OWN:
3978  return m_OwnAllocation.m_MemoryTypeIndex;
3979  default:
3980  VMA_ASSERT(0);
3981  return UINT32_MAX;
3982  }
3983 }
3984 
3985 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
3986 {
3987  switch(m_Type)
3988  {
3989  case ALLOCATION_TYPE_BLOCK:
3990  return m_BlockAllocation.m_Block->m_BlockVectorType;
3991  case ALLOCATION_TYPE_OWN:
3992  return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
3993  default:
3994  VMA_ASSERT(0);
3995  return VMA_BLOCK_VECTOR_TYPE_COUNT;
3996  }
3997 }
3998 
3999 void* VmaAllocation_T::GetMappedData() const
4000 {
4001  switch(m_Type)
4002  {
4003  case ALLOCATION_TYPE_BLOCK:
4004  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
4005  {
4006  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
4007  }
4008  else
4009  {
4010  return VMA_NULL;
4011  }
4012  break;
4013  case ALLOCATION_TYPE_OWN:
4014  return m_OwnAllocation.m_pMappedData;
4015  default:
4016  VMA_ASSERT(0);
4017  return VMA_NULL;
4018  }
4019 }
4020 
4021 bool VmaAllocation_T::CanBecomeLost() const
4022 {
4023  switch(m_Type)
4024  {
4025  case ALLOCATION_TYPE_BLOCK:
4026  return m_BlockAllocation.m_CanBecomeLost;
4027  case ALLOCATION_TYPE_OWN:
4028  return false;
4029  default:
4030  VMA_ASSERT(0);
4031  return false;
4032  }
4033 }
4034 
4035 VmaPool VmaAllocation_T::GetPool() const
4036 {
4037  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4038  return m_BlockAllocation.m_hPool;
4039 }
4040 
4041 VkResult VmaAllocation_T::OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator)
4042 {
4043  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4044  if(m_OwnAllocation.m_PersistentMap)
4045  {
4046  return (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4047  hAllocator->m_hDevice,
4048  m_OwnAllocation.m_hMemory,
4049  0,
4050  VK_WHOLE_SIZE,
4051  0,
4052  &m_OwnAllocation.m_pMappedData);
4053  }
4054  return VK_SUCCESS;
4055 }
4056 void VmaAllocation_T::OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator)
4057 {
4058  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4059  if(m_OwnAllocation.m_pMappedData)
4060  {
4061  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
4062  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_OwnAllocation.m_hMemory);
4063  m_OwnAllocation.m_pMappedData = VMA_NULL;
4064  }
4065 }
4066 
4067 
4068 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4069 {
4070  VMA_ASSERT(CanBecomeLost());
4071 
4072  /*
4073  Warning: This is a carefully designed algorithm.
4074  Do not modify unless you really know what you're doing :)
4075  */
4076  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4077  for(;;)
4078  {
4079  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4080  {
4081  VMA_ASSERT(0);
4082  return false;
4083  }
4084  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4085  {
4086  return false;
4087  }
4088  else // Last use time earlier than current time.
4089  {
4090  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4091  {
4092  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4093  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4094  return true;
4095  }
4096  }
4097  }
4098 }
4099 
4100 #if VMA_STATS_STRING_ENABLED
4101 
4102 // Correspond to values of enum VmaSuballocationType.
4103 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4104  "FREE",
4105  "UNKNOWN",
4106  "BUFFER",
4107  "IMAGE_UNKNOWN",
4108  "IMAGE_LINEAR",
4109  "IMAGE_OPTIMAL",
4110 };
4111 
4112 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4113 {
4114  json.BeginObject();
4115 
4116  json.WriteString("Blocks");
4117  json.WriteNumber(stat.blockCount);
4118 
4119  json.WriteString("Allocations");
4120  json.WriteNumber(stat.allocationCount);
4121 
4122  json.WriteString("UnusedRanges");
4123  json.WriteNumber(stat.unusedRangeCount);
4124 
4125  json.WriteString("UsedBytes");
4126  json.WriteNumber(stat.usedBytes);
4127 
4128  json.WriteString("UnusedBytes");
4129  json.WriteNumber(stat.unusedBytes);
4130 
4131  if(stat.allocationCount > 1)
4132  {
4133  json.WriteString("AllocationSize");
4134  json.BeginObject(true);
4135  json.WriteString("Min");
4136  json.WriteNumber(stat.allocationSizeMin);
4137  json.WriteString("Avg");
4138  json.WriteNumber(stat.allocationSizeAvg);
4139  json.WriteString("Max");
4140  json.WriteNumber(stat.allocationSizeMax);
4141  json.EndObject();
4142  }
4143 
4144  if(stat.unusedRangeCount > 1)
4145  {
4146  json.WriteString("UnusedRangeSize");
4147  json.BeginObject(true);
4148  json.WriteString("Min");
4149  json.WriteNumber(stat.unusedRangeSizeMin);
4150  json.WriteString("Avg");
4151  json.WriteNumber(stat.unusedRangeSizeAvg);
4152  json.WriteString("Max");
4153  json.WriteNumber(stat.unusedRangeSizeMax);
4154  json.EndObject();
4155  }
4156 
4157  json.EndObject();
4158 }
4159 
4160 #endif // #if VMA_STATS_STRING_ENABLED
4161 
4162 struct VmaSuballocationItemSizeLess
4163 {
4164  bool operator()(
4165  const VmaSuballocationList::iterator lhs,
4166  const VmaSuballocationList::iterator rhs) const
4167  {
4168  return lhs->size < rhs->size;
4169  }
4170  bool operator()(
4171  const VmaSuballocationList::iterator lhs,
4172  VkDeviceSize rhsSize) const
4173  {
4174  return lhs->size < rhsSize;
4175  }
4176 };
4177 
4179 // class VmaBlockMetadata
4180 
4181 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4182  m_Size(0),
4183  m_FreeCount(0),
4184  m_SumFreeSize(0),
4185  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4186  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4187 {
4188 }
4189 
4190 VmaBlockMetadata::~VmaBlockMetadata()
4191 {
4192 }
4193 
4194 void VmaBlockMetadata::Init(VkDeviceSize size)
4195 {
4196  m_Size = size;
4197  m_FreeCount = 1;
4198  m_SumFreeSize = size;
4199 
4200  VmaSuballocation suballoc = {};
4201  suballoc.offset = 0;
4202  suballoc.size = size;
4203  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4204  suballoc.hAllocation = VK_NULL_HANDLE;
4205 
4206  m_Suballocations.push_back(suballoc);
4207  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4208  --suballocItem;
4209  m_FreeSuballocationsBySize.push_back(suballocItem);
4210 }
4211 
4212 bool VmaBlockMetadata::Validate() const
4213 {
4214  if(m_Suballocations.empty())
4215  {
4216  return false;
4217  }
4218 
4219  // Expected offset of new suballocation as calculates from previous ones.
4220  VkDeviceSize calculatedOffset = 0;
4221  // Expected number of free suballocations as calculated from traversing their list.
4222  uint32_t calculatedFreeCount = 0;
4223  // Expected sum size of free suballocations as calculated from traversing their list.
4224  VkDeviceSize calculatedSumFreeSize = 0;
4225  // Expected number of free suballocations that should be registered in
4226  // m_FreeSuballocationsBySize calculated from traversing their list.
4227  size_t freeSuballocationsToRegister = 0;
4228  // True if previous visisted suballocation was free.
4229  bool prevFree = false;
4230 
4231  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4232  suballocItem != m_Suballocations.cend();
4233  ++suballocItem)
4234  {
4235  const VmaSuballocation& subAlloc = *suballocItem;
4236 
4237  // Actual offset of this suballocation doesn't match expected one.
4238  if(subAlloc.offset != calculatedOffset)
4239  {
4240  return false;
4241  }
4242 
4243  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4244  // Two adjacent free suballocations are invalid. They should be merged.
4245  if(prevFree && currFree)
4246  {
4247  return false;
4248  }
4249  prevFree = currFree;
4250 
4251  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4252  {
4253  return false;
4254  }
4255 
4256  if(currFree)
4257  {
4258  calculatedSumFreeSize += subAlloc.size;
4259  ++calculatedFreeCount;
4260  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4261  {
4262  ++freeSuballocationsToRegister;
4263  }
4264  }
4265 
4266  calculatedOffset += subAlloc.size;
4267  }
4268 
4269  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4270  // match expected one.
4271  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4272  {
4273  return false;
4274  }
4275 
4276  VkDeviceSize lastSize = 0;
4277  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4278  {
4279  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4280 
4281  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4282  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4283  {
4284  return false;
4285  }
4286  // They must be sorted by size ascending.
4287  if(suballocItem->size < lastSize)
4288  {
4289  return false;
4290  }
4291 
4292  lastSize = suballocItem->size;
4293  }
4294 
4295  // Check if totals match calculacted values.
4296  return
4297  ValidateFreeSuballocationList() &&
4298  (calculatedOffset == m_Size) &&
4299  (calculatedSumFreeSize == m_SumFreeSize) &&
4300  (calculatedFreeCount == m_FreeCount);
4301 }
4302 
4303 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
4304 {
4305  if(!m_FreeSuballocationsBySize.empty())
4306  {
4307  return m_FreeSuballocationsBySize.back()->size;
4308  }
4309  else
4310  {
4311  return 0;
4312  }
4313 }
4314 
4315 bool VmaBlockMetadata::IsEmpty() const
4316 {
4317  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
4318 }
4319 
4320 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
4321 {
4322  outInfo.blockCount = 1;
4323 
4324  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4325  outInfo.allocationCount = rangeCount - m_FreeCount;
4326  outInfo.unusedRangeCount = m_FreeCount;
4327 
4328  outInfo.unusedBytes = m_SumFreeSize;
4329  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
4330 
4331  outInfo.allocationSizeMin = UINT64_MAX;
4332  outInfo.allocationSizeMax = 0;
4333  outInfo.unusedRangeSizeMin = UINT64_MAX;
4334  outInfo.unusedRangeSizeMax = 0;
4335 
4336  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4337  suballocItem != m_Suballocations.cend();
4338  ++suballocItem)
4339  {
4340  const VmaSuballocation& suballoc = *suballocItem;
4341  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
4342  {
4343  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
4344  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
4345  }
4346  else
4347  {
4348  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
4349  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
4350  }
4351  }
4352 }
4353 
4354 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
4355 {
4356  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4357 
4358  inoutStats.size += m_Size;
4359  inoutStats.unusedSize += m_SumFreeSize;
4360  inoutStats.allocationCount += rangeCount - m_FreeCount;
4361  inoutStats.unusedRangeCount += m_FreeCount;
4362  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
4363 }
4364 
4365 #if VMA_STATS_STRING_ENABLED
4366 
4367 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
4368 {
4369  json.BeginObject();
4370 
4371  json.WriteString("TotalBytes");
4372  json.WriteNumber(m_Size);
4373 
4374  json.WriteString("UnusedBytes");
4375  json.WriteNumber(m_SumFreeSize);
4376 
4377  json.WriteString("Allocations");
4378  json.WriteNumber(m_Suballocations.size() - m_FreeCount);
4379 
4380  json.WriteString("UnusedRanges");
4381  json.WriteNumber(m_FreeCount);
4382 
4383  json.WriteString("Suballocations");
4384  json.BeginArray();
4385  size_t i = 0;
4386  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4387  suballocItem != m_Suballocations.cend();
4388  ++suballocItem, ++i)
4389  {
4390  json.BeginObject(true);
4391 
4392  json.WriteString("Type");
4393  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
4394 
4395  json.WriteString("Size");
4396  json.WriteNumber(suballocItem->size);
4397 
4398  json.WriteString("Offset");
4399  json.WriteNumber(suballocItem->offset);
4400 
4401  json.EndObject();
4402  }
4403  json.EndArray();
4404 
4405  json.EndObject();
4406 }
4407 
4408 #endif // #if VMA_STATS_STRING_ENABLED
4409 
4410 /*
4411 How many suitable free suballocations to analyze before choosing best one.
4412 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
4413  be chosen.
4414 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
4415  suballocations will be analized and best one will be chosen.
4416 - Any other value is also acceptable.
4417 */
4418 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
4419 
4420 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
4421 {
4422  VMA_ASSERT(IsEmpty());
4423  pAllocationRequest->offset = 0;
4424  pAllocationRequest->sumFreeSize = m_SumFreeSize;
4425  pAllocationRequest->sumItemSize = 0;
4426  pAllocationRequest->item = m_Suballocations.begin();
4427  pAllocationRequest->itemsToMakeLostCount = 0;
4428 }
4429 
4430 bool VmaBlockMetadata::CreateAllocationRequest(
4431  uint32_t currentFrameIndex,
4432  uint32_t frameInUseCount,
4433  VkDeviceSize bufferImageGranularity,
4434  VkDeviceSize allocSize,
4435  VkDeviceSize allocAlignment,
4436  VmaSuballocationType allocType,
4437  bool canMakeOtherLost,
4438  VmaAllocationRequest* pAllocationRequest)
4439 {
4440  VMA_ASSERT(allocSize > 0);
4441  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4442  VMA_ASSERT(pAllocationRequest != VMA_NULL);
4443  VMA_HEAVY_ASSERT(Validate());
4444 
4445  // There is not enough total free space in this block to fullfill the request: Early return.
4446  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
4447  {
4448  return false;
4449  }
4450 
4451  // New algorithm, efficiently searching freeSuballocationsBySize.
4452  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
4453  if(freeSuballocCount > 0)
4454  {
4455  if(VMA_BEST_FIT)
4456  {
4457  // Find first free suballocation with size not less than allocSize.
4458  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
4459  m_FreeSuballocationsBySize.data(),
4460  m_FreeSuballocationsBySize.data() + freeSuballocCount,
4461  allocSize,
4462  VmaSuballocationItemSizeLess());
4463  size_t index = it - m_FreeSuballocationsBySize.data();
4464  for(; index < freeSuballocCount; ++index)
4465  {
4466  if(CheckAllocation(
4467  currentFrameIndex,
4468  frameInUseCount,
4469  bufferImageGranularity,
4470  allocSize,
4471  allocAlignment,
4472  allocType,
4473  m_FreeSuballocationsBySize[index],
4474  false, // canMakeOtherLost
4475  &pAllocationRequest->offset,
4476  &pAllocationRequest->itemsToMakeLostCount,
4477  &pAllocationRequest->sumFreeSize,
4478  &pAllocationRequest->sumItemSize))
4479  {
4480  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4481  return true;
4482  }
4483  }
4484  }
4485  else
4486  {
4487  // Search staring from biggest suballocations.
4488  for(size_t index = freeSuballocCount; index--; )
4489  {
4490  if(CheckAllocation(
4491  currentFrameIndex,
4492  frameInUseCount,
4493  bufferImageGranularity,
4494  allocSize,
4495  allocAlignment,
4496  allocType,
4497  m_FreeSuballocationsBySize[index],
4498  false, // canMakeOtherLost
4499  &pAllocationRequest->offset,
4500  &pAllocationRequest->itemsToMakeLostCount,
4501  &pAllocationRequest->sumFreeSize,
4502  &pAllocationRequest->sumItemSize))
4503  {
4504  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4505  return true;
4506  }
4507  }
4508  }
4509  }
4510 
4511  if(canMakeOtherLost)
4512  {
4513  // Brute-force algorithm. TODO: Come up with something better.
4514 
4515  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
4516  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
4517 
4518  VmaAllocationRequest tmpAllocRequest = {};
4519  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
4520  suballocIt != m_Suballocations.end();
4521  ++suballocIt)
4522  {
4523  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
4524  suballocIt->hAllocation->CanBecomeLost())
4525  {
4526  if(CheckAllocation(
4527  currentFrameIndex,
4528  frameInUseCount,
4529  bufferImageGranularity,
4530  allocSize,
4531  allocAlignment,
4532  allocType,
4533  suballocIt,
4534  canMakeOtherLost,
4535  &tmpAllocRequest.offset,
4536  &tmpAllocRequest.itemsToMakeLostCount,
4537  &tmpAllocRequest.sumFreeSize,
4538  &tmpAllocRequest.sumItemSize))
4539  {
4540  tmpAllocRequest.item = suballocIt;
4541 
4542  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
4543  {
4544  *pAllocationRequest = tmpAllocRequest;
4545  }
4546  }
4547  }
4548  }
4549 
4550  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
4551  {
4552  return true;
4553  }
4554  }
4555 
4556  return false;
4557 }
4558 
4559 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
4560  uint32_t currentFrameIndex,
4561  uint32_t frameInUseCount,
4562  VmaAllocationRequest* pAllocationRequest)
4563 {
4564  while(pAllocationRequest->itemsToMakeLostCount > 0)
4565  {
4566  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
4567  {
4568  ++pAllocationRequest->item;
4569  }
4570  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4571  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
4572  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
4573  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4574  {
4575  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
4576  --pAllocationRequest->itemsToMakeLostCount;
4577  }
4578  else
4579  {
4580  return false;
4581  }
4582  }
4583 
4584  VMA_HEAVY_ASSERT(Validate());
4585  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4586  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
4587 
4588  return true;
4589 }
4590 
4591 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4592 {
4593  uint32_t lostAllocationCount = 0;
4594  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
4595  it != m_Suballocations.end();
4596  ++it)
4597  {
4598  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
4599  it->hAllocation->CanBecomeLost() &&
4600  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4601  {
4602  it = FreeSuballocation(it);
4603  ++lostAllocationCount;
4604  }
4605  }
4606  return lostAllocationCount;
4607 }
4608 
4609 void VmaBlockMetadata::Alloc(
4610  const VmaAllocationRequest& request,
4611  VmaSuballocationType type,
4612  VkDeviceSize allocSize,
4613  VmaAllocation hAllocation)
4614 {
4615  VMA_ASSERT(request.item != m_Suballocations.end());
4616  VmaSuballocation& suballoc = *request.item;
4617  // Given suballocation is a free block.
4618  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4619  // Given offset is inside this suballocation.
4620  VMA_ASSERT(request.offset >= suballoc.offset);
4621  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
4622  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
4623  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
4624 
4625  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
4626  // it to become used.
4627  UnregisterFreeSuballocation(request.item);
4628 
4629  suballoc.offset = request.offset;
4630  suballoc.size = allocSize;
4631  suballoc.type = type;
4632  suballoc.hAllocation = hAllocation;
4633 
4634  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
4635  if(paddingEnd)
4636  {
4637  VmaSuballocation paddingSuballoc = {};
4638  paddingSuballoc.offset = request.offset + allocSize;
4639  paddingSuballoc.size = paddingEnd;
4640  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4641  VmaSuballocationList::iterator next = request.item;
4642  ++next;
4643  const VmaSuballocationList::iterator paddingEndItem =
4644  m_Suballocations.insert(next, paddingSuballoc);
4645  RegisterFreeSuballocation(paddingEndItem);
4646  }
4647 
4648  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
4649  if(paddingBegin)
4650  {
4651  VmaSuballocation paddingSuballoc = {};
4652  paddingSuballoc.offset = request.offset - paddingBegin;
4653  paddingSuballoc.size = paddingBegin;
4654  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4655  const VmaSuballocationList::iterator paddingBeginItem =
4656  m_Suballocations.insert(request.item, paddingSuballoc);
4657  RegisterFreeSuballocation(paddingBeginItem);
4658  }
4659 
4660  // Update totals.
4661  m_FreeCount = m_FreeCount - 1;
4662  if(paddingBegin > 0)
4663  {
4664  ++m_FreeCount;
4665  }
4666  if(paddingEnd > 0)
4667  {
4668  ++m_FreeCount;
4669  }
4670  m_SumFreeSize -= allocSize;
4671 }
4672 
4673 void VmaBlockMetadata::Free(const VmaAllocation allocation)
4674 {
4675  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
4676  suballocItem != m_Suballocations.end();
4677  ++suballocItem)
4678  {
4679  VmaSuballocation& suballoc = *suballocItem;
4680  if(suballoc.hAllocation == allocation)
4681  {
4682  FreeSuballocation(suballocItem);
4683  VMA_HEAVY_ASSERT(Validate());
4684  return;
4685  }
4686  }
4687  VMA_ASSERT(0 && "Not found!");
4688 }
4689 
4690 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
4691 {
4692  VkDeviceSize lastSize = 0;
4693  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
4694  {
4695  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
4696 
4697  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
4698  {
4699  VMA_ASSERT(0);
4700  return false;
4701  }
4702  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4703  {
4704  VMA_ASSERT(0);
4705  return false;
4706  }
4707  if(it->size < lastSize)
4708  {
4709  VMA_ASSERT(0);
4710  return false;
4711  }
4712 
4713  lastSize = it->size;
4714  }
4715  return true;
4716 }
4717 
4718 bool VmaBlockMetadata::CheckAllocation(
4719  uint32_t currentFrameIndex,
4720  uint32_t frameInUseCount,
4721  VkDeviceSize bufferImageGranularity,
4722  VkDeviceSize allocSize,
4723  VkDeviceSize allocAlignment,
4724  VmaSuballocationType allocType,
4725  VmaSuballocationList::const_iterator suballocItem,
4726  bool canMakeOtherLost,
4727  VkDeviceSize* pOffset,
4728  size_t* itemsToMakeLostCount,
4729  VkDeviceSize* pSumFreeSize,
4730  VkDeviceSize* pSumItemSize) const
4731 {
4732  VMA_ASSERT(allocSize > 0);
4733  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4734  VMA_ASSERT(suballocItem != m_Suballocations.cend());
4735  VMA_ASSERT(pOffset != VMA_NULL);
4736 
4737  *itemsToMakeLostCount = 0;
4738  *pSumFreeSize = 0;
4739  *pSumItemSize = 0;
4740 
4741  if(canMakeOtherLost)
4742  {
4743  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4744  {
4745  *pSumFreeSize = suballocItem->size;
4746  }
4747  else
4748  {
4749  if(suballocItem->hAllocation->CanBecomeLost() &&
4750  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4751  {
4752  ++*itemsToMakeLostCount;
4753  *pSumItemSize = suballocItem->size;
4754  }
4755  else
4756  {
4757  return false;
4758  }
4759  }
4760 
4761  // Remaining size is too small for this request: Early return.
4762  if(m_Size - suballocItem->offset < allocSize)
4763  {
4764  return false;
4765  }
4766 
4767  // Start from offset equal to beginning of this suballocation.
4768  *pOffset = suballocItem->offset;
4769 
4770  // Apply VMA_DEBUG_MARGIN at the beginning.
4771  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4772  {
4773  *pOffset += VMA_DEBUG_MARGIN;
4774  }
4775 
4776  // Apply alignment.
4777  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4778  *pOffset = VmaAlignUp(*pOffset, alignment);
4779 
4780  // Check previous suballocations for BufferImageGranularity conflicts.
4781  // Make bigger alignment if necessary.
4782  if(bufferImageGranularity > 1)
4783  {
4784  bool bufferImageGranularityConflict = false;
4785  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4786  while(prevSuballocItem != m_Suballocations.cbegin())
4787  {
4788  --prevSuballocItem;
4789  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4790  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4791  {
4792  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4793  {
4794  bufferImageGranularityConflict = true;
4795  break;
4796  }
4797  }
4798  else
4799  // Already on previous page.
4800  break;
4801  }
4802  if(bufferImageGranularityConflict)
4803  {
4804  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4805  }
4806  }
4807 
4808  // Now that we have final *pOffset, check if we are past suballocItem.
4809  // If yes, return false - this function should be called for another suballocItem as starting point.
4810  if(*pOffset >= suballocItem->offset + suballocItem->size)
4811  {
4812  return false;
4813  }
4814 
4815  // Calculate padding at the beginning based on current offset.
4816  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
4817 
4818  // Calculate required margin at the end if this is not last suballocation.
4819  VmaSuballocationList::const_iterator next = suballocItem;
4820  ++next;
4821  const VkDeviceSize requiredEndMargin =
4822  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4823 
4824  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
4825  // Another early return check.
4826  if(suballocItem->offset + totalSize > m_Size)
4827  {
4828  return false;
4829  }
4830 
4831  // Advance lastSuballocItem until desired size is reached.
4832  // Update itemsToMakeLostCount.
4833  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
4834  if(totalSize > suballocItem->size)
4835  {
4836  VkDeviceSize remainingSize = totalSize - suballocItem->size;
4837  while(remainingSize > 0)
4838  {
4839  ++lastSuballocItem;
4840  if(lastSuballocItem == m_Suballocations.cend())
4841  {
4842  return false;
4843  }
4844  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4845  {
4846  *pSumFreeSize += lastSuballocItem->size;
4847  }
4848  else
4849  {
4850  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
4851  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
4852  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4853  {
4854  ++*itemsToMakeLostCount;
4855  *pSumItemSize += lastSuballocItem->size;
4856  }
4857  else
4858  {
4859  return false;
4860  }
4861  }
4862  remainingSize = (lastSuballocItem->size < remainingSize) ?
4863  remainingSize - lastSuballocItem->size : 0;
4864  }
4865  }
4866 
4867  // Check next suballocations for BufferImageGranularity conflicts.
4868  // If conflict exists, we must mark more allocations lost or fail.
4869  if(bufferImageGranularity > 1)
4870  {
4871  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
4872  ++nextSuballocItem;
4873  while(nextSuballocItem != m_Suballocations.cend())
4874  {
4875  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4876  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4877  {
4878  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4879  {
4880  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
4881  if(nextSuballoc.hAllocation->CanBecomeLost() &&
4882  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4883  {
4884  ++*itemsToMakeLostCount;
4885  }
4886  else
4887  {
4888  return false;
4889  }
4890  }
4891  }
4892  else
4893  {
4894  // Already on next page.
4895  break;
4896  }
4897  ++nextSuballocItem;
4898  }
4899  }
4900  }
4901  else
4902  {
4903  const VmaSuballocation& suballoc = *suballocItem;
4904  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4905 
4906  *pSumFreeSize = suballoc.size;
4907 
4908  // Size of this suballocation is too small for this request: Early return.
4909  if(suballoc.size < allocSize)
4910  {
4911  return false;
4912  }
4913 
4914  // Start from offset equal to beginning of this suballocation.
4915  *pOffset = suballoc.offset;
4916 
4917  // Apply VMA_DEBUG_MARGIN at the beginning.
4918  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4919  {
4920  *pOffset += VMA_DEBUG_MARGIN;
4921  }
4922 
4923  // Apply alignment.
4924  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4925  *pOffset = VmaAlignUp(*pOffset, alignment);
4926 
4927  // Check previous suballocations for BufferImageGranularity conflicts.
4928  // Make bigger alignment if necessary.
4929  if(bufferImageGranularity > 1)
4930  {
4931  bool bufferImageGranularityConflict = false;
4932  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4933  while(prevSuballocItem != m_Suballocations.cbegin())
4934  {
4935  --prevSuballocItem;
4936  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4937  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4938  {
4939  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4940  {
4941  bufferImageGranularityConflict = true;
4942  break;
4943  }
4944  }
4945  else
4946  // Already on previous page.
4947  break;
4948  }
4949  if(bufferImageGranularityConflict)
4950  {
4951  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4952  }
4953  }
4954 
4955  // Calculate padding at the beginning based on current offset.
4956  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
4957 
4958  // Calculate required margin at the end if this is not last suballocation.
4959  VmaSuballocationList::const_iterator next = suballocItem;
4960  ++next;
4961  const VkDeviceSize requiredEndMargin =
4962  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4963 
4964  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
4965  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
4966  {
4967  return false;
4968  }
4969 
4970  // Check next suballocations for BufferImageGranularity conflicts.
4971  // If conflict exists, allocation cannot be made here.
4972  if(bufferImageGranularity > 1)
4973  {
4974  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
4975  ++nextSuballocItem;
4976  while(nextSuballocItem != m_Suballocations.cend())
4977  {
4978  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4979  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4980  {
4981  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4982  {
4983  return false;
4984  }
4985  }
4986  else
4987  {
4988  // Already on next page.
4989  break;
4990  }
4991  ++nextSuballocItem;
4992  }
4993  }
4994  }
4995 
4996  // All tests passed: Success. pOffset is already filled.
4997  return true;
4998 }
4999 
5000 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5001 {
5002  VMA_ASSERT(item != m_Suballocations.end());
5003  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5004 
5005  VmaSuballocationList::iterator nextItem = item;
5006  ++nextItem;
5007  VMA_ASSERT(nextItem != m_Suballocations.end());
5008  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5009 
5010  item->size += nextItem->size;
5011  --m_FreeCount;
5012  m_Suballocations.erase(nextItem);
5013 }
5014 
5015 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5016 {
5017  // Change this suballocation to be marked as free.
5018  VmaSuballocation& suballoc = *suballocItem;
5019  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5020  suballoc.hAllocation = VK_NULL_HANDLE;
5021 
5022  // Update totals.
5023  ++m_FreeCount;
5024  m_SumFreeSize += suballoc.size;
5025 
5026  // Merge with previous and/or next suballocation if it's also free.
5027  bool mergeWithNext = false;
5028  bool mergeWithPrev = false;
5029 
5030  VmaSuballocationList::iterator nextItem = suballocItem;
5031  ++nextItem;
5032  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5033  {
5034  mergeWithNext = true;
5035  }
5036 
5037  VmaSuballocationList::iterator prevItem = suballocItem;
5038  if(suballocItem != m_Suballocations.begin())
5039  {
5040  --prevItem;
5041  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5042  {
5043  mergeWithPrev = true;
5044  }
5045  }
5046 
5047  if(mergeWithNext)
5048  {
5049  UnregisterFreeSuballocation(nextItem);
5050  MergeFreeWithNext(suballocItem);
5051  }
5052 
5053  if(mergeWithPrev)
5054  {
5055  UnregisterFreeSuballocation(prevItem);
5056  MergeFreeWithNext(prevItem);
5057  RegisterFreeSuballocation(prevItem);
5058  return prevItem;
5059  }
5060  else
5061  {
5062  RegisterFreeSuballocation(suballocItem);
5063  return suballocItem;
5064  }
5065 }
5066 
5067 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5068 {
5069  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5070  VMA_ASSERT(item->size > 0);
5071 
5072  // You may want to enable this validation at the beginning or at the end of
5073  // this function, depending on what do you want to check.
5074  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5075 
5076  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5077  {
5078  if(m_FreeSuballocationsBySize.empty())
5079  {
5080  m_FreeSuballocationsBySize.push_back(item);
5081  }
5082  else
5083  {
5084  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5085  }
5086  }
5087 
5088  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5089 }
5090 
5091 
5092 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5093 {
5094  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5095  VMA_ASSERT(item->size > 0);
5096 
5097  // You may want to enable this validation at the beginning or at the end of
5098  // this function, depending on what do you want to check.
5099  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5100 
5101  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5102  {
5103  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5104  m_FreeSuballocationsBySize.data(),
5105  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5106  item,
5107  VmaSuballocationItemSizeLess());
5108  for(size_t index = it - m_FreeSuballocationsBySize.data();
5109  index < m_FreeSuballocationsBySize.size();
5110  ++index)
5111  {
5112  if(m_FreeSuballocationsBySize[index] == item)
5113  {
5114  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5115  return;
5116  }
5117  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5118  }
5119  VMA_ASSERT(0 && "Not found.");
5120  }
5121 
5122  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5123 }
5124 
5126 // class VmaDeviceMemoryBlock
5127 
5128 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5129  m_MemoryTypeIndex(UINT32_MAX),
5130  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
5131  m_hMemory(VK_NULL_HANDLE),
5132  m_PersistentMap(false),
5133  m_pMappedData(VMA_NULL),
5134  m_Metadata(hAllocator)
5135 {
5136 }
5137 
5138 void VmaDeviceMemoryBlock::Init(
5139  uint32_t newMemoryTypeIndex,
5140  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
5141  VkDeviceMemory newMemory,
5142  VkDeviceSize newSize,
5143  bool persistentMap,
5144  void* pMappedData)
5145 {
5146  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5147 
5148  m_MemoryTypeIndex = newMemoryTypeIndex;
5149  m_BlockVectorType = newBlockVectorType;
5150  m_hMemory = newMemory;
5151  m_PersistentMap = persistentMap;
5152  m_pMappedData = pMappedData;
5153 
5154  m_Metadata.Init(newSize);
5155 }
5156 
5157 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5158 {
5159  // This is the most important assert in the entire library.
5160  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5161  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5162 
5163  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5164  if(m_pMappedData != VMA_NULL)
5165  {
5166  (allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, m_hMemory);
5167  m_pMappedData = VMA_NULL;
5168  }
5169 
5170  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5171  m_hMemory = VK_NULL_HANDLE;
5172 }
5173 
5174 bool VmaDeviceMemoryBlock::Validate() const
5175 {
5176  if((m_hMemory == VK_NULL_HANDLE) ||
5177  (m_Metadata.GetSize() == 0))
5178  {
5179  return false;
5180  }
5181 
5182  return m_Metadata.Validate();
5183 }
5184 
5185 static void InitStatInfo(VmaStatInfo& outInfo)
5186 {
5187  memset(&outInfo, 0, sizeof(outInfo));
5188  outInfo.allocationSizeMin = UINT64_MAX;
5189  outInfo.unusedRangeSizeMin = UINT64_MAX;
5190 }
5191 
5192 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5193 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5194 {
5195  inoutInfo.blockCount += srcInfo.blockCount;
5196  inoutInfo.allocationCount += srcInfo.allocationCount;
5197  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
5198  inoutInfo.usedBytes += srcInfo.usedBytes;
5199  inoutInfo.unusedBytes += srcInfo.unusedBytes;
5200  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
5201  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
5202  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
5203  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
5204 }
5205 
5206 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
5207 {
5208  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
5209  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
5210  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
5211  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
5212 }
5213 
5214 VmaPool_T::VmaPool_T(
5215  VmaAllocator hAllocator,
5216  const VmaPoolCreateInfo& createInfo) :
5217  m_BlockVector(
5218  hAllocator,
5219  createInfo.memoryTypeIndex,
5220  (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ?
5221  VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
5222  createInfo.blockSize,
5223  createInfo.minBlockCount,
5224  createInfo.maxBlockCount,
5225  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
5226  createInfo.frameInUseCount,
5227  true) // isCustomPool
5228 {
5229 }
5230 
5231 VmaPool_T::~VmaPool_T()
5232 {
5233 }
5234 
5235 #if VMA_STATS_STRING_ENABLED
5236 
5237 #endif // #if VMA_STATS_STRING_ENABLED
5238 
5239 VmaBlockVector::VmaBlockVector(
5240  VmaAllocator hAllocator,
5241  uint32_t memoryTypeIndex,
5242  VMA_BLOCK_VECTOR_TYPE blockVectorType,
5243  VkDeviceSize preferredBlockSize,
5244  size_t minBlockCount,
5245  size_t maxBlockCount,
5246  VkDeviceSize bufferImageGranularity,
5247  uint32_t frameInUseCount,
5248  bool isCustomPool) :
5249  m_hAllocator(hAllocator),
5250  m_MemoryTypeIndex(memoryTypeIndex),
5251  m_BlockVectorType(blockVectorType),
5252  m_PreferredBlockSize(preferredBlockSize),
5253  m_MinBlockCount(minBlockCount),
5254  m_MaxBlockCount(maxBlockCount),
5255  m_BufferImageGranularity(bufferImageGranularity),
5256  m_FrameInUseCount(frameInUseCount),
5257  m_IsCustomPool(isCustomPool),
5258  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
5259  m_HasEmptyBlock(false),
5260  m_pDefragmentator(VMA_NULL)
5261 {
5262 }
5263 
5264 VmaBlockVector::~VmaBlockVector()
5265 {
5266  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
5267 
5268  for(size_t i = m_Blocks.size(); i--; )
5269  {
5270  m_Blocks[i]->Destroy(m_hAllocator);
5271  vma_delete(m_hAllocator, m_Blocks[i]);
5272  }
5273 }
5274 
5275 VkResult VmaBlockVector::CreateMinBlocks()
5276 {
5277  for(size_t i = 0; i < m_MinBlockCount; ++i)
5278  {
5279  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
5280  if(res != VK_SUCCESS)
5281  {
5282  return res;
5283  }
5284  }
5285  return VK_SUCCESS;
5286 }
5287 
5288 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
5289 {
5290  pStats->size = 0;
5291  pStats->unusedSize = 0;
5292  pStats->allocationCount = 0;
5293  pStats->unusedRangeCount = 0;
5294  pStats->unusedRangeSizeMax = 0;
5295 
5296  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5297 
5298  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5299  {
5300  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5301  VMA_ASSERT(pBlock);
5302  VMA_HEAVY_ASSERT(pBlock->Validate());
5303  pBlock->m_Metadata.AddPoolStats(*pStats);
5304  }
5305 }
5306 
5307 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
5308 
5309 VkResult VmaBlockVector::Allocate(
5310  VmaPool hCurrentPool,
5311  uint32_t currentFrameIndex,
5312  const VkMemoryRequirements& vkMemReq,
5313  const VmaAllocationCreateInfo& createInfo,
5314  VmaSuballocationType suballocType,
5315  VmaAllocation* pAllocation)
5316 {
5317  // Validate flags.
5318  if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) !=
5319  (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED))
5320  {
5321  VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT.");
5322  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5323  }
5324 
5325  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5326 
5327  // 1. Search existing allocations. Try to allocate without making other allocations lost.
5328  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5329  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5330  {
5331  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5332  VMA_ASSERT(pCurrBlock);
5333  VmaAllocationRequest currRequest = {};
5334  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5335  currentFrameIndex,
5336  m_FrameInUseCount,
5337  m_BufferImageGranularity,
5338  vkMemReq.size,
5339  vkMemReq.alignment,
5340  suballocType,
5341  false, // canMakeOtherLost
5342  &currRequest))
5343  {
5344  // Allocate from pCurrBlock.
5345  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
5346 
5347  // We no longer have an empty Allocation.
5348  if(pCurrBlock->m_Metadata.IsEmpty())
5349  {
5350  m_HasEmptyBlock = false;
5351  }
5352 
5353  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5354  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
5355  (*pAllocation)->InitBlockAllocation(
5356  hCurrentPool,
5357  pCurrBlock,
5358  currRequest.offset,
5359  vkMemReq.alignment,
5360  vkMemReq.size,
5361  suballocType,
5362  createInfo.pUserData,
5363  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5364  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
5365  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5366  return VK_SUCCESS;
5367  }
5368  }
5369 
5370  const bool canCreateNewBlock =
5371  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
5372  (m_Blocks.size() < m_MaxBlockCount);
5373 
5374  // 2. Try to create new block.
5375  if(canCreateNewBlock)
5376  {
5377  // 2.1. Start with full preferredBlockSize.
5378  VkDeviceSize blockSize = m_PreferredBlockSize;
5379  size_t newBlockIndex = 0;
5380  VkResult res = CreateBlock(blockSize, &newBlockIndex);
5381  // Allocating blocks of other sizes is allowed only in default pools.
5382  // In custom pools block size is fixed.
5383  if(res < 0 && m_IsCustomPool == false)
5384  {
5385  // 2.2. Try half the size.
5386  blockSize /= 2;
5387  if(blockSize >= vkMemReq.size)
5388  {
5389  res = CreateBlock(blockSize, &newBlockIndex);
5390  if(res < 0)
5391  {
5392  // 2.3. Try quarter the size.
5393  blockSize /= 2;
5394  if(blockSize >= vkMemReq.size)
5395  {
5396  res = CreateBlock(blockSize, &newBlockIndex);
5397  }
5398  }
5399  }
5400  }
5401  if(res == VK_SUCCESS)
5402  {
5403  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
5404  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
5405 
5406  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
5407  VmaAllocationRequest allocRequest;
5408  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
5409  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5410  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
5411  (*pAllocation)->InitBlockAllocation(
5412  hCurrentPool,
5413  pBlock,
5414  allocRequest.offset,
5415  vkMemReq.alignment,
5416  vkMemReq.size,
5417  suballocType,
5418  createInfo.pUserData,
5419  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5420  VMA_HEAVY_ASSERT(pBlock->Validate());
5421  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
5422 
5423  return VK_SUCCESS;
5424  }
5425  }
5426 
5427  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
5428 
5429  // 3. Try to allocate from existing blocks with making other allocations lost.
5430  if(canMakeOtherLost)
5431  {
5432  uint32_t tryIndex = 0;
5433  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
5434  {
5435  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
5436  VmaAllocationRequest bestRequest = {};
5437  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
5438 
5439  // 1. Search existing allocations.
5440  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5441  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5442  {
5443  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5444  VMA_ASSERT(pCurrBlock);
5445  VmaAllocationRequest currRequest = {};
5446  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5447  currentFrameIndex,
5448  m_FrameInUseCount,
5449  m_BufferImageGranularity,
5450  vkMemReq.size,
5451  vkMemReq.alignment,
5452  suballocType,
5453  canMakeOtherLost,
5454  &currRequest))
5455  {
5456  const VkDeviceSize currRequestCost = currRequest.CalcCost();
5457  if(pBestRequestBlock == VMA_NULL ||
5458  currRequestCost < bestRequestCost)
5459  {
5460  pBestRequestBlock = pCurrBlock;
5461  bestRequest = currRequest;
5462  bestRequestCost = currRequestCost;
5463 
5464  if(bestRequestCost == 0)
5465  {
5466  break;
5467  }
5468  }
5469  }
5470  }
5471 
5472  if(pBestRequestBlock != VMA_NULL)
5473  {
5474  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
5475  currentFrameIndex,
5476  m_FrameInUseCount,
5477  &bestRequest))
5478  {
5479  // We no longer have an empty Allocation.
5480  if(pBestRequestBlock->m_Metadata.IsEmpty())
5481  {
5482  m_HasEmptyBlock = false;
5483  }
5484  // Allocate from this pBlock.
5485  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5486  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
5487  (*pAllocation)->InitBlockAllocation(
5488  hCurrentPool,
5489  pBestRequestBlock,
5490  bestRequest.offset,
5491  vkMemReq.alignment,
5492  vkMemReq.size,
5493  suballocType,
5494  createInfo.pUserData,
5495  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5496  VMA_HEAVY_ASSERT(pBlock->Validate());
5497  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5498  return VK_SUCCESS;
5499  }
5500  // else: Some allocations must have been touched while we are here. Next try.
5501  }
5502  else
5503  {
5504  // Could not find place in any of the blocks - break outer loop.
5505  break;
5506  }
5507  }
5508  /* Maximum number of tries exceeded - a very unlike event when many other
5509  threads are simultaneously touching allocations making it impossible to make
5510  lost at the same time as we try to allocate. */
5511  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
5512  {
5513  return VK_ERROR_TOO_MANY_OBJECTS;
5514  }
5515  }
5516 
5517  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5518 }
5519 
5520 void VmaBlockVector::Free(
5521  VmaAllocation hAllocation)
5522 {
5523  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
5524 
5525  // Scope for lock.
5526  {
5527  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5528 
5529  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
5530 
5531  pBlock->m_Metadata.Free(hAllocation);
5532  VMA_HEAVY_ASSERT(pBlock->Validate());
5533 
5534  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
5535 
5536  // pBlock became empty after this deallocation.
5537  if(pBlock->m_Metadata.IsEmpty())
5538  {
5539  // Already has empty Allocation. We don't want to have two, so delete this one.
5540  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
5541  {
5542  pBlockToDelete = pBlock;
5543  Remove(pBlock);
5544  }
5545  // We now have first empty Allocation.
5546  else
5547  {
5548  m_HasEmptyBlock = true;
5549  }
5550  }
5551  // Must be called after srcBlockIndex is used, because later it may become invalid!
5552  IncrementallySortBlocks();
5553  }
5554 
5555  // Destruction of a free Allocation. Deferred until this point, outside of mutex
5556  // lock, for performance reason.
5557  if(pBlockToDelete != VMA_NULL)
5558  {
5559  VMA_DEBUG_LOG(" Deleted empty allocation");
5560  pBlockToDelete->Destroy(m_hAllocator);
5561  vma_delete(m_hAllocator, pBlockToDelete);
5562  }
5563 }
5564 
5565 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
5566 {
5567  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5568  {
5569  if(m_Blocks[blockIndex] == pBlock)
5570  {
5571  VmaVectorRemove(m_Blocks, blockIndex);
5572  return;
5573  }
5574  }
5575  VMA_ASSERT(0);
5576 }
5577 
5578 void VmaBlockVector::IncrementallySortBlocks()
5579 {
5580  // Bubble sort only until first swap.
5581  for(size_t i = 1; i < m_Blocks.size(); ++i)
5582  {
5583  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
5584  {
5585  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
5586  return;
5587  }
5588  }
5589 }
5590 
5591 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
5592 {
5593  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
5594  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
5595  allocInfo.allocationSize = blockSize;
5596  VkDeviceMemory mem = VK_NULL_HANDLE;
5597  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
5598  if(res < 0)
5599  {
5600  return res;
5601  }
5602 
5603  // New VkDeviceMemory successfully created.
5604 
5605  // Map memory if needed.
5606  void* pMappedData = VMA_NULL;
5607  const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED);
5608  if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0)
5609  {
5610  res = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5611  m_hAllocator->m_hDevice,
5612  mem,
5613  0,
5614  VK_WHOLE_SIZE,
5615  0,
5616  &pMappedData);
5617  if(res < 0)
5618  {
5619  VMA_DEBUG_LOG(" vkMapMemory FAILED");
5620  m_hAllocator->FreeVulkanMemory(m_MemoryTypeIndex, blockSize, mem);
5621  return res;
5622  }
5623  }
5624 
5625  // Create new Allocation for it.
5626  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
5627  pBlock->Init(
5628  m_MemoryTypeIndex,
5629  (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType,
5630  mem,
5631  allocInfo.allocationSize,
5632  persistentMap,
5633  pMappedData);
5634 
5635  m_Blocks.push_back(pBlock);
5636  if(pNewBlockIndex != VMA_NULL)
5637  {
5638  *pNewBlockIndex = m_Blocks.size() - 1;
5639  }
5640 
5641  return VK_SUCCESS;
5642 }
5643 
5644 #if VMA_STATS_STRING_ENABLED
5645 
5646 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
5647 {
5648  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5649 
5650  json.BeginObject();
5651 
5652  if(m_IsCustomPool)
5653  {
5654  json.WriteString("MemoryTypeIndex");
5655  json.WriteNumber(m_MemoryTypeIndex);
5656 
5657  if(m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
5658  {
5659  json.WriteString("Mapped");
5660  json.WriteBool(true);
5661  }
5662 
5663  json.WriteString("BlockSize");
5664  json.WriteNumber(m_PreferredBlockSize);
5665 
5666  json.WriteString("BlockCount");
5667  json.BeginObject(true);
5668  if(m_MinBlockCount > 0)
5669  {
5670  json.WriteString("Min");
5671  json.WriteNumber(m_MinBlockCount);
5672  }
5673  if(m_MaxBlockCount < SIZE_MAX)
5674  {
5675  json.WriteString("Max");
5676  json.WriteNumber(m_MaxBlockCount);
5677  }
5678  json.WriteString("Cur");
5679  json.WriteNumber(m_Blocks.size());
5680  json.EndObject();
5681 
5682  if(m_FrameInUseCount > 0)
5683  {
5684  json.WriteString("FrameInUseCount");
5685  json.WriteNumber(m_FrameInUseCount);
5686  }
5687  }
5688  else
5689  {
5690  json.WriteString("PreferredBlockSize");
5691  json.WriteNumber(m_PreferredBlockSize);
5692  }
5693 
5694  json.WriteString("Blocks");
5695  json.BeginArray();
5696  for(size_t i = 0; i < m_Blocks.size(); ++i)
5697  {
5698  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
5699  }
5700  json.EndArray();
5701 
5702  json.EndObject();
5703 }
5704 
5705 #endif // #if VMA_STATS_STRING_ENABLED
5706 
5707 void VmaBlockVector::UnmapPersistentlyMappedMemory()
5708 {
5709  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5710 
5711  for(size_t i = m_Blocks.size(); i--; )
5712  {
5713  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5714  if(pBlock->m_pMappedData != VMA_NULL)
5715  {
5716  VMA_ASSERT(pBlock->m_PersistentMap != false);
5717  (m_hAllocator->GetVulkanFunctions().vkUnmapMemory)(m_hAllocator->m_hDevice, pBlock->m_hMemory);
5718  pBlock->m_pMappedData = VMA_NULL;
5719  }
5720  }
5721 }
5722 
5723 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
5724 {
5725  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5726 
5727  VkResult finalResult = VK_SUCCESS;
5728  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
5729  {
5730  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5731  if(pBlock->m_PersistentMap)
5732  {
5733  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
5734  VkResult localResult = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5735  m_hAllocator->m_hDevice,
5736  pBlock->m_hMemory,
5737  0,
5738  VK_WHOLE_SIZE,
5739  0,
5740  &pBlock->m_pMappedData);
5741  if(localResult != VK_SUCCESS)
5742  {
5743  finalResult = localResult;
5744  }
5745  }
5746  }
5747  return finalResult;
5748 }
5749 
5750 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
5751  VmaAllocator hAllocator,
5752  uint32_t currentFrameIndex)
5753 {
5754  if(m_pDefragmentator == VMA_NULL)
5755  {
5756  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
5757  hAllocator,
5758  this,
5759  currentFrameIndex);
5760  }
5761 
5762  return m_pDefragmentator;
5763 }
5764 
5765 VkResult VmaBlockVector::Defragment(
5766  VmaDefragmentationStats* pDefragmentationStats,
5767  VkDeviceSize& maxBytesToMove,
5768  uint32_t& maxAllocationsToMove)
5769 {
5770  if(m_pDefragmentator == VMA_NULL)
5771  {
5772  return VK_SUCCESS;
5773  }
5774 
5775  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5776 
5777  // Defragment.
5778  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
5779 
5780  // Accumulate statistics.
5781  if(pDefragmentationStats != VMA_NULL)
5782  {
5783  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
5784  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
5785  pDefragmentationStats->bytesMoved += bytesMoved;
5786  pDefragmentationStats->allocationsMoved += allocationsMoved;
5787  VMA_ASSERT(bytesMoved <= maxBytesToMove);
5788  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
5789  maxBytesToMove -= bytesMoved;
5790  maxAllocationsToMove -= allocationsMoved;
5791  }
5792 
5793  // Free empty blocks.
5794  m_HasEmptyBlock = false;
5795  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
5796  {
5797  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
5798  if(pBlock->m_Metadata.IsEmpty())
5799  {
5800  if(m_Blocks.size() > m_MinBlockCount)
5801  {
5802  if(pDefragmentationStats != VMA_NULL)
5803  {
5804  ++pDefragmentationStats->deviceMemoryBlocksFreed;
5805  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
5806  }
5807 
5808  VmaVectorRemove(m_Blocks, blockIndex);
5809  pBlock->Destroy(m_hAllocator);
5810  vma_delete(m_hAllocator, pBlock);
5811  }
5812  else
5813  {
5814  m_HasEmptyBlock = true;
5815  }
5816  }
5817  }
5818 
5819  return result;
5820 }
5821 
5822 void VmaBlockVector::DestroyDefragmentator()
5823 {
5824  if(m_pDefragmentator != VMA_NULL)
5825  {
5826  vma_delete(m_hAllocator, m_pDefragmentator);
5827  m_pDefragmentator = VMA_NULL;
5828  }
5829 }
5830 
5831 void VmaBlockVector::MakePoolAllocationsLost(
5832  uint32_t currentFrameIndex,
5833  size_t* pLostAllocationCount)
5834 {
5835  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5836 
5837  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5838  {
5839  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5840  VMA_ASSERT(pBlock);
5841  pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
5842  }
5843 }
5844 
5845 void VmaBlockVector::AddStats(VmaStats* pStats)
5846 {
5847  const uint32_t memTypeIndex = m_MemoryTypeIndex;
5848  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
5849 
5850  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5851 
5852  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5853  {
5854  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5855  VMA_ASSERT(pBlock);
5856  VMA_HEAVY_ASSERT(pBlock->Validate());
5857  VmaStatInfo allocationStatInfo;
5858  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
5859  VmaAddStatInfo(pStats->total, allocationStatInfo);
5860  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
5861  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
5862  }
5863 }
5864 
5866 // VmaDefragmentator members definition
5867 
5868 VmaDefragmentator::VmaDefragmentator(
5869  VmaAllocator hAllocator,
5870  VmaBlockVector* pBlockVector,
5871  uint32_t currentFrameIndex) :
5872  m_hAllocator(hAllocator),
5873  m_pBlockVector(pBlockVector),
5874  m_CurrentFrameIndex(currentFrameIndex),
5875  m_BytesMoved(0),
5876  m_AllocationsMoved(0),
5877  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
5878  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
5879 {
5880 }
5881 
5882 VmaDefragmentator::~VmaDefragmentator()
5883 {
5884  for(size_t i = m_Blocks.size(); i--; )
5885  {
5886  vma_delete(m_hAllocator, m_Blocks[i]);
5887  }
5888 }
5889 
5890 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
5891 {
5892  AllocationInfo allocInfo;
5893  allocInfo.m_hAllocation = hAlloc;
5894  allocInfo.m_pChanged = pChanged;
5895  m_Allocations.push_back(allocInfo);
5896 }
5897 
5898 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
5899 {
5900  // It has already been mapped for defragmentation.
5901  if(m_pMappedDataForDefragmentation)
5902  {
5903  *ppMappedData = m_pMappedDataForDefragmentation;
5904  return VK_SUCCESS;
5905  }
5906 
5907  // It is persistently mapped.
5908  if(m_pBlock->m_PersistentMap)
5909  {
5910  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
5911  *ppMappedData = m_pBlock->m_pMappedData;
5912  return VK_SUCCESS;
5913  }
5914 
5915  // Map on first usage.
5916  VkResult res = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5917  hAllocator->m_hDevice,
5918  m_pBlock->m_hMemory,
5919  0,
5920  VK_WHOLE_SIZE,
5921  0,
5922  &m_pMappedDataForDefragmentation);
5923  *ppMappedData = m_pMappedDataForDefragmentation;
5924  return res;
5925 }
5926 
5927 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
5928 {
5929  if(m_pMappedDataForDefragmentation != VMA_NULL)
5930  {
5931  (hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_pBlock->m_hMemory);
5932  }
5933 }
5934 
5935 VkResult VmaDefragmentator::DefragmentRound(
5936  VkDeviceSize maxBytesToMove,
5937  uint32_t maxAllocationsToMove)
5938 {
5939  if(m_Blocks.empty())
5940  {
5941  return VK_SUCCESS;
5942  }
5943 
5944  size_t srcBlockIndex = m_Blocks.size() - 1;
5945  size_t srcAllocIndex = SIZE_MAX;
5946  for(;;)
5947  {
5948  // 1. Find next allocation to move.
5949  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
5950  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
5951  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
5952  {
5953  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
5954  {
5955  // Finished: no more allocations to process.
5956  if(srcBlockIndex == 0)
5957  {
5958  return VK_SUCCESS;
5959  }
5960  else
5961  {
5962  --srcBlockIndex;
5963  srcAllocIndex = SIZE_MAX;
5964  }
5965  }
5966  else
5967  {
5968  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
5969  }
5970  }
5971 
5972  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
5973  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
5974 
5975  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
5976  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
5977  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
5978  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
5979 
5980  // 2. Try to find new place for this allocation in preceding or current block.
5981  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
5982  {
5983  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
5984  VmaAllocationRequest dstAllocRequest;
5985  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
5986  m_CurrentFrameIndex,
5987  m_pBlockVector->GetFrameInUseCount(),
5988  m_pBlockVector->GetBufferImageGranularity(),
5989  size,
5990  alignment,
5991  suballocType,
5992  false, // canMakeOtherLost
5993  &dstAllocRequest) &&
5994  MoveMakesSense(
5995  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
5996  {
5997  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
5998 
5999  // Reached limit on number of allocations or bytes to move.
6000  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6001  (m_BytesMoved + size > maxBytesToMove))
6002  {
6003  return VK_INCOMPLETE;
6004  }
6005 
6006  void* pDstMappedData = VMA_NULL;
6007  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6008  if(res != VK_SUCCESS)
6009  {
6010  return res;
6011  }
6012 
6013  void* pSrcMappedData = VMA_NULL;
6014  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6015  if(res != VK_SUCCESS)
6016  {
6017  return res;
6018  }
6019 
6020  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6021  memcpy(
6022  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6023  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6024  static_cast<size_t>(size));
6025 
6026  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6027  pSrcBlockInfo->m_pBlock->m_Metadata.Free(allocInfo.m_hAllocation);
6028 
6029  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6030 
6031  if(allocInfo.m_pChanged != VMA_NULL)
6032  {
6033  *allocInfo.m_pChanged = VK_TRUE;
6034  }
6035 
6036  ++m_AllocationsMoved;
6037  m_BytesMoved += size;
6038 
6039  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6040 
6041  break;
6042  }
6043  }
6044 
6045  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6046 
6047  if(srcAllocIndex > 0)
6048  {
6049  --srcAllocIndex;
6050  }
6051  else
6052  {
6053  if(srcBlockIndex > 0)
6054  {
6055  --srcBlockIndex;
6056  srcAllocIndex = SIZE_MAX;
6057  }
6058  else
6059  {
6060  return VK_SUCCESS;
6061  }
6062  }
6063  }
6064 }
6065 
6066 VkResult VmaDefragmentator::Defragment(
6067  VkDeviceSize maxBytesToMove,
6068  uint32_t maxAllocationsToMove)
6069 {
6070  if(m_Allocations.empty())
6071  {
6072  return VK_SUCCESS;
6073  }
6074 
6075  // Create block info for each block.
6076  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6077  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6078  {
6079  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6080  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6081  m_Blocks.push_back(pBlockInfo);
6082  }
6083 
6084  // Sort them by m_pBlock pointer value.
6085  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6086 
6087  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6088  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6089  {
6090  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6091  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6092  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6093  {
6094  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6095  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6096  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6097  {
6098  (*it)->m_Allocations.push_back(allocInfo);
6099  }
6100  else
6101  {
6102  VMA_ASSERT(0);
6103  }
6104  }
6105  }
6106  m_Allocations.clear();
6107 
6108  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6109  {
6110  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6111  pBlockInfo->CalcHasNonMovableAllocations();
6112  pBlockInfo->SortAllocationsBySizeDescecnding();
6113  }
6114 
6115  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6116  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6117 
6118  // Execute defragmentation rounds (the main part).
6119  VkResult result = VK_SUCCESS;
6120  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6121  {
6122  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6123  }
6124 
6125  // Unmap blocks that were mapped for defragmentation.
6126  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6127  {
6128  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6129  }
6130 
6131  return result;
6132 }
6133 
6134 bool VmaDefragmentator::MoveMakesSense(
6135  size_t dstBlockIndex, VkDeviceSize dstOffset,
6136  size_t srcBlockIndex, VkDeviceSize srcOffset)
6137 {
6138  if(dstBlockIndex < srcBlockIndex)
6139  {
6140  return true;
6141  }
6142  if(dstBlockIndex > srcBlockIndex)
6143  {
6144  return false;
6145  }
6146  if(dstOffset < srcOffset)
6147  {
6148  return true;
6149  }
6150  return false;
6151 }
6152 
6154 // VmaAllocator_T
6155 
6156 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6157  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6158  m_PhysicalDevice(pCreateInfo->physicalDevice),
6159  m_hDevice(pCreateInfo->device),
6160  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6161  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6162  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6163  m_UnmapPersistentlyMappedMemoryCounter(0),
6164  m_PreferredLargeHeapBlockSize(0),
6165  m_PreferredSmallHeapBlockSize(0),
6166  m_CurrentFrameIndex(0),
6167  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6168 {
6169  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6170 
6171  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6172  memset(&m_MemProps, 0, sizeof(m_MemProps));
6173  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6174 
6175  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6176  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
6177 
6178  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6179  {
6180  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6181  }
6182 
6183  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6184  {
6185  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6186  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6187  }
6188 
6189  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6190 
6191  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6192  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6193 
6194  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6195  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6196  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
6197  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
6198 
6199  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
6200  {
6201  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
6202  {
6203  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
6204  if(limit != VK_WHOLE_SIZE)
6205  {
6206  m_HeapSizeLimit[heapIndex] = limit;
6207  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
6208  {
6209  m_MemProps.memoryHeaps[heapIndex].size = limit;
6210  }
6211  }
6212  }
6213  }
6214 
6215  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6216  {
6217  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
6218 
6219  for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex)
6220  {
6221  m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)(
6222  this,
6223  memTypeIndex,
6224  static_cast<VMA_BLOCK_VECTOR_TYPE>(blockVectorTypeIndex),
6225  preferredBlockSize,
6226  0,
6227  SIZE_MAX,
6228  GetBufferImageGranularity(),
6229  pCreateInfo->frameInUseCount,
6230  false); // isCustomPool
6231  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
6232  // becase minBlockCount is 0.
6233  m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
6234  }
6235  }
6236 }
6237 
6238 VmaAllocator_T::~VmaAllocator_T()
6239 {
6240  VMA_ASSERT(m_Pools.empty());
6241 
6242  for(size_t i = GetMemoryTypeCount(); i--; )
6243  {
6244  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
6245  {
6246  vma_delete(this, m_pOwnAllocations[i][j]);
6247  vma_delete(this, m_pBlockVectors[i][j]);
6248  }
6249  }
6250 }
6251 
6252 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
6253 {
6254 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6255  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
6256  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
6257  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
6258  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
6259  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
6260  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
6261  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
6262  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
6263  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
6264  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
6265  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
6266  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
6267  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
6268  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
6269 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6270 
6271  if(pVulkanFunctions != VMA_NULL)
6272  {
6273  m_VulkanFunctions = *pVulkanFunctions;
6274  }
6275 
6276  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
6277  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
6278  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
6279  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
6280  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
6281  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
6282  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
6283  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
6284  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
6285  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
6286  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
6287  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
6288  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
6289  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
6290  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
6291  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
6292 }
6293 
6294 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
6295 {
6296  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6297  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
6298  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
6299  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
6300 }
6301 
6302 VkResult VmaAllocator_T::AllocateMemoryOfType(
6303  const VkMemoryRequirements& vkMemReq,
6304  const VmaAllocationCreateInfo& createInfo,
6305  uint32_t memTypeIndex,
6306  VmaSuballocationType suballocType,
6307  VmaAllocation* pAllocation)
6308 {
6309  VMA_ASSERT(pAllocation != VMA_NULL);
6310  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
6311 
6312  uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags);
6313  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6314  VMA_ASSERT(blockVector);
6315 
6316  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
6317  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
6318  const bool ownMemory =
6319  (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 ||
6320  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
6321  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
6322  vkMemReq.size > preferredBlockSize / 2);
6323 
6324  if(ownMemory)
6325  {
6326  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6327  {
6328  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6329  }
6330  else
6331  {
6332  return AllocateOwnMemory(
6333  vkMemReq.size,
6334  suballocType,
6335  memTypeIndex,
6336  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6337  createInfo.pUserData,
6338  pAllocation);
6339  }
6340  }
6341  else
6342  {
6343  VkResult res = blockVector->Allocate(
6344  VK_NULL_HANDLE, // hCurrentPool
6345  m_CurrentFrameIndex.load(),
6346  vkMemReq,
6347  createInfo,
6348  suballocType,
6349  pAllocation);
6350  if(res == VK_SUCCESS)
6351  {
6352  return res;
6353  }
6354 
6355  // 5. Try own memory.
6356  res = AllocateOwnMemory(
6357  vkMemReq.size,
6358  suballocType,
6359  memTypeIndex,
6360  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6361  createInfo.pUserData,
6362  pAllocation);
6363  if(res == VK_SUCCESS)
6364  {
6365  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
6366  VMA_DEBUG_LOG(" Allocated as OwnMemory");
6367  return VK_SUCCESS;
6368  }
6369  else
6370  {
6371  // Everything failed: Return error code.
6372  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6373  return res;
6374  }
6375  }
6376 }
6377 
6378 VkResult VmaAllocator_T::AllocateOwnMemory(
6379  VkDeviceSize size,
6380  VmaSuballocationType suballocType,
6381  uint32_t memTypeIndex,
6382  bool map,
6383  void* pUserData,
6384  VmaAllocation* pAllocation)
6385 {
6386  VMA_ASSERT(pAllocation);
6387 
6388  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6389  allocInfo.memoryTypeIndex = memTypeIndex;
6390  allocInfo.allocationSize = size;
6391 
6392  // Allocate VkDeviceMemory.
6393  VkDeviceMemory hMemory = VK_NULL_HANDLE;
6394  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
6395  if(res < 0)
6396  {
6397  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6398  return res;
6399  }
6400 
6401  void* pMappedData = nullptr;
6402  if(map)
6403  {
6404  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
6405  {
6406  res = (*m_VulkanFunctions.vkMapMemory)(
6407  m_hDevice,
6408  hMemory,
6409  0,
6410  VK_WHOLE_SIZE,
6411  0,
6412  &pMappedData);
6413  if(res < 0)
6414  {
6415  VMA_DEBUG_LOG(" vkMapMemory FAILED");
6416  FreeVulkanMemory(memTypeIndex, size, hMemory);
6417  return res;
6418  }
6419  }
6420  }
6421 
6422  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load());
6423  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
6424 
6425  // Register it in m_pOwnAllocations.
6426  {
6427  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6428  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
6429  VMA_ASSERT(pOwnAllocations);
6430  VmaVectorInsertSorted<VmaPointerLess>(*pOwnAllocations, *pAllocation);
6431  }
6432 
6433  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
6434 
6435  return VK_SUCCESS;
6436 }
6437 
6438 VkResult VmaAllocator_T::AllocateMemory(
6439  const VkMemoryRequirements& vkMemReq,
6440  const VmaAllocationCreateInfo& createInfo,
6441  VmaSuballocationType suballocType,
6442  VmaAllocation* pAllocation)
6443 {
6444  if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 &&
6445  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6446  {
6447  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
6448  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6449  }
6450  if((createInfo.pool != VK_NULL_HANDLE) &&
6451  ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0))
6452  {
6453  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid.");
6454  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6455  }
6456 
6457  if(createInfo.pool != VK_NULL_HANDLE)
6458  {
6459  return createInfo.pool->m_BlockVector.Allocate(
6460  createInfo.pool,
6461  m_CurrentFrameIndex.load(),
6462  vkMemReq,
6463  createInfo,
6464  suballocType,
6465  pAllocation);
6466  }
6467  else
6468  {
6469  // Bit mask of memory Vulkan types acceptable for this allocation.
6470  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
6471  uint32_t memTypeIndex = UINT32_MAX;
6472  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6473  if(res == VK_SUCCESS)
6474  {
6475  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6476  // Succeeded on first try.
6477  if(res == VK_SUCCESS)
6478  {
6479  return res;
6480  }
6481  // Allocation from this memory type failed. Try other compatible memory types.
6482  else
6483  {
6484  for(;;)
6485  {
6486  // Remove old memTypeIndex from list of possibilities.
6487  memoryTypeBits &= ~(1u << memTypeIndex);
6488  // Find alternative memTypeIndex.
6489  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6490  if(res == VK_SUCCESS)
6491  {
6492  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6493  // Allocation from this alternative memory type succeeded.
6494  if(res == VK_SUCCESS)
6495  {
6496  return res;
6497  }
6498  // else: Allocation from this memory type failed. Try next one - next loop iteration.
6499  }
6500  // No other matching memory type index could be found.
6501  else
6502  {
6503  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
6504  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6505  }
6506  }
6507  }
6508  }
6509  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
6510  else
6511  return res;
6512  }
6513 }
6514 
6515 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
6516 {
6517  VMA_ASSERT(allocation);
6518 
6519  if(allocation->CanBecomeLost() == false ||
6520  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6521  {
6522  switch(allocation->GetType())
6523  {
6524  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
6525  {
6526  VmaBlockVector* pBlockVector = VMA_NULL;
6527  VmaPool hPool = allocation->GetPool();
6528  if(hPool != VK_NULL_HANDLE)
6529  {
6530  pBlockVector = &hPool->m_BlockVector;
6531  }
6532  else
6533  {
6534  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
6535  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
6536  pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6537  }
6538  pBlockVector->Free(allocation);
6539  }
6540  break;
6541  case VmaAllocation_T::ALLOCATION_TYPE_OWN:
6542  FreeOwnMemory(allocation);
6543  break;
6544  default:
6545  VMA_ASSERT(0);
6546  }
6547  }
6548 
6549  vma_delete(this, allocation);
6550 }
6551 
6552 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
6553 {
6554  // Initialize.
6555  InitStatInfo(pStats->total);
6556  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
6557  InitStatInfo(pStats->memoryType[i]);
6558  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6559  InitStatInfo(pStats->memoryHeap[i]);
6560 
6561  // Process default pools.
6562  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6563  {
6564  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6565  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6566  {
6567  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6568  VMA_ASSERT(pBlockVector);
6569  pBlockVector->AddStats(pStats);
6570  }
6571  }
6572 
6573  // Process custom pools.
6574  {
6575  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6576  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6577  {
6578  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
6579  }
6580  }
6581 
6582  // Process own allocations.
6583  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6584  {
6585  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6586  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6587  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6588  {
6589  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
6590  VMA_ASSERT(pOwnAllocVector);
6591  for(size_t allocIndex = 0, allocCount = pOwnAllocVector->size(); allocIndex < allocCount; ++allocIndex)
6592  {
6593  VmaStatInfo allocationStatInfo;
6594  (*pOwnAllocVector)[allocIndex]->OwnAllocCalcStatsInfo(allocationStatInfo);
6595  VmaAddStatInfo(pStats->total, allocationStatInfo);
6596  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6597  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6598  }
6599  }
6600  }
6601 
6602  // Postprocess.
6603  VmaPostprocessCalcStatInfo(pStats->total);
6604  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
6605  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
6606  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
6607  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
6608 }
6609 
6610 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
6611 
6612 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
6613 {
6614  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
6615  {
6616  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6617  {
6618  for(uint32_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
6619  {
6620  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6621  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6622  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6623  {
6624  // Process OwnAllocations.
6625  {
6626  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6627  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6628  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
6629  {
6630  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
6631  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(this);
6632  }
6633  }
6634 
6635  // Process normal Allocations.
6636  {
6637  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6638  pBlockVector->UnmapPersistentlyMappedMemory();
6639  }
6640  }
6641  }
6642 
6643  // Process custom pools.
6644  {
6645  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6646  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6647  {
6648  m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory();
6649  }
6650  }
6651  }
6652  }
6653 }
6654 
6655 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
6656 {
6657  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
6658  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
6659  {
6660  VkResult finalResult = VK_SUCCESS;
6661  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6662  {
6663  // Process custom pools.
6664  {
6665  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6666  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6667  {
6668  m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory();
6669  }
6670  }
6671 
6672  for(uint32_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
6673  {
6674  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6675  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6676  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6677  {
6678  // Process OwnAllocations.
6679  {
6680  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6681  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6682  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
6683  {
6684  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
6685  hAlloc->OwnAllocMapPersistentlyMappedMemory(this);
6686  }
6687  }
6688 
6689  // Process normal Allocations.
6690  {
6691  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6692  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
6693  if(localResult != VK_SUCCESS)
6694  {
6695  finalResult = localResult;
6696  }
6697  }
6698  }
6699  }
6700  }
6701  return finalResult;
6702  }
6703  else
6704  return VK_SUCCESS;
6705 }
6706 
6707 VkResult VmaAllocator_T::Defragment(
6708  VmaAllocation* pAllocations,
6709  size_t allocationCount,
6710  VkBool32* pAllocationsChanged,
6711  const VmaDefragmentationInfo* pDefragmentationInfo,
6712  VmaDefragmentationStats* pDefragmentationStats)
6713 {
6714  if(pAllocationsChanged != VMA_NULL)
6715  {
6716  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
6717  }
6718  if(pDefragmentationStats != VMA_NULL)
6719  {
6720  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
6721  }
6722 
6723  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
6724  {
6725  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
6726  return VK_ERROR_MEMORY_MAP_FAILED;
6727  }
6728 
6729  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
6730 
6731  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
6732 
6733  const size_t poolCount = m_Pools.size();
6734 
6735  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
6736  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
6737  {
6738  VmaAllocation hAlloc = pAllocations[allocIndex];
6739  VMA_ASSERT(hAlloc);
6740  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
6741  // OwnAlloc cannot be defragmented.
6742  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
6743  // Only HOST_VISIBLE memory types can be defragmented.
6744  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
6745  // Lost allocation cannot be defragmented.
6746  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
6747  {
6748  VmaBlockVector* pAllocBlockVector = nullptr;
6749 
6750  const VmaPool hAllocPool = hAlloc->GetPool();
6751  // This allocation belongs to custom pool.
6752  if(hAllocPool != VK_NULL_HANDLE)
6753  {
6754  pAllocBlockVector = &hAllocPool->GetBlockVector();
6755  }
6756  // This allocation belongs to general pool.
6757  else
6758  {
6759  pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()];
6760  }
6761 
6762  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
6763 
6764  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
6765  &pAllocationsChanged[allocIndex] : VMA_NULL;
6766  pDefragmentator->AddAllocation(hAlloc, pChanged);
6767  }
6768  }
6769 
6770  VkResult result = VK_SUCCESS;
6771 
6772  // ======== Main processing.
6773 
6774  VkDeviceSize maxBytesToMove = SIZE_MAX;
6775  uint32_t maxAllocationsToMove = UINT32_MAX;
6776  if(pDefragmentationInfo != VMA_NULL)
6777  {
6778  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
6779  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
6780  }
6781 
6782  // Process standard memory.
6783  for(uint32_t memTypeIndex = 0;
6784  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
6785  ++memTypeIndex)
6786  {
6787  // Only HOST_VISIBLE memory types can be defragmented.
6788  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6789  {
6790  for(uint32_t blockVectorType = 0;
6791  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
6792  ++blockVectorType)
6793  {
6794  result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment(
6795  pDefragmentationStats,
6796  maxBytesToMove,
6797  maxAllocationsToMove);
6798  }
6799  }
6800  }
6801 
6802  // Process custom pools.
6803  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
6804  {
6805  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
6806  pDefragmentationStats,
6807  maxBytesToMove,
6808  maxAllocationsToMove);
6809  }
6810 
6811  // ======== Destroy defragmentators.
6812 
6813  // Process custom pools.
6814  for(size_t poolIndex = poolCount; poolIndex--; )
6815  {
6816  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
6817  }
6818 
6819  // Process standard memory.
6820  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
6821  {
6822  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6823  {
6824  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
6825  {
6826  m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator();
6827  }
6828  }
6829  }
6830 
6831  return result;
6832 }
6833 
6834 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
6835 {
6836  if(hAllocation->CanBecomeLost())
6837  {
6838  /*
6839  Warning: This is a carefully designed algorithm.
6840  Do not modify unless you really know what you're doing :)
6841  */
6842  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
6843  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
6844  for(;;)
6845  {
6846  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6847  {
6848  pAllocationInfo->memoryType = UINT32_MAX;
6849  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
6850  pAllocationInfo->offset = 0;
6851  pAllocationInfo->size = hAllocation->GetSize();
6852  pAllocationInfo->pMappedData = VMA_NULL;
6853  pAllocationInfo->pUserData = hAllocation->GetUserData();
6854  return;
6855  }
6856  else if(localLastUseFrameIndex == localCurrFrameIndex)
6857  {
6858  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6859  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6860  pAllocationInfo->offset = hAllocation->GetOffset();
6861  pAllocationInfo->size = hAllocation->GetSize();
6862  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6863  pAllocationInfo->pUserData = hAllocation->GetUserData();
6864  return;
6865  }
6866  else // Last use time earlier than current time.
6867  {
6868  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
6869  {
6870  localLastUseFrameIndex = localCurrFrameIndex;
6871  }
6872  }
6873  }
6874  }
6875  // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic.
6876  else
6877  {
6878  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6879  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6880  pAllocationInfo->offset = hAllocation->GetOffset();
6881  pAllocationInfo->size = hAllocation->GetSize();
6882  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6883  pAllocationInfo->pUserData = hAllocation->GetUserData();
6884  }
6885 }
6886 
6887 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
6888 {
6889  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
6890 
6891  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
6892 
6893  if(newCreateInfo.maxBlockCount == 0)
6894  {
6895  newCreateInfo.maxBlockCount = SIZE_MAX;
6896  }
6897  if(newCreateInfo.blockSize == 0)
6898  {
6899  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
6900  }
6901 
6902  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
6903 
6904  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
6905  if(res != VK_SUCCESS)
6906  {
6907  vma_delete(this, *pPool);
6908  *pPool = VMA_NULL;
6909  return res;
6910  }
6911 
6912  // Add to m_Pools.
6913  {
6914  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6915  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
6916  }
6917 
6918  return VK_SUCCESS;
6919 }
6920 
6921 void VmaAllocator_T::DestroyPool(VmaPool pool)
6922 {
6923  // Remove from m_Pools.
6924  {
6925  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6926  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
6927  VMA_ASSERT(success && "Pool not found in Allocator.");
6928  }
6929 
6930  vma_delete(this, pool);
6931 }
6932 
6933 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
6934 {
6935  pool->m_BlockVector.GetPoolStats(pPoolStats);
6936 }
6937 
6938 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
6939 {
6940  m_CurrentFrameIndex.store(frameIndex);
6941 }
6942 
6943 void VmaAllocator_T::MakePoolAllocationsLost(
6944  VmaPool hPool,
6945  size_t* pLostAllocationCount)
6946 {
6947  hPool->m_BlockVector.MakePoolAllocationsLost(
6948  m_CurrentFrameIndex.load(),
6949  pLostAllocationCount);
6950 }
6951 
6952 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
6953 {
6954  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST);
6955  (*pAllocation)->InitLost();
6956 }
6957 
6958 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
6959 {
6960  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
6961 
6962  VkResult res;
6963  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
6964  {
6965  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
6966  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
6967  {
6968  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6969  if(res == VK_SUCCESS)
6970  {
6971  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
6972  }
6973  }
6974  else
6975  {
6976  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
6977  }
6978  }
6979  else
6980  {
6981  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6982  }
6983 
6984  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
6985  {
6986  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
6987  }
6988 
6989  return res;
6990 }
6991 
6992 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
6993 {
6994  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
6995  {
6996  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
6997  }
6998 
6999  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7000 
7001  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7002  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7003  {
7004  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7005  m_HeapSizeLimit[heapIndex] += size;
7006  }
7007 }
7008 
7009 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
7010 {
7011  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
7012 
7013  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7014  {
7015  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7016  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
7017  VMA_ASSERT(pOwnAllocations);
7018  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pOwnAllocations, allocation);
7019  VMA_ASSERT(success);
7020  }
7021 
7022  VkDeviceMemory hMemory = allocation->GetMemory();
7023 
7024  if(allocation->GetMappedData() != VMA_NULL)
7025  {
7026  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7027  }
7028 
7029  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7030 
7031  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
7032 }
7033 
7034 #if VMA_STATS_STRING_ENABLED
7035 
7036 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7037 {
7038  bool ownAllocationsStarted = false;
7039  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7040  {
7041  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7042  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7043  {
7044  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
7045  VMA_ASSERT(pOwnAllocVector);
7046  if(pOwnAllocVector->empty() == false)
7047  {
7048  if(ownAllocationsStarted == false)
7049  {
7050  ownAllocationsStarted = true;
7051  json.WriteString("OwnAllocations");
7052  json.BeginObject();
7053  }
7054 
7055  json.BeginString("Type ");
7056  json.ContinueString(memTypeIndex);
7057  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7058  {
7059  json.ContinueString(" Mapped");
7060  }
7061  json.EndString();
7062 
7063  json.BeginArray();
7064 
7065  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
7066  {
7067  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
7068  json.BeginObject(true);
7069 
7070  json.WriteString("Size");
7071  json.WriteNumber(hAlloc->GetSize());
7072 
7073  json.WriteString("Type");
7074  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7075 
7076  json.EndObject();
7077  }
7078 
7079  json.EndArray();
7080  }
7081  }
7082  }
7083  if(ownAllocationsStarted)
7084  {
7085  json.EndObject();
7086  }
7087 
7088  {
7089  bool allocationsStarted = false;
7090  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7091  {
7092  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7093  {
7094  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
7095  {
7096  if(allocationsStarted == false)
7097  {
7098  allocationsStarted = true;
7099  json.WriteString("DefaultPools");
7100  json.BeginObject();
7101  }
7102 
7103  json.BeginString("Type ");
7104  json.ContinueString(memTypeIndex);
7105  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7106  {
7107  json.ContinueString(" Mapped");
7108  }
7109  json.EndString();
7110 
7111  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json);
7112  }
7113  }
7114  }
7115  if(allocationsStarted)
7116  {
7117  json.EndObject();
7118  }
7119  }
7120 
7121  {
7122  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7123  const size_t poolCount = m_Pools.size();
7124  if(poolCount > 0)
7125  {
7126  json.WriteString("Pools");
7127  json.BeginArray();
7128  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
7129  {
7130  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
7131  }
7132  json.EndArray();
7133  }
7134  }
7135 }
7136 
7137 #endif // #if VMA_STATS_STRING_ENABLED
7138 
7139 static VkResult AllocateMemoryForImage(
7140  VmaAllocator allocator,
7141  VkImage image,
7142  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7143  VmaSuballocationType suballocType,
7144  VmaAllocation* pAllocation)
7145 {
7146  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
7147 
7148  VkMemoryRequirements vkMemReq = {};
7149  (*allocator->GetVulkanFunctions().vkGetImageMemoryRequirements)(allocator->m_hDevice, image, &vkMemReq);
7150 
7151  return allocator->AllocateMemory(
7152  vkMemReq,
7153  *pAllocationCreateInfo,
7154  suballocType,
7155  pAllocation);
7156 }
7157 
7159 // Public interface
7160 
7161 VkResult vmaCreateAllocator(
7162  const VmaAllocatorCreateInfo* pCreateInfo,
7163  VmaAllocator* pAllocator)
7164 {
7165  VMA_ASSERT(pCreateInfo && pAllocator);
7166  VMA_DEBUG_LOG("vmaCreateAllocator");
7167  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
7168  return VK_SUCCESS;
7169 }
7170 
7171 void vmaDestroyAllocator(
7172  VmaAllocator allocator)
7173 {
7174  if(allocator != VK_NULL_HANDLE)
7175  {
7176  VMA_DEBUG_LOG("vmaDestroyAllocator");
7177  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
7178  vma_delete(&allocationCallbacks, allocator);
7179  }
7180 }
7181 
7182 void vmaGetPhysicalDeviceProperties(
7183  VmaAllocator allocator,
7184  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
7185 {
7186  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
7187  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
7188 }
7189 
7190 void vmaGetMemoryProperties(
7191  VmaAllocator allocator,
7192  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
7193 {
7194  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
7195  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
7196 }
7197 
7198 void vmaGetMemoryTypeProperties(
7199  VmaAllocator allocator,
7200  uint32_t memoryTypeIndex,
7201  VkMemoryPropertyFlags* pFlags)
7202 {
7203  VMA_ASSERT(allocator && pFlags);
7204  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
7205  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
7206 }
7207 
7208 void vmaSetCurrentFrameIndex(
7209  VmaAllocator allocator,
7210  uint32_t frameIndex)
7211 {
7212  VMA_ASSERT(allocator);
7213  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
7214 
7215  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7216 
7217  allocator->SetCurrentFrameIndex(frameIndex);
7218 }
7219 
7220 void vmaCalculateStats(
7221  VmaAllocator allocator,
7222  VmaStats* pStats)
7223 {
7224  VMA_ASSERT(allocator && pStats);
7225  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7226  allocator->CalculateStats(pStats);
7227 }
7228 
7229 #if VMA_STATS_STRING_ENABLED
7230 
7231 void vmaBuildStatsString(
7232  VmaAllocator allocator,
7233  char** ppStatsString,
7234  VkBool32 detailedMap)
7235 {
7236  VMA_ASSERT(allocator && ppStatsString);
7237  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7238 
7239  VmaStringBuilder sb(allocator);
7240  {
7241  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
7242  json.BeginObject();
7243 
7244  VmaStats stats;
7245  allocator->CalculateStats(&stats);
7246 
7247  json.WriteString("Total");
7248  VmaPrintStatInfo(json, stats.total);
7249 
7250  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
7251  {
7252  json.BeginString("Heap ");
7253  json.ContinueString(heapIndex);
7254  json.EndString();
7255  json.BeginObject();
7256 
7257  json.WriteString("Size");
7258  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
7259 
7260  json.WriteString("Flags");
7261  json.BeginArray(true);
7262  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
7263  {
7264  json.WriteString("DEVICE_LOCAL");
7265  }
7266  json.EndArray();
7267 
7268  if(stats.memoryHeap[heapIndex].blockCount > 0)
7269  {
7270  json.WriteString("Stats");
7271  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
7272  }
7273 
7274  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
7275  {
7276  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
7277  {
7278  json.BeginString("Type ");
7279  json.ContinueString(typeIndex);
7280  json.EndString();
7281 
7282  json.BeginObject();
7283 
7284  json.WriteString("Flags");
7285  json.BeginArray(true);
7286  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
7287  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
7288  {
7289  json.WriteString("DEVICE_LOCAL");
7290  }
7291  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7292  {
7293  json.WriteString("HOST_VISIBLE");
7294  }
7295  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
7296  {
7297  json.WriteString("HOST_COHERENT");
7298  }
7299  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
7300  {
7301  json.WriteString("HOST_CACHED");
7302  }
7303  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
7304  {
7305  json.WriteString("LAZILY_ALLOCATED");
7306  }
7307  json.EndArray();
7308 
7309  if(stats.memoryType[typeIndex].blockCount > 0)
7310  {
7311  json.WriteString("Stats");
7312  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
7313  }
7314 
7315  json.EndObject();
7316  }
7317  }
7318 
7319  json.EndObject();
7320  }
7321  if(detailedMap == VK_TRUE)
7322  {
7323  allocator->PrintDetailedMap(json);
7324  }
7325 
7326  json.EndObject();
7327  }
7328 
7329  const size_t len = sb.GetLength();
7330  char* const pChars = vma_new_array(allocator, char, len + 1);
7331  if(len > 0)
7332  {
7333  memcpy(pChars, sb.GetData(), len);
7334  }
7335  pChars[len] = '\0';
7336  *ppStatsString = pChars;
7337 }
7338 
7339 void vmaFreeStatsString(
7340  VmaAllocator allocator,
7341  char* pStatsString)
7342 {
7343  if(pStatsString != VMA_NULL)
7344  {
7345  VMA_ASSERT(allocator);
7346  size_t len = strlen(pStatsString);
7347  vma_delete_array(allocator, pStatsString, len + 1);
7348  }
7349 }
7350 
7351 #endif // #if VMA_STATS_STRING_ENABLED
7352 
7355 VkResult vmaFindMemoryTypeIndex(
7356  VmaAllocator allocator,
7357  uint32_t memoryTypeBits,
7358  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7359  uint32_t* pMemoryTypeIndex)
7360 {
7361  VMA_ASSERT(allocator != VK_NULL_HANDLE);
7362  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
7363  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
7364 
7365  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
7366  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
7367  if(preferredFlags == 0)
7368  {
7369  preferredFlags = requiredFlags;
7370  }
7371  // preferredFlags, if not 0, must be a superset of requiredFlags.
7372  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
7373 
7374  // Convert usage to requiredFlags and preferredFlags.
7375  switch(pAllocationCreateInfo->usage)
7376  {
7377  case VMA_MEMORY_USAGE_UNKNOWN:
7378  break;
7379  case VMA_MEMORY_USAGE_GPU_ONLY:
7380  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7381  break;
7382  case VMA_MEMORY_USAGE_CPU_ONLY:
7383  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7384  break;
7385  case VMA_MEMORY_USAGE_CPU_TO_GPU:
7386  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7387  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7388  break;
7389  case VMA_MEMORY_USAGE_GPU_TO_CPU:
7390  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7391  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
7392  break;
7393  default:
7394  break;
7395  }
7396 
7397  if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0)
7398  {
7399  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7400  }
7401 
7402  *pMemoryTypeIndex = UINT32_MAX;
7403  uint32_t minCost = UINT32_MAX;
7404  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
7405  memTypeIndex < allocator->GetMemoryTypeCount();
7406  ++memTypeIndex, memTypeBit <<= 1)
7407  {
7408  // This memory type is acceptable according to memoryTypeBits bitmask.
7409  if((memTypeBit & memoryTypeBits) != 0)
7410  {
7411  const VkMemoryPropertyFlags currFlags =
7412  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
7413  // This memory type contains requiredFlags.
7414  if((requiredFlags & ~currFlags) == 0)
7415  {
7416  // Calculate cost as number of bits from preferredFlags not present in this memory type.
7417  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
7418  // Remember memory type with lowest cost.
7419  if(currCost < minCost)
7420  {
7421  *pMemoryTypeIndex = memTypeIndex;
7422  if(currCost == 0)
7423  {
7424  return VK_SUCCESS;
7425  }
7426  minCost = currCost;
7427  }
7428  }
7429  }
7430  }
7431  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
7432 }
7433 
7434 VkResult vmaCreatePool(
7435  VmaAllocator allocator,
7436  const VmaPoolCreateInfo* pCreateInfo,
7437  VmaPool* pPool)
7438 {
7439  VMA_ASSERT(allocator && pCreateInfo && pPool);
7440 
7441  VMA_DEBUG_LOG("vmaCreatePool");
7442 
7443  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7444 
7445  return allocator->CreatePool(pCreateInfo, pPool);
7446 }
7447 
7448 void vmaDestroyPool(
7449  VmaAllocator allocator,
7450  VmaPool pool)
7451 {
7452  VMA_ASSERT(allocator && pool);
7453 
7454  VMA_DEBUG_LOG("vmaDestroyPool");
7455 
7456  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7457 
7458  allocator->DestroyPool(pool);
7459 }
7460 
7461 void vmaGetPoolStats(
7462  VmaAllocator allocator,
7463  VmaPool pool,
7464  VmaPoolStats* pPoolStats)
7465 {
7466  VMA_ASSERT(allocator && pool && pPoolStats);
7467 
7468  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7469 
7470  allocator->GetPoolStats(pool, pPoolStats);
7471 }
7472 
7473 void vmaMakePoolAllocationsLost(
7474  VmaAllocator allocator,
7475  VmaPool pool,
7476  size_t* pLostAllocationCount)
7477 {
7478  VMA_ASSERT(allocator && pool);
7479 
7480  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7481 
7482  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
7483 }
7484 
7485 VkResult vmaAllocateMemory(
7486  VmaAllocator allocator,
7487  const VkMemoryRequirements* pVkMemoryRequirements,
7488  const VmaAllocationCreateInfo* pCreateInfo,
7489  VmaAllocation* pAllocation,
7490  VmaAllocationInfo* pAllocationInfo)
7491 {
7492  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
7493 
7494  VMA_DEBUG_LOG("vmaAllocateMemory");
7495 
7496  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7497 
7498  VkResult result = allocator->AllocateMemory(
7499  *pVkMemoryRequirements,
7500  *pCreateInfo,
7501  VMA_SUBALLOCATION_TYPE_UNKNOWN,
7502  pAllocation);
7503 
7504  if(pAllocationInfo && result == VK_SUCCESS)
7505  {
7506  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7507  }
7508 
7509  return result;
7510 }
7511 
7512 VkResult vmaAllocateMemoryForBuffer(
7513  VmaAllocator allocator,
7514  VkBuffer buffer,
7515  const VmaAllocationCreateInfo* pCreateInfo,
7516  VmaAllocation* pAllocation,
7517  VmaAllocationInfo* pAllocationInfo)
7518 {
7519  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7520 
7521  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
7522 
7523  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7524 
7525  VkMemoryRequirements vkMemReq = {};
7526  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, buffer, &vkMemReq);
7527 
7528  VkResult result = allocator->AllocateMemory(
7529  vkMemReq,
7530  *pCreateInfo,
7531  VMA_SUBALLOCATION_TYPE_BUFFER,
7532  pAllocation);
7533 
7534  if(pAllocationInfo && result == VK_SUCCESS)
7535  {
7536  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7537  }
7538 
7539  return result;
7540 }
7541 
7542 VkResult vmaAllocateMemoryForImage(
7543  VmaAllocator allocator,
7544  VkImage image,
7545  const VmaAllocationCreateInfo* pCreateInfo,
7546  VmaAllocation* pAllocation,
7547  VmaAllocationInfo* pAllocationInfo)
7548 {
7549  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7550 
7551  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
7552 
7553  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7554 
7555  VkResult result = AllocateMemoryForImage(
7556  allocator,
7557  image,
7558  pCreateInfo,
7559  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
7560  pAllocation);
7561 
7562  if(pAllocationInfo && result == VK_SUCCESS)
7563  {
7564  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7565  }
7566 
7567  return result;
7568 }
7569 
7570 void vmaFreeMemory(
7571  VmaAllocator allocator,
7572  VmaAllocation allocation)
7573 {
7574  VMA_ASSERT(allocator && allocation);
7575 
7576  VMA_DEBUG_LOG("vmaFreeMemory");
7577 
7578  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7579 
7580  allocator->FreeMemory(allocation);
7581 }
7582 
7583 void vmaGetAllocationInfo(
7584  VmaAllocator allocator,
7585  VmaAllocation allocation,
7586  VmaAllocationInfo* pAllocationInfo)
7587 {
7588  VMA_ASSERT(allocator && allocation && pAllocationInfo);
7589 
7590  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7591 
7592  allocator->GetAllocationInfo(allocation, pAllocationInfo);
7593 }
7594 
7595 void vmaSetAllocationUserData(
7596  VmaAllocator allocator,
7597  VmaAllocation allocation,
7598  void* pUserData)
7599 {
7600  VMA_ASSERT(allocator && allocation);
7601 
7602  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7603 
7604  allocation->SetUserData(pUserData);
7605 }
7606 
7607 void vmaCreateLostAllocation(
7608  VmaAllocator allocator,
7609  VmaAllocation* pAllocation)
7610 {
7611  VMA_ASSERT(allocator && pAllocation);
7612 
7613  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
7614 
7615  allocator->CreateLostAllocation(pAllocation);
7616 }
7617 
7618 VkResult vmaMapMemory(
7619  VmaAllocator allocator,
7620  VmaAllocation allocation,
7621  void** ppData)
7622 {
7623  VMA_ASSERT(allocator && allocation && ppData);
7624 
7625  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7626 
7627  return (*allocator->GetVulkanFunctions().vkMapMemory)(
7628  allocator->m_hDevice,
7629  allocation->GetMemory(),
7630  allocation->GetOffset(),
7631  allocation->GetSize(),
7632  0,
7633  ppData);
7634 }
7635 
7636 void vmaUnmapMemory(
7637  VmaAllocator allocator,
7638  VmaAllocation allocation)
7639 {
7640  VMA_ASSERT(allocator && allocation);
7641 
7642  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7643 
7644  (*allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, allocation->GetMemory());
7645 }
7646 
7647 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
7648 {
7649  VMA_ASSERT(allocator);
7650 
7651  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7652 
7653  allocator->UnmapPersistentlyMappedMemory();
7654 }
7655 
7656 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
7657 {
7658  VMA_ASSERT(allocator);
7659 
7660  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7661 
7662  return allocator->MapPersistentlyMappedMemory();
7663 }
7664 
7665 VkResult vmaDefragment(
7666  VmaAllocator allocator,
7667  VmaAllocation* pAllocations,
7668  size_t allocationCount,
7669  VkBool32* pAllocationsChanged,
7670  const VmaDefragmentationInfo *pDefragmentationInfo,
7671  VmaDefragmentationStats* pDefragmentationStats)
7672 {
7673  VMA_ASSERT(allocator && pAllocations);
7674 
7675  VMA_DEBUG_LOG("vmaDefragment");
7676 
7677  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7678 
7679  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
7680 }
7681 
7682 VkResult vmaCreateBuffer(
7683  VmaAllocator allocator,
7684  const VkBufferCreateInfo* pBufferCreateInfo,
7685  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7686  VkBuffer* pBuffer,
7687  VmaAllocation* pAllocation,
7688  VmaAllocationInfo* pAllocationInfo)
7689 {
7690  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
7691 
7692  VMA_DEBUG_LOG("vmaCreateBuffer");
7693 
7694  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7695 
7696  *pBuffer = VK_NULL_HANDLE;
7697  *pAllocation = VK_NULL_HANDLE;
7698 
7699  // 1. Create VkBuffer.
7700  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
7701  allocator->m_hDevice,
7702  pBufferCreateInfo,
7703  allocator->GetAllocationCallbacks(),
7704  pBuffer);
7705  if(res >= 0)
7706  {
7707  // 2. vkGetBufferMemoryRequirements.
7708  VkMemoryRequirements vkMemReq = {};
7709  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, *pBuffer, &vkMemReq);
7710 
7711  // 3. Allocate memory using allocator.
7712  res = allocator->AllocateMemory(
7713  vkMemReq,
7714  *pAllocationCreateInfo,
7715  VMA_SUBALLOCATION_TYPE_BUFFER,
7716  pAllocation);
7717  if(res >= 0)
7718  {
7719  // 3. Bind buffer with memory.
7720  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
7721  allocator->m_hDevice,
7722  *pBuffer,
7723  (*pAllocation)->GetMemory(),
7724  (*pAllocation)->GetOffset());
7725  if(res >= 0)
7726  {
7727  // All steps succeeded.
7728  if(pAllocationInfo != VMA_NULL)
7729  {
7730  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7731  }
7732  return VK_SUCCESS;
7733  }
7734  allocator->FreeMemory(*pAllocation);
7735  *pAllocation = VK_NULL_HANDLE;
7736  return res;
7737  }
7738  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
7739  *pBuffer = VK_NULL_HANDLE;
7740  return res;
7741  }
7742  return res;
7743 }
7744 
7745 void vmaDestroyBuffer(
7746  VmaAllocator allocator,
7747  VkBuffer buffer,
7748  VmaAllocation allocation)
7749 {
7750  if(buffer != VK_NULL_HANDLE)
7751  {
7752  VMA_ASSERT(allocator);
7753 
7754  VMA_DEBUG_LOG("vmaDestroyBuffer");
7755 
7756  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7757 
7758  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
7759 
7760  allocator->FreeMemory(allocation);
7761  }
7762 }
7763 
7764 VkResult vmaCreateImage(
7765  VmaAllocator allocator,
7766  const VkImageCreateInfo* pImageCreateInfo,
7767  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7768  VkImage* pImage,
7769  VmaAllocation* pAllocation,
7770  VmaAllocationInfo* pAllocationInfo)
7771 {
7772  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
7773 
7774  VMA_DEBUG_LOG("vmaCreateImage");
7775 
7776  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7777 
7778  *pImage = VK_NULL_HANDLE;
7779  *pAllocation = VK_NULL_HANDLE;
7780 
7781  // 1. Create VkImage.
7782  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
7783  allocator->m_hDevice,
7784  pImageCreateInfo,
7785  allocator->GetAllocationCallbacks(),
7786  pImage);
7787  if(res >= 0)
7788  {
7789  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
7790  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
7791  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
7792 
7793  // 2. Allocate memory using allocator.
7794  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
7795  if(res >= 0)
7796  {
7797  // 3. Bind image with memory.
7798  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
7799  allocator->m_hDevice,
7800  *pImage,
7801  (*pAllocation)->GetMemory(),
7802  (*pAllocation)->GetOffset());
7803  if(res >= 0)
7804  {
7805  // All steps succeeded.
7806  if(pAllocationInfo != VMA_NULL)
7807  {
7808  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7809  }
7810  return VK_SUCCESS;
7811  }
7812  allocator->FreeMemory(*pAllocation);
7813  *pAllocation = VK_NULL_HANDLE;
7814  return res;
7815  }
7816  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
7817  *pImage = VK_NULL_HANDLE;
7818  return res;
7819  }
7820  return res;
7821 }
7822 
7823 void vmaDestroyImage(
7824  VmaAllocator allocator,
7825  VkImage image,
7826  VmaAllocation allocation)
7827 {
7828  if(image != VK_NULL_HANDLE)
7829  {
7830  VMA_ASSERT(allocator);
7831 
7832  VMA_DEBUG_LOG("vmaDestroyImage");
7833 
7834  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7835 
7836  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
7837 
7838  allocator->FreeMemory(allocation);
7839  }
7840 }
7841 
7842 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:446
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
393 #include <vulkan/vulkan.h>
394 
396 
400 VK_DEFINE_HANDLE(VmaAllocator)
401 
402 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
404  VmaAllocator allocator,
405  uint32_t memoryType,
406  VkDeviceMemory memory,
407  VkDeviceSize size);
409 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
410  VmaAllocator allocator,
411  uint32_t memoryType,
412  VkDeviceMemory memory,
413  VkDeviceSize size);
414 
422 typedef struct VmaDeviceMemoryCallbacks {
424  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
426  PFN_vmaFreeDeviceMemoryFunction pfnFree;
427 } VmaDeviceMemoryCallbacks;
428 
430 typedef enum VmaAllocatorFlagBits {
435  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
436 
437  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
438 } VmaAllocatorFlagBits;
439 typedef VkFlags VmaAllocatorFlags;
440 
445 typedef struct VmaVulkanFunctions {
446  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
447  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
448  PFN_vkAllocateMemory vkAllocateMemory;
449  PFN_vkFreeMemory vkFreeMemory;
450  PFN_vkMapMemory vkMapMemory;
451  PFN_vkUnmapMemory vkUnmapMemory;
452  PFN_vkBindBufferMemory vkBindBufferMemory;
453  PFN_vkBindImageMemory vkBindImageMemory;
454  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
455  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
456  PFN_vkCreateBuffer vkCreateBuffer;
457  PFN_vkDestroyBuffer vkDestroyBuffer;
458  PFN_vkCreateImage vkCreateImage;
459  PFN_vkDestroyImage vkDestroyImage;
460 } VmaVulkanFunctions;
461 
463 typedef struct VmaAllocatorCreateInfo
464 {
466  VmaAllocatorFlags flags;
468 
469  VkPhysicalDevice physicalDevice;
471 
472  VkDevice device;
474 
475  VkDeviceSize preferredLargeHeapBlockSize;
477 
478  VkDeviceSize preferredSmallHeapBlockSize;
480 
481  const VkAllocationCallbacks* pAllocationCallbacks;
483 
484  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
498  uint32_t frameInUseCount;
516  const VkDeviceSize* pHeapSizeLimit;
528  const VmaVulkanFunctions* pVulkanFunctions;
529 } VmaAllocatorCreateInfo;
530 
532 VkResult vmaCreateAllocator(
533  const VmaAllocatorCreateInfo* pCreateInfo,
534  VmaAllocator* pAllocator);
535 
537 void vmaDestroyAllocator(
538  VmaAllocator allocator);
539 
544 void vmaGetPhysicalDeviceProperties(
545  VmaAllocator allocator,
546  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
547 
552 void vmaGetMemoryProperties(
553  VmaAllocator allocator,
554  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
555 
562 void vmaGetMemoryTypeProperties(
563  VmaAllocator allocator,
564  uint32_t memoryTypeIndex,
565  VkMemoryPropertyFlags* pFlags);
566 
575 void vmaSetCurrentFrameIndex(
576  VmaAllocator allocator,
577  uint32_t frameIndex);
578 
581 typedef struct VmaStatInfo
582 {
584  uint32_t blockCount;
586  uint32_t allocationCount;
588  uint32_t unusedRangeCount;
590  VkDeviceSize usedBytes;
592  VkDeviceSize unusedBytes;
593  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
594  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
595 } VmaStatInfo;
596 
598 typedef struct VmaStats
599 {
600  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
601  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
602  VmaStatInfo total;
603 } VmaStats;
604 
606 void vmaCalculateStats(
607  VmaAllocator allocator,
608  VmaStats* pStats);
609 
610 #define VMA_STATS_STRING_ENABLED 1
611 
612 #if VMA_STATS_STRING_ENABLED
613 
615 
617 void vmaBuildStatsString(
618  VmaAllocator allocator,
619  char** ppStatsString,
620  VkBool32 detailedMap);
621 
622 void vmaFreeStatsString(
623  VmaAllocator allocator,
624  char* pStatsString);
625 
626 #endif // #if VMA_STATS_STRING_ENABLED
627 
630 
635 VK_DEFINE_HANDLE(VmaPool)
636 
637 typedef enum VmaMemoryUsage
638 {
640  VMA_MEMORY_USAGE_UNKNOWN = 0,
642  VMA_MEMORY_USAGE_GPU_ONLY = 1,
644 
645  VMA_MEMORY_USAGE_CPU_ONLY = 2,
647 
648  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
650 
651  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
652  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
653 } VmaMemoryUsage;
654 
656 typedef enum VmaAllocationCreateFlagBits {
668  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001,
669 
679  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
695  VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004,
706  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
713  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
714 
715  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
716 } VmaAllocationCreateFlagBits;
717 typedef VkFlags VmaAllocationCreateFlags;
718 
719 typedef struct VmaAllocationCreateInfo
720 {
722  VmaAllocationCreateFlags flags;
728  VmaMemoryUsage usage;
733  VkMemoryPropertyFlags requiredFlags;
739  VkMemoryPropertyFlags preferredFlags;
741  void* pUserData;
746  VmaPool pool;
747 } VmaAllocationCreateInfo;
748 
763 VkResult vmaFindMemoryTypeIndex(
764  VmaAllocator allocator,
765  uint32_t memoryTypeBits,
766  const VmaAllocationCreateInfo* pAllocationCreateInfo,
767  uint32_t* pMemoryTypeIndex);
768 
771 
776 typedef enum VmaPoolCreateFlagBits {
786  VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001,
804  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
805 
806  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
807 } VmaPoolCreateFlagBits;
808 typedef VkFlags VmaPoolCreateFlags;
809 
812 typedef struct VmaPoolCreateInfo {
815  uint32_t memoryTypeIndex;
818  VmaPoolCreateFlags flags;
823  VkDeviceSize blockSize;
828  size_t minBlockCount;
836  size_t maxBlockCount;
850  uint32_t frameInUseCount;
851 } VmaPoolCreateInfo;
852 
855 typedef struct VmaPoolStats {
858  VkDeviceSize size;
861  VkDeviceSize unusedSize;
864  size_t allocationCount;
867  size_t unusedRangeCount;
874  VkDeviceSize unusedRangeSizeMax;
875 } VmaPoolStats;
876 
883 VkResult vmaCreatePool(
884  VmaAllocator allocator,
885  const VmaPoolCreateInfo* pCreateInfo,
886  VmaPool* pPool);
887 
890 void vmaDestroyPool(
891  VmaAllocator allocator,
892  VmaPool pool);
893 
900 void vmaGetPoolStats(
901  VmaAllocator allocator,
902  VmaPool pool,
903  VmaPoolStats* pPoolStats);
904 
911 void vmaMakePoolAllocationsLost(
912  VmaAllocator allocator,
913  VmaPool pool,
914  size_t* pLostAllocationCount);
915 
916 VK_DEFINE_HANDLE(VmaAllocation)
917 
918 
920 typedef struct VmaAllocationInfo {
925  uint32_t memoryType;
934  VkDeviceMemory deviceMemory;
939  VkDeviceSize offset;
944  VkDeviceSize size;
950  void* pMappedData;
955  void* pUserData;
956 } VmaAllocationInfo;
957 
968 VkResult vmaAllocateMemory(
969  VmaAllocator allocator,
970  const VkMemoryRequirements* pVkMemoryRequirements,
971  const VmaAllocationCreateInfo* pCreateInfo,
972  VmaAllocation* pAllocation,
973  VmaAllocationInfo* pAllocationInfo);
974 
981 VkResult vmaAllocateMemoryForBuffer(
982  VmaAllocator allocator,
983  VkBuffer buffer,
984  const VmaAllocationCreateInfo* pCreateInfo,
985  VmaAllocation* pAllocation,
986  VmaAllocationInfo* pAllocationInfo);
987 
989 VkResult vmaAllocateMemoryForImage(
990  VmaAllocator allocator,
991  VkImage image,
992  const VmaAllocationCreateInfo* pCreateInfo,
993  VmaAllocation* pAllocation,
994  VmaAllocationInfo* pAllocationInfo);
995 
997 void vmaFreeMemory(
998  VmaAllocator allocator,
999  VmaAllocation allocation);
1000 
1002 void vmaGetAllocationInfo(
1003  VmaAllocator allocator,
1004  VmaAllocation allocation,
1005  VmaAllocationInfo* pAllocationInfo);
1006 
1008 void vmaSetAllocationUserData(
1009  VmaAllocator allocator,
1010  VmaAllocation allocation,
1011  void* pUserData);
1012 
1023 void vmaCreateLostAllocation(
1024  VmaAllocator allocator,
1025  VmaAllocation* pAllocation);
1026 
1035 VkResult vmaMapMemory(
1036  VmaAllocator allocator,
1037  VmaAllocation allocation,
1038  void** ppData);
1039 
1040 void vmaUnmapMemory(
1041  VmaAllocator allocator,
1042  VmaAllocation allocation);
1043 
1065 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
1066 
1074 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
1075 
1077 typedef struct VmaDefragmentationInfo {
1082  VkDeviceSize maxBytesToMove;
1087  uint32_t maxAllocationsToMove;
1088 } VmaDefragmentationInfo;
1089 
1091 typedef struct VmaDefragmentationStats {
1093  VkDeviceSize bytesMoved;
1095  VkDeviceSize bytesFreed;
1097  uint32_t allocationsMoved;
1099  uint32_t deviceMemoryBlocksFreed;
1100 } VmaDefragmentationStats;
1101 
1172 VkResult vmaDefragment(
1173  VmaAllocator allocator,
1174  VmaAllocation* pAllocations,
1175  size_t allocationCount,
1176  VkBool32* pAllocationsChanged,
1177  const VmaDefragmentationInfo *pDefragmentationInfo,
1178  VmaDefragmentationStats* pDefragmentationStats);
1179 
1182 
1205 VkResult vmaCreateBuffer(
1206  VmaAllocator allocator,
1207  const VkBufferCreateInfo* pBufferCreateInfo,
1208  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1209  VkBuffer* pBuffer,
1210  VmaAllocation* pAllocation,
1211  VmaAllocationInfo* pAllocationInfo);
1212 
1221 void vmaDestroyBuffer(
1222  VmaAllocator allocator,
1223  VkBuffer buffer,
1224  VmaAllocation allocation);
1225 
1227 VkResult vmaCreateImage(
1228  VmaAllocator allocator,
1229  const VkImageCreateInfo* pImageCreateInfo,
1230  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1231  VkImage* pImage,
1232  VmaAllocation* pAllocation,
1233  VmaAllocationInfo* pAllocationInfo);
1234 
1243 void vmaDestroyImage(
1244  VmaAllocator allocator,
1245  VkImage image,
1246  VmaAllocation allocation);
1247 
1250 #ifdef __cplusplus
1251 }
1252 #endif
1253 
1254 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1255 
1256 // For Visual Studio IntelliSense.
1257 #ifdef __INTELLISENSE__
1258 #define VMA_IMPLEMENTATION
1259 #endif
1260 
1261 #ifdef VMA_IMPLEMENTATION
1262 #undef VMA_IMPLEMENTATION
1263 
1264 #include <cstdint>
1265 #include <cstdlib>
1266 #include <cstring>
1267 
1268 /*******************************************************************************
1269 CONFIGURATION SECTION
1270 
1271 Define some of these macros before each #include of this header or change them
1272 here if you need other then default behavior depending on your environment.
1273 */
1274 
1275 /*
1276 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1277 internally, like:
1278 
1279  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1280 
1281 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1282 VmaAllocatorCreateInfo::pVulkanFunctions.
1283 */
1284 #ifndef VMA_STATIC_VULKAN_FUNCTIONS
1285 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1286 #endif
1287 
1288 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1289 //#define VMA_USE_STL_CONTAINERS 1
1290 
1291 /* Set this macro to 1 to make the library including and using STL containers:
1292 std::pair, std::vector, std::list, std::unordered_map.
1293 
1294 Set it to 0 or undefined to make the library using its own implementation of
1295 the containers.
1296 */
1297 #if VMA_USE_STL_CONTAINERS
1298  #define VMA_USE_STL_VECTOR 1
1299  #define VMA_USE_STL_UNORDERED_MAP 1
1300  #define VMA_USE_STL_LIST 1
1301 #endif
1302 
1303 #if VMA_USE_STL_VECTOR
1304  #include <vector>
1305 #endif
1306 
1307 #if VMA_USE_STL_UNORDERED_MAP
1308  #include <unordered_map>
1309 #endif
1310 
1311 #if VMA_USE_STL_LIST
1312  #include <list>
1313 #endif
1314 
1315 /*
1316 Following headers are used in this CONFIGURATION section only, so feel free to
1317 remove them if not needed.
1318 */
1319 #include <cassert> // for assert
1320 #include <algorithm> // for min, max
1321 #include <mutex> // for std::mutex
1322 #include <atomic> // for std::atomic
1323 
1324 #if !defined(_WIN32)
1325  #include <malloc.h> // for aligned_alloc()
1326 #endif
1327 
1328 // Normal assert to check for programmer's errors, especially in Debug configuration.
1329 #ifndef VMA_ASSERT
1330  #ifdef _DEBUG
1331  #define VMA_ASSERT(expr) assert(expr)
1332  #else
1333  #define VMA_ASSERT(expr)
1334  #endif
1335 #endif
1336 
1337 // Assert that will be called very often, like inside data structures e.g. operator[].
1338 // Making it non-empty can make program slow.
1339 #ifndef VMA_HEAVY_ASSERT
1340  #ifdef _DEBUG
1341  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1342  #else
1343  #define VMA_HEAVY_ASSERT(expr)
1344  #endif
1345 #endif
1346 
1347 #ifndef VMA_NULL
1348  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1349  #define VMA_NULL nullptr
1350 #endif
1351 
1352 #ifndef VMA_ALIGN_OF
1353  #define VMA_ALIGN_OF(type) (__alignof(type))
1354 #endif
1355 
1356 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1357  #if defined(_WIN32)
1358  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1359  #else
1360  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1361  #endif
1362 #endif
1363 
1364 #ifndef VMA_SYSTEM_FREE
1365  #if defined(_WIN32)
1366  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1367  #else
1368  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1369  #endif
1370 #endif
1371 
1372 #ifndef VMA_MIN
1373  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1374 #endif
1375 
1376 #ifndef VMA_MAX
1377  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1378 #endif
1379 
1380 #ifndef VMA_SWAP
1381  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1382 #endif
1383 
1384 #ifndef VMA_SORT
1385  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1386 #endif
1387 
1388 #ifndef VMA_DEBUG_LOG
1389  #define VMA_DEBUG_LOG(format, ...)
1390  /*
1391  #define VMA_DEBUG_LOG(format, ...) do { \
1392  printf(format, __VA_ARGS__); \
1393  printf("\n"); \
1394  } while(false)
1395  */
1396 #endif
1397 
1398 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1399 #if VMA_STATS_STRING_ENABLED
1400  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1401  {
1402  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1403  }
1404  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1405  {
1406  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1407  }
1408  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1409  {
1410  snprintf(outStr, strLen, "%p", ptr);
1411  }
1412 #endif
1413 
1414 #ifndef VMA_MUTEX
1415  class VmaMutex
1416  {
1417  public:
1418  VmaMutex() { }
1419  ~VmaMutex() { }
1420  void Lock() { m_Mutex.lock(); }
1421  void Unlock() { m_Mutex.unlock(); }
1422  private:
1423  std::mutex m_Mutex;
1424  };
1425  #define VMA_MUTEX VmaMutex
1426 #endif
1427 
1428 /*
1429 If providing your own implementation, you need to implement a subset of std::atomic:
1430 
1431 - Constructor(uint32_t desired)
1432 - uint32_t load() const
1433 - void store(uint32_t desired)
1434 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1435 */
1436 #ifndef VMA_ATOMIC_UINT32
1437  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1438 #endif
1439 
1440 #ifndef VMA_BEST_FIT
1441 
1453  #define VMA_BEST_FIT (1)
1454 #endif
1455 
1456 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
1457 
1461  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
1462 #endif
1463 
1464 #ifndef VMA_DEBUG_ALIGNMENT
1465 
1469  #define VMA_DEBUG_ALIGNMENT (1)
1470 #endif
1471 
1472 #ifndef VMA_DEBUG_MARGIN
1473 
1477  #define VMA_DEBUG_MARGIN (0)
1478 #endif
1479 
1480 #ifndef VMA_DEBUG_GLOBAL_MUTEX
1481 
1485  #define VMA_DEBUG_GLOBAL_MUTEX (0)
1486 #endif
1487 
1488 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
1489 
1493  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
1494 #endif
1495 
1496 #ifndef VMA_SMALL_HEAP_MAX_SIZE
1497  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
1499 #endif
1500 
1501 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
1502  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
1504 #endif
1505 
1506 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
1507  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
1509 #endif
1510 
1511 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
1512 
1513 /*******************************************************************************
1514 END OF CONFIGURATION
1515 */
1516 
1517 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
1518  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
1519 
1520 // Returns number of bits set to 1 in (v).
1521 static inline uint32_t CountBitsSet(uint32_t v)
1522 {
1523  uint32_t c = v - ((v >> 1) & 0x55555555);
1524  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
1525  c = ((c >> 4) + c) & 0x0F0F0F0F;
1526  c = ((c >> 8) + c) & 0x00FF00FF;
1527  c = ((c >> 16) + c) & 0x0000FFFF;
1528  return c;
1529 }
1530 
1531 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
1532 // Use types like uint32_t, uint64_t as T.
1533 template <typename T>
1534 static inline T VmaAlignUp(T val, T align)
1535 {
1536  return (val + align - 1) / align * align;
1537 }
1538 
1539 // Division with mathematical rounding to nearest number.
1540 template <typename T>
1541 inline T VmaRoundDiv(T x, T y)
1542 {
1543  return (x + (y / (T)2)) / y;
1544 }
1545 
1546 #ifndef VMA_SORT
1547 
1548 template<typename Iterator, typename Compare>
1549 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
1550 {
1551  Iterator centerValue = end; --centerValue;
1552  Iterator insertIndex = beg;
1553  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
1554  {
1555  if(cmp(*memTypeIndex, *centerValue))
1556  {
1557  if(insertIndex != memTypeIndex)
1558  {
1559  VMA_SWAP(*memTypeIndex, *insertIndex);
1560  }
1561  ++insertIndex;
1562  }
1563  }
1564  if(insertIndex != centerValue)
1565  {
1566  VMA_SWAP(*insertIndex, *centerValue);
1567  }
1568  return insertIndex;
1569 }
1570 
1571 template<typename Iterator, typename Compare>
1572 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
1573 {
1574  if(beg < end)
1575  {
1576  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
1577  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
1578  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
1579  }
1580 }
1581 
1582 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
1583 
1584 #endif // #ifndef VMA_SORT
1585 
1586 /*
1587 Returns true if two memory blocks occupy overlapping pages.
1588 ResourceA must be in less memory offset than ResourceB.
1589 
1590 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1591 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1592 */
1593 static inline bool VmaBlocksOnSamePage(
1594  VkDeviceSize resourceAOffset,
1595  VkDeviceSize resourceASize,
1596  VkDeviceSize resourceBOffset,
1597  VkDeviceSize pageSize)
1598 {
1599  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1600  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1601  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1602  VkDeviceSize resourceBStart = resourceBOffset;
1603  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1604  return resourceAEndPage == resourceBStartPage;
1605 }
1606 
1607 enum VmaSuballocationType
1608 {
1609  VMA_SUBALLOCATION_TYPE_FREE = 0,
1610  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1611  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1612  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1613  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1614  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1615  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1616 };
1617 
1618 /*
1619 Returns true if given suballocation types could conflict and must respect
1620 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1621 or linear image and another one is optimal image. If type is unknown, behave
1622 conservatively.
1623 */
1624 static inline bool VmaIsBufferImageGranularityConflict(
1625  VmaSuballocationType suballocType1,
1626  VmaSuballocationType suballocType2)
1627 {
1628  if(suballocType1 > suballocType2)
1629  {
1630  VMA_SWAP(suballocType1, suballocType2);
1631  }
1632 
1633  switch(suballocType1)
1634  {
1635  case VMA_SUBALLOCATION_TYPE_FREE:
1636  return false;
1637  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1638  return true;
1639  case VMA_SUBALLOCATION_TYPE_BUFFER:
1640  return
1641  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1642  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1643  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1644  return
1645  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1646  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1647  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1648  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1649  return
1650  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1651  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1652  return false;
1653  default:
1654  VMA_ASSERT(0);
1655  return true;
1656  }
1657 }
1658 
1659 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1660 struct VmaMutexLock
1661 {
1662 public:
1663  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1664  m_pMutex(useMutex ? &mutex : VMA_NULL)
1665  {
1666  if(m_pMutex)
1667  {
1668  m_pMutex->Lock();
1669  }
1670  }
1671 
1672  ~VmaMutexLock()
1673  {
1674  if(m_pMutex)
1675  {
1676  m_pMutex->Unlock();
1677  }
1678  }
1679 
1680 private:
1681  VMA_MUTEX* m_pMutex;
1682 };
1683 
1684 #if VMA_DEBUG_GLOBAL_MUTEX
1685  static VMA_MUTEX gDebugGlobalMutex;
1686  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
1687 #else
1688  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1689 #endif
1690 
1691 // Minimum size of a free suballocation to register it in the free suballocation collection.
1692 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1693 
1694 /*
1695 Performs binary search and returns iterator to first element that is greater or
1696 equal to (key), according to comparison (cmp).
1697 
1698 Cmp should return true if first argument is less than second argument.
1699 
1700 Returned value is the found element, if present in the collection or place where
1701 new element with value (key) should be inserted.
1702 */
1703 template <typename IterT, typename KeyT, typename CmpT>
1704 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1705 {
1706  size_t down = 0, up = (end - beg);
1707  while(down < up)
1708  {
1709  const size_t mid = (down + up) / 2;
1710  if(cmp(*(beg+mid), key))
1711  {
1712  down = mid + 1;
1713  }
1714  else
1715  {
1716  up = mid;
1717  }
1718  }
1719  return beg + down;
1720 }
1721 
1723 // Memory allocation
1724 
1725 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1726 {
1727  if((pAllocationCallbacks != VMA_NULL) &&
1728  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1729  {
1730  return (*pAllocationCallbacks->pfnAllocation)(
1731  pAllocationCallbacks->pUserData,
1732  size,
1733  alignment,
1734  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1735  }
1736  else
1737  {
1738  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1739  }
1740 }
1741 
1742 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1743 {
1744  if((pAllocationCallbacks != VMA_NULL) &&
1745  (pAllocationCallbacks->pfnFree != VMA_NULL))
1746  {
1747  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1748  }
1749  else
1750  {
1751  VMA_SYSTEM_FREE(ptr);
1752  }
1753 }
1754 
1755 template<typename T>
1756 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1757 {
1758  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1759 }
1760 
1761 template<typename T>
1762 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1763 {
1764  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1765 }
1766 
1767 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1768 
1769 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1770 
1771 template<typename T>
1772 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1773 {
1774  ptr->~T();
1775  VmaFree(pAllocationCallbacks, ptr);
1776 }
1777 
1778 template<typename T>
1779 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1780 {
1781  if(ptr != VMA_NULL)
1782  {
1783  for(size_t i = count; i--; )
1784  {
1785  ptr[i].~T();
1786  }
1787  VmaFree(pAllocationCallbacks, ptr);
1788  }
1789 }
1790 
1791 // STL-compatible allocator.
1792 template<typename T>
1793 class VmaStlAllocator
1794 {
1795 public:
1796  const VkAllocationCallbacks* const m_pCallbacks;
1797  typedef T value_type;
1798 
1799  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1800  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1801 
1802  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1803  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1804 
1805  template<typename U>
1806  bool operator==(const VmaStlAllocator<U>& rhs) const
1807  {
1808  return m_pCallbacks == rhs.m_pCallbacks;
1809  }
1810  template<typename U>
1811  bool operator!=(const VmaStlAllocator<U>& rhs) const
1812  {
1813  return m_pCallbacks != rhs.m_pCallbacks;
1814  }
1815 
1816  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1817 };
1818 
1819 #if VMA_USE_STL_VECTOR
1820 
1821 #define VmaVector std::vector
1822 
1823 template<typename T, typename allocatorT>
1824 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1825 {
1826  vec.insert(vec.begin() + index, item);
1827 }
1828 
1829 template<typename T, typename allocatorT>
1830 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1831 {
1832  vec.erase(vec.begin() + index);
1833 }
1834 
1835 #else // #if VMA_USE_STL_VECTOR
1836 
1837 /* Class with interface compatible with subset of std::vector.
1838 T must be POD because constructors and destructors are not called and memcpy is
1839 used for these objects. */
1840 template<typename T, typename AllocatorT>
1841 class VmaVector
1842 {
1843 public:
1844  typedef T value_type;
1845 
1846  VmaVector(const AllocatorT& allocator) :
1847  m_Allocator(allocator),
1848  m_pArray(VMA_NULL),
1849  m_Count(0),
1850  m_Capacity(0)
1851  {
1852  }
1853 
1854  VmaVector(size_t count, const AllocatorT& allocator) :
1855  m_Allocator(allocator),
1856  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
1857  m_Count(count),
1858  m_Capacity(count)
1859  {
1860  }
1861 
1862  VmaVector(const VmaVector<T, AllocatorT>& src) :
1863  m_Allocator(src.m_Allocator),
1864  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
1865  m_Count(src.m_Count),
1866  m_Capacity(src.m_Count)
1867  {
1868  if(m_Count != 0)
1869  {
1870  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1871  }
1872  }
1873 
1874  ~VmaVector()
1875  {
1876  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1877  }
1878 
1879  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1880  {
1881  if(&rhs != this)
1882  {
1883  resize(rhs.m_Count);
1884  if(m_Count != 0)
1885  {
1886  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1887  }
1888  }
1889  return *this;
1890  }
1891 
1892  bool empty() const { return m_Count == 0; }
1893  size_t size() const { return m_Count; }
1894  T* data() { return m_pArray; }
1895  const T* data() const { return m_pArray; }
1896 
1897  T& operator[](size_t index)
1898  {
1899  VMA_HEAVY_ASSERT(index < m_Count);
1900  return m_pArray[index];
1901  }
1902  const T& operator[](size_t index) const
1903  {
1904  VMA_HEAVY_ASSERT(index < m_Count);
1905  return m_pArray[index];
1906  }
1907 
1908  T& front()
1909  {
1910  VMA_HEAVY_ASSERT(m_Count > 0);
1911  return m_pArray[0];
1912  }
1913  const T& front() const
1914  {
1915  VMA_HEAVY_ASSERT(m_Count > 0);
1916  return m_pArray[0];
1917  }
1918  T& back()
1919  {
1920  VMA_HEAVY_ASSERT(m_Count > 0);
1921  return m_pArray[m_Count - 1];
1922  }
1923  const T& back() const
1924  {
1925  VMA_HEAVY_ASSERT(m_Count > 0);
1926  return m_pArray[m_Count - 1];
1927  }
1928 
1929  void reserve(size_t newCapacity, bool freeMemory = false)
1930  {
1931  newCapacity = VMA_MAX(newCapacity, m_Count);
1932 
1933  if((newCapacity < m_Capacity) && !freeMemory)
1934  {
1935  newCapacity = m_Capacity;
1936  }
1937 
1938  if(newCapacity != m_Capacity)
1939  {
1940  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1941  if(m_Count != 0)
1942  {
1943  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1944  }
1945  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1946  m_Capacity = newCapacity;
1947  m_pArray = newArray;
1948  }
1949  }
1950 
1951  void resize(size_t newCount, bool freeMemory = false)
1952  {
1953  size_t newCapacity = m_Capacity;
1954  if(newCount > m_Capacity)
1955  {
1956  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1957  }
1958  else if(freeMemory)
1959  {
1960  newCapacity = newCount;
1961  }
1962 
1963  if(newCapacity != m_Capacity)
1964  {
1965  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1966  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1967  if(elementsToCopy != 0)
1968  {
1969  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1970  }
1971  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1972  m_Capacity = newCapacity;
1973  m_pArray = newArray;
1974  }
1975 
1976  m_Count = newCount;
1977  }
1978 
1979  void clear(bool freeMemory = false)
1980  {
1981  resize(0, freeMemory);
1982  }
1983 
1984  void insert(size_t index, const T& src)
1985  {
1986  VMA_HEAVY_ASSERT(index <= m_Count);
1987  const size_t oldCount = size();
1988  resize(oldCount + 1);
1989  if(index < oldCount)
1990  {
1991  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1992  }
1993  m_pArray[index] = src;
1994  }
1995 
1996  void remove(size_t index)
1997  {
1998  VMA_HEAVY_ASSERT(index < m_Count);
1999  const size_t oldCount = size();
2000  if(index < oldCount - 1)
2001  {
2002  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2003  }
2004  resize(oldCount - 1);
2005  }
2006 
2007  void push_back(const T& src)
2008  {
2009  const size_t newIndex = size();
2010  resize(newIndex + 1);
2011  m_pArray[newIndex] = src;
2012  }
2013 
2014  void pop_back()
2015  {
2016  VMA_HEAVY_ASSERT(m_Count > 0);
2017  resize(size() - 1);
2018  }
2019 
2020  void push_front(const T& src)
2021  {
2022  insert(0, src);
2023  }
2024 
2025  void pop_front()
2026  {
2027  VMA_HEAVY_ASSERT(m_Count > 0);
2028  remove(0);
2029  }
2030 
2031  typedef T* iterator;
2032 
2033  iterator begin() { return m_pArray; }
2034  iterator end() { return m_pArray + m_Count; }
2035 
2036 private:
2037  AllocatorT m_Allocator;
2038  T* m_pArray;
2039  size_t m_Count;
2040  size_t m_Capacity;
2041 };
2042 
2043 template<typename T, typename allocatorT>
2044 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2045 {
2046  vec.insert(index, item);
2047 }
2048 
2049 template<typename T, typename allocatorT>
2050 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2051 {
2052  vec.remove(index);
2053 }
2054 
2055 #endif // #if VMA_USE_STL_VECTOR
2056 
2057 template<typename CmpLess, typename VectorT>
2058 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2059 {
2060  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2061  vector.data(),
2062  vector.data() + vector.size(),
2063  value,
2064  CmpLess()) - vector.data();
2065  VmaVectorInsert(vector, indexToInsert, value);
2066  return indexToInsert;
2067 }
2068 
2069 template<typename CmpLess, typename VectorT>
2070 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2071 {
2072  CmpLess comparator;
2073  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2074  vector.begin(),
2075  vector.end(),
2076  value,
2077  comparator);
2078  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2079  {
2080  size_t indexToRemove = it - vector.begin();
2081  VmaVectorRemove(vector, indexToRemove);
2082  return true;
2083  }
2084  return false;
2085 }
2086 
2087 template<typename CmpLess, typename VectorT>
2088 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2089 {
2090  CmpLess comparator;
2091  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2092  vector.data(),
2093  vector.data() + vector.size(),
2094  value,
2095  comparator);
2096  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2097  {
2098  return it - vector.begin();
2099  }
2100  else
2101  {
2102  return vector.size();
2103  }
2104 }
2105 
2107 // class VmaPoolAllocator
2108 
2109 /*
2110 Allocator for objects of type T using a list of arrays (pools) to speed up
2111 allocation. Number of elements that can be allocated is not bounded because
2112 allocator can create multiple blocks.
2113 */
2114 template<typename T>
2115 class VmaPoolAllocator
2116 {
2117 public:
2118  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2119  ~VmaPoolAllocator();
2120  void Clear();
2121  T* Alloc();
2122  void Free(T* ptr);
2123 
2124 private:
2125  union Item
2126  {
2127  uint32_t NextFreeIndex;
2128  T Value;
2129  };
2130 
2131  struct ItemBlock
2132  {
2133  Item* pItems;
2134  uint32_t FirstFreeIndex;
2135  };
2136 
2137  const VkAllocationCallbacks* m_pAllocationCallbacks;
2138  size_t m_ItemsPerBlock;
2139  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2140 
2141  ItemBlock& CreateNewBlock();
2142 };
2143 
2144 template<typename T>
2145 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2146  m_pAllocationCallbacks(pAllocationCallbacks),
2147  m_ItemsPerBlock(itemsPerBlock),
2148  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2149 {
2150  VMA_ASSERT(itemsPerBlock > 0);
2151 }
2152 
2153 template<typename T>
2154 VmaPoolAllocator<T>::~VmaPoolAllocator()
2155 {
2156  Clear();
2157 }
2158 
2159 template<typename T>
2160 void VmaPoolAllocator<T>::Clear()
2161 {
2162  for(size_t i = m_ItemBlocks.size(); i--; )
2163  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2164  m_ItemBlocks.clear();
2165 }
2166 
2167 template<typename T>
2168 T* VmaPoolAllocator<T>::Alloc()
2169 {
2170  for(size_t i = m_ItemBlocks.size(); i--; )
2171  {
2172  ItemBlock& block = m_ItemBlocks[i];
2173  // This block has some free items: Use first one.
2174  if(block.FirstFreeIndex != UINT32_MAX)
2175  {
2176  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2177  block.FirstFreeIndex = pItem->NextFreeIndex;
2178  return &pItem->Value;
2179  }
2180  }
2181 
2182  // No block has free item: Create new one and use it.
2183  ItemBlock& newBlock = CreateNewBlock();
2184  Item* const pItem = &newBlock.pItems[0];
2185  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2186  return &pItem->Value;
2187 }
2188 
2189 template<typename T>
2190 void VmaPoolAllocator<T>::Free(T* ptr)
2191 {
2192  // Search all memory blocks to find ptr.
2193  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2194  {
2195  ItemBlock& block = m_ItemBlocks[i];
2196 
2197  // Casting to union.
2198  Item* pItemPtr;
2199  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2200 
2201  // Check if pItemPtr is in address range of this block.
2202  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2203  {
2204  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2205  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2206  block.FirstFreeIndex = index;
2207  return;
2208  }
2209  }
2210  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2211 }
2212 
2213 template<typename T>
2214 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2215 {
2216  ItemBlock newBlock = {
2217  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2218 
2219  m_ItemBlocks.push_back(newBlock);
2220 
2221  // Setup singly-linked list of all free items in this block.
2222  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2223  newBlock.pItems[i].NextFreeIndex = i + 1;
2224  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2225  return m_ItemBlocks.back();
2226 }
2227 
2229 // class VmaRawList, VmaList
2230 
2231 #if VMA_USE_STL_LIST
2232 
2233 #define VmaList std::list
2234 
2235 #else // #if VMA_USE_STL_LIST
2236 
2237 template<typename T>
2238 struct VmaListItem
2239 {
2240  VmaListItem* pPrev;
2241  VmaListItem* pNext;
2242  T Value;
2243 };
2244 
2245 // Doubly linked list.
2246 template<typename T>
2247 class VmaRawList
2248 {
2249 public:
2250  typedef VmaListItem<T> ItemType;
2251 
2252  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2253  ~VmaRawList();
2254  void Clear();
2255 
2256  size_t GetCount() const { return m_Count; }
2257  bool IsEmpty() const { return m_Count == 0; }
2258 
2259  ItemType* Front() { return m_pFront; }
2260  const ItemType* Front() const { return m_pFront; }
2261  ItemType* Back() { return m_pBack; }
2262  const ItemType* Back() const { return m_pBack; }
2263 
2264  ItemType* PushBack();
2265  ItemType* PushFront();
2266  ItemType* PushBack(const T& value);
2267  ItemType* PushFront(const T& value);
2268  void PopBack();
2269  void PopFront();
2270 
2271  // Item can be null - it means PushBack.
2272  ItemType* InsertBefore(ItemType* pItem);
2273  // Item can be null - it means PushFront.
2274  ItemType* InsertAfter(ItemType* pItem);
2275 
2276  ItemType* InsertBefore(ItemType* pItem, const T& value);
2277  ItemType* InsertAfter(ItemType* pItem, const T& value);
2278 
2279  void Remove(ItemType* pItem);
2280 
2281 private:
2282  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2283  VmaPoolAllocator<ItemType> m_ItemAllocator;
2284  ItemType* m_pFront;
2285  ItemType* m_pBack;
2286  size_t m_Count;
2287 
2288  // Declared not defined, to block copy constructor and assignment operator.
2289  VmaRawList(const VmaRawList<T>& src);
2290  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2291 };
2292 
2293 template<typename T>
2294 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2295  m_pAllocationCallbacks(pAllocationCallbacks),
2296  m_ItemAllocator(pAllocationCallbacks, 128),
2297  m_pFront(VMA_NULL),
2298  m_pBack(VMA_NULL),
2299  m_Count(0)
2300 {
2301 }
2302 
2303 template<typename T>
2304 VmaRawList<T>::~VmaRawList()
2305 {
2306  // Intentionally not calling Clear, because that would be unnecessary
2307  // computations to return all items to m_ItemAllocator as free.
2308 }
2309 
2310 template<typename T>
2311 void VmaRawList<T>::Clear()
2312 {
2313  if(IsEmpty() == false)
2314  {
2315  ItemType* pItem = m_pBack;
2316  while(pItem != VMA_NULL)
2317  {
2318  ItemType* const pPrevItem = pItem->pPrev;
2319  m_ItemAllocator.Free(pItem);
2320  pItem = pPrevItem;
2321  }
2322  m_pFront = VMA_NULL;
2323  m_pBack = VMA_NULL;
2324  m_Count = 0;
2325  }
2326 }
2327 
2328 template<typename T>
2329 VmaListItem<T>* VmaRawList<T>::PushBack()
2330 {
2331  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2332  pNewItem->pNext = VMA_NULL;
2333  if(IsEmpty())
2334  {
2335  pNewItem->pPrev = VMA_NULL;
2336  m_pFront = pNewItem;
2337  m_pBack = pNewItem;
2338  m_Count = 1;
2339  }
2340  else
2341  {
2342  pNewItem->pPrev = m_pBack;
2343  m_pBack->pNext = pNewItem;
2344  m_pBack = pNewItem;
2345  ++m_Count;
2346  }
2347  return pNewItem;
2348 }
2349 
2350 template<typename T>
2351 VmaListItem<T>* VmaRawList<T>::PushFront()
2352 {
2353  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2354  pNewItem->pPrev = VMA_NULL;
2355  if(IsEmpty())
2356  {
2357  pNewItem->pNext = VMA_NULL;
2358  m_pFront = pNewItem;
2359  m_pBack = pNewItem;
2360  m_Count = 1;
2361  }
2362  else
2363  {
2364  pNewItem->pNext = m_pFront;
2365  m_pFront->pPrev = pNewItem;
2366  m_pFront = pNewItem;
2367  ++m_Count;
2368  }
2369  return pNewItem;
2370 }
2371 
2372 template<typename T>
2373 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2374 {
2375  ItemType* const pNewItem = PushBack();
2376  pNewItem->Value = value;
2377  return pNewItem;
2378 }
2379 
2380 template<typename T>
2381 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2382 {
2383  ItemType* const pNewItem = PushFront();
2384  pNewItem->Value = value;
2385  return pNewItem;
2386 }
2387 
2388 template<typename T>
2389 void VmaRawList<T>::PopBack()
2390 {
2391  VMA_HEAVY_ASSERT(m_Count > 0);
2392  ItemType* const pBackItem = m_pBack;
2393  ItemType* const pPrevItem = pBackItem->pPrev;
2394  if(pPrevItem != VMA_NULL)
2395  {
2396  pPrevItem->pNext = VMA_NULL;
2397  }
2398  m_pBack = pPrevItem;
2399  m_ItemAllocator.Free(pBackItem);
2400  --m_Count;
2401 }
2402 
2403 template<typename T>
2404 void VmaRawList<T>::PopFront()
2405 {
2406  VMA_HEAVY_ASSERT(m_Count > 0);
2407  ItemType* const pFrontItem = m_pFront;
2408  ItemType* const pNextItem = pFrontItem->pNext;
2409  if(pNextItem != VMA_NULL)
2410  {
2411  pNextItem->pPrev = VMA_NULL;
2412  }
2413  m_pFront = pNextItem;
2414  m_ItemAllocator.Free(pFrontItem);
2415  --m_Count;
2416 }
2417 
2418 template<typename T>
2419 void VmaRawList<T>::Remove(ItemType* pItem)
2420 {
2421  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2422  VMA_HEAVY_ASSERT(m_Count > 0);
2423 
2424  if(pItem->pPrev != VMA_NULL)
2425  {
2426  pItem->pPrev->pNext = pItem->pNext;
2427  }
2428  else
2429  {
2430  VMA_HEAVY_ASSERT(m_pFront == pItem);
2431  m_pFront = pItem->pNext;
2432  }
2433 
2434  if(pItem->pNext != VMA_NULL)
2435  {
2436  pItem->pNext->pPrev = pItem->pPrev;
2437  }
2438  else
2439  {
2440  VMA_HEAVY_ASSERT(m_pBack == pItem);
2441  m_pBack = pItem->pPrev;
2442  }
2443 
2444  m_ItemAllocator.Free(pItem);
2445  --m_Count;
2446 }
2447 
2448 template<typename T>
2449 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2450 {
2451  if(pItem != VMA_NULL)
2452  {
2453  ItemType* const prevItem = pItem->pPrev;
2454  ItemType* const newItem = m_ItemAllocator.Alloc();
2455  newItem->pPrev = prevItem;
2456  newItem->pNext = pItem;
2457  pItem->pPrev = newItem;
2458  if(prevItem != VMA_NULL)
2459  {
2460  prevItem->pNext = newItem;
2461  }
2462  else
2463  {
2464  VMA_HEAVY_ASSERT(m_pFront == pItem);
2465  m_pFront = newItem;
2466  }
2467  ++m_Count;
2468  return newItem;
2469  }
2470  else
2471  return PushBack();
2472 }
2473 
2474 template<typename T>
2475 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2476 {
2477  if(pItem != VMA_NULL)
2478  {
2479  ItemType* const nextItem = pItem->pNext;
2480  ItemType* const newItem = m_ItemAllocator.Alloc();
2481  newItem->pNext = nextItem;
2482  newItem->pPrev = pItem;
2483  pItem->pNext = newItem;
2484  if(nextItem != VMA_NULL)
2485  {
2486  nextItem->pPrev = newItem;
2487  }
2488  else
2489  {
2490  VMA_HEAVY_ASSERT(m_pBack == pItem);
2491  m_pBack = newItem;
2492  }
2493  ++m_Count;
2494  return newItem;
2495  }
2496  else
2497  return PushFront();
2498 }
2499 
2500 template<typename T>
2501 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
2502 {
2503  ItemType* const newItem = InsertBefore(pItem);
2504  newItem->Value = value;
2505  return newItem;
2506 }
2507 
2508 template<typename T>
2509 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
2510 {
2511  ItemType* const newItem = InsertAfter(pItem);
2512  newItem->Value = value;
2513  return newItem;
2514 }
2515 
2516 template<typename T, typename AllocatorT>
2517 class VmaList
2518 {
2519 public:
2520  class iterator
2521  {
2522  public:
2523  iterator() :
2524  m_pList(VMA_NULL),
2525  m_pItem(VMA_NULL)
2526  {
2527  }
2528 
2529  T& operator*() const
2530  {
2531  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2532  return m_pItem->Value;
2533  }
2534  T* operator->() const
2535  {
2536  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2537  return &m_pItem->Value;
2538  }
2539 
2540  iterator& operator++()
2541  {
2542  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2543  m_pItem = m_pItem->pNext;
2544  return *this;
2545  }
2546  iterator& operator--()
2547  {
2548  if(m_pItem != VMA_NULL)
2549  {
2550  m_pItem = m_pItem->pPrev;
2551  }
2552  else
2553  {
2554  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
2555  m_pItem = m_pList->Back();
2556  }
2557  return *this;
2558  }
2559 
2560  iterator operator++(int)
2561  {
2562  iterator result = *this;
2563  ++*this;
2564  return result;
2565  }
2566  iterator operator--(int)
2567  {
2568  iterator result = *this;
2569  --*this;
2570  return result;
2571  }
2572 
2573  bool operator==(const iterator& rhs) const
2574  {
2575  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2576  return m_pItem == rhs.m_pItem;
2577  }
2578  bool operator!=(const iterator& rhs) const
2579  {
2580  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2581  return m_pItem != rhs.m_pItem;
2582  }
2583 
2584  private:
2585  VmaRawList<T>* m_pList;
2586  VmaListItem<T>* m_pItem;
2587 
2588  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
2589  m_pList(pList),
2590  m_pItem(pItem)
2591  {
2592  }
2593 
2594  friend class VmaList<T, AllocatorT>;
2595  };
2596 
2597  class const_iterator
2598  {
2599  public:
2600  const_iterator() :
2601  m_pList(VMA_NULL),
2602  m_pItem(VMA_NULL)
2603  {
2604  }
2605 
2606  const_iterator(const iterator& src) :
2607  m_pList(src.m_pList),
2608  m_pItem(src.m_pItem)
2609  {
2610  }
2611 
2612  const T& operator*() const
2613  {
2614  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2615  return m_pItem->Value;
2616  }
2617  const T* operator->() const
2618  {
2619  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2620  return &m_pItem->Value;
2621  }
2622 
2623  const_iterator& operator++()
2624  {
2625  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2626  m_pItem = m_pItem->pNext;
2627  return *this;
2628  }
2629  const_iterator& operator--()
2630  {
2631  if(m_pItem != VMA_NULL)
2632  {
2633  m_pItem = m_pItem->pPrev;
2634  }
2635  else
2636  {
2637  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2638  m_pItem = m_pList->Back();
2639  }
2640  return *this;
2641  }
2642 
2643  const_iterator operator++(int)
2644  {
2645  const_iterator result = *this;
2646  ++*this;
2647  return result;
2648  }
2649  const_iterator operator--(int)
2650  {
2651  const_iterator result = *this;
2652  --*this;
2653  return result;
2654  }
2655 
2656  bool operator==(const const_iterator& rhs) const
2657  {
2658  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2659  return m_pItem == rhs.m_pItem;
2660  }
2661  bool operator!=(const const_iterator& rhs) const
2662  {
2663  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2664  return m_pItem != rhs.m_pItem;
2665  }
2666 
2667  private:
2668  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2669  m_pList(pList),
2670  m_pItem(pItem)
2671  {
2672  }
2673 
2674  const VmaRawList<T>* m_pList;
2675  const VmaListItem<T>* m_pItem;
2676 
2677  friend class VmaList<T, AllocatorT>;
2678  };
2679 
2680  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2681 
2682  bool empty() const { return m_RawList.IsEmpty(); }
2683  size_t size() const { return m_RawList.GetCount(); }
2684 
2685  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2686  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2687 
2688  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2689  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2690 
2691  void clear() { m_RawList.Clear(); }
2692  void push_back(const T& value) { m_RawList.PushBack(value); }
2693  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2694  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2695 
2696 private:
2697  VmaRawList<T> m_RawList;
2698 };
2699 
2700 #endif // #if VMA_USE_STL_LIST
2701 
2703 // class VmaMap
2704 
2705 // Unused in this version.
2706 #if 0
2707 
2708 #if VMA_USE_STL_UNORDERED_MAP
2709 
2710 #define VmaPair std::pair
2711 
2712 #define VMA_MAP_TYPE(KeyT, ValueT) \
2713  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2714 
2715 #else // #if VMA_USE_STL_UNORDERED_MAP
2716 
2717 template<typename T1, typename T2>
2718 struct VmaPair
2719 {
2720  T1 first;
2721  T2 second;
2722 
2723  VmaPair() : first(), second() { }
2724  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2725 };
2726 
2727 /* Class compatible with subset of interface of std::unordered_map.
2728 KeyT, ValueT must be POD because they will be stored in VmaVector.
2729 */
2730 template<typename KeyT, typename ValueT>
2731 class VmaMap
2732 {
2733 public:
2734  typedef VmaPair<KeyT, ValueT> PairType;
2735  typedef PairType* iterator;
2736 
2737  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2738 
2739  iterator begin() { return m_Vector.begin(); }
2740  iterator end() { return m_Vector.end(); }
2741 
2742  void insert(const PairType& pair);
2743  iterator find(const KeyT& key);
2744  void erase(iterator it);
2745 
2746 private:
2747  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2748 };
2749 
2750 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2751 
2752 template<typename FirstT, typename SecondT>
2753 struct VmaPairFirstLess
2754 {
2755  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2756  {
2757  return lhs.first < rhs.first;
2758  }
2759  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2760  {
2761  return lhs.first < rhsFirst;
2762  }
2763 };
2764 
2765 template<typename KeyT, typename ValueT>
2766 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2767 {
2768  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2769  m_Vector.data(),
2770  m_Vector.data() + m_Vector.size(),
2771  pair,
2772  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2773  VmaVectorInsert(m_Vector, indexToInsert, pair);
2774 }
2775 
2776 template<typename KeyT, typename ValueT>
2777 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2778 {
2779  PairType* it = VmaBinaryFindFirstNotLess(
2780  m_Vector.data(),
2781  m_Vector.data() + m_Vector.size(),
2782  key,
2783  VmaPairFirstLess<KeyT, ValueT>());
2784  if((it != m_Vector.end()) && (it->first == key))
2785  {
2786  return it;
2787  }
2788  else
2789  {
2790  return m_Vector.end();
2791  }
2792 }
2793 
2794 template<typename KeyT, typename ValueT>
2795 void VmaMap<KeyT, ValueT>::erase(iterator it)
2796 {
2797  VmaVectorRemove(m_Vector, it - m_Vector.begin());
2798 }
2799 
2800 #endif // #if VMA_USE_STL_UNORDERED_MAP
2801 
2802 #endif // #if 0
2803 
2805 
2806 class VmaDeviceMemoryBlock;
2807 
2808 enum VMA_BLOCK_VECTOR_TYPE
2809 {
2810  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2811  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2812  VMA_BLOCK_VECTOR_TYPE_COUNT
2813 };
2814 
2815 static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags)
2816 {
2817  return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ?
2818  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2819  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2820 }
2821 
2822 struct VmaAllocation_T
2823 {
2824 public:
2825  enum ALLOCATION_TYPE
2826  {
2827  ALLOCATION_TYPE_NONE,
2828  ALLOCATION_TYPE_BLOCK,
2829  ALLOCATION_TYPE_OWN,
2830  };
2831 
2832  VmaAllocation_T(uint32_t currentFrameIndex) :
2833  m_Alignment(1),
2834  m_Size(0),
2835  m_pUserData(VMA_NULL),
2836  m_Type(ALLOCATION_TYPE_NONE),
2837  m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN),
2838  m_LastUseFrameIndex(currentFrameIndex)
2839  {
2840  }
2841 
2842  void InitBlockAllocation(
2843  VmaPool hPool,
2844  VmaDeviceMemoryBlock* block,
2845  VkDeviceSize offset,
2846  VkDeviceSize alignment,
2847  VkDeviceSize size,
2848  VmaSuballocationType suballocationType,
2849  void* pUserData,
2850  bool canBecomeLost)
2851  {
2852  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2853  VMA_ASSERT(block != VMA_NULL);
2854  m_Type = ALLOCATION_TYPE_BLOCK;
2855  m_Alignment = alignment;
2856  m_Size = size;
2857  m_pUserData = pUserData;
2858  m_SuballocationType = suballocationType;
2859  m_BlockAllocation.m_hPool = hPool;
2860  m_BlockAllocation.m_Block = block;
2861  m_BlockAllocation.m_Offset = offset;
2862  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
2863  }
2864 
2865  void InitLost()
2866  {
2867  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2868  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
2869  m_Type = ALLOCATION_TYPE_BLOCK;
2870  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
2871  m_BlockAllocation.m_Block = VMA_NULL;
2872  m_BlockAllocation.m_Offset = 0;
2873  m_BlockAllocation.m_CanBecomeLost = true;
2874  }
2875 
2876  void ChangeBlockAllocation(
2877  VmaDeviceMemoryBlock* block,
2878  VkDeviceSize offset)
2879  {
2880  VMA_ASSERT(block != VMA_NULL);
2881  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2882  m_BlockAllocation.m_Block = block;
2883  m_BlockAllocation.m_Offset = offset;
2884  }
2885 
2886  void InitOwnAllocation(
2887  uint32_t memoryTypeIndex,
2888  VkDeviceMemory hMemory,
2889  VmaSuballocationType suballocationType,
2890  bool persistentMap,
2891  void* pMappedData,
2892  VkDeviceSize size,
2893  void* pUserData)
2894  {
2895  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2896  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2897  m_Type = ALLOCATION_TYPE_OWN;
2898  m_Alignment = 0;
2899  m_Size = size;
2900  m_pUserData = pUserData;
2901  m_SuballocationType = suballocationType;
2902  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2903  m_OwnAllocation.m_hMemory = hMemory;
2904  m_OwnAllocation.m_PersistentMap = persistentMap;
2905  m_OwnAllocation.m_pMappedData = pMappedData;
2906  }
2907 
2908  ALLOCATION_TYPE GetType() const { return m_Type; }
2909  VkDeviceSize GetAlignment() const { return m_Alignment; }
2910  VkDeviceSize GetSize() const { return m_Size; }
2911  void* GetUserData() const { return m_pUserData; }
2912  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2913  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2914 
2915  VmaDeviceMemoryBlock* GetBlock() const
2916  {
2917  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2918  return m_BlockAllocation.m_Block;
2919  }
2920  VkDeviceSize GetOffset() const;
2921  VkDeviceMemory GetMemory() const;
2922  uint32_t GetMemoryTypeIndex() const;
2923  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2924  void* GetMappedData() const;
2925  bool CanBecomeLost() const;
2926  VmaPool GetPool() const;
2927 
2928  VkResult OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator);
2929  void OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator);
2930 
2931  uint32_t GetLastUseFrameIndex() const
2932  {
2933  return m_LastUseFrameIndex.load();
2934  }
2935  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
2936  {
2937  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
2938  }
2939  /*
2940  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
2941  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
2942  - Else, returns false.
2943 
2944  If hAllocation is already lost, assert - you should not call it then.
2945  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
2946  */
2947  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
2948 
2949  void OwnAllocCalcStatsInfo(VmaStatInfo& outInfo)
2950  {
2951  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2952  outInfo.blockCount = 1;
2953  outInfo.allocationCount = 1;
2954  outInfo.unusedRangeCount = 0;
2955  outInfo.usedBytes = m_Size;
2956  outInfo.unusedBytes = 0;
2957  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
2958  outInfo.unusedRangeSizeMin = UINT64_MAX;
2959  outInfo.unusedRangeSizeMax = 0;
2960  }
2961 
2962 private:
2963  VkDeviceSize m_Alignment;
2964  VkDeviceSize m_Size;
2965  void* m_pUserData;
2966  ALLOCATION_TYPE m_Type;
2967  VmaSuballocationType m_SuballocationType;
2968  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
2969 
2970  // Allocation out of VmaDeviceMemoryBlock.
2971  struct BlockAllocation
2972  {
2973  VmaPool m_hPool; // Null if belongs to general memory.
2974  VmaDeviceMemoryBlock* m_Block;
2975  VkDeviceSize m_Offset;
2976  bool m_CanBecomeLost;
2977  };
2978 
2979  // Allocation for an object that has its own private VkDeviceMemory.
2980  struct OwnAllocation
2981  {
2982  uint32_t m_MemoryTypeIndex;
2983  VkDeviceMemory m_hMemory;
2984  bool m_PersistentMap;
2985  void* m_pMappedData;
2986  };
2987 
2988  union
2989  {
2990  // Allocation out of VmaDeviceMemoryBlock.
2991  BlockAllocation m_BlockAllocation;
2992  // Allocation for an object that has its own private VkDeviceMemory.
2993  OwnAllocation m_OwnAllocation;
2994  };
2995 };
2996 
2997 /*
2998 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
2999 allocated memory block or free.
3000 */
3001 struct VmaSuballocation
3002 {
3003  VkDeviceSize offset;
3004  VkDeviceSize size;
3005  VmaAllocation hAllocation;
3006  VmaSuballocationType type;
3007 };
3008 
3009 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3010 
3011 // Cost of one additional allocation lost, as equivalent in bytes.
3012 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3013 
3014 /*
3015 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3016 
3017 If canMakeOtherLost was false:
3018 - item points to a FREE suballocation.
3019 - itemsToMakeLostCount is 0.
3020 
3021 If canMakeOtherLost was true:
3022 - item points to first of sequence of suballocations, which are either FREE,
3023  or point to VmaAllocations that can become lost.
3024 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3025  the requested allocation to succeed.
3026 */
3027 struct VmaAllocationRequest
3028 {
3029  VkDeviceSize offset;
3030  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3031  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3032  VmaSuballocationList::iterator item;
3033  size_t itemsToMakeLostCount;
3034 
3035  VkDeviceSize CalcCost() const
3036  {
3037  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3038  }
3039 };
3040 
3041 /*
3042 Data structure used for bookkeeping of allocations and unused ranges of memory
3043 in a single VkDeviceMemory block.
3044 */
3045 class VmaBlockMetadata
3046 {
3047 public:
3048  VmaBlockMetadata(VmaAllocator hAllocator);
3049  ~VmaBlockMetadata();
3050  void Init(VkDeviceSize size);
3051 
3052  // Validates all data structures inside this object. If not valid, returns false.
3053  bool Validate() const;
3054  VkDeviceSize GetSize() const { return m_Size; }
3055  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3056  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3057  VkDeviceSize GetUnusedRangeSizeMax() const;
3058  // Returns true if this block is empty - contains only single free suballocation.
3059  bool IsEmpty() const;
3060 
3061  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3062  void AddPoolStats(VmaPoolStats& inoutStats) const;
3063 
3064 #if VMA_STATS_STRING_ENABLED
3065  void PrintDetailedMap(class VmaJsonWriter& json) const;
3066 #endif
3067 
3068  // Creates trivial request for case when block is empty.
3069  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3070 
3071  // Tries to find a place for suballocation with given parameters inside this block.
3072  // If succeeded, fills pAllocationRequest and returns true.
3073  // If failed, returns false.
3074  bool CreateAllocationRequest(
3075  uint32_t currentFrameIndex,
3076  uint32_t frameInUseCount,
3077  VkDeviceSize bufferImageGranularity,
3078  VkDeviceSize allocSize,
3079  VkDeviceSize allocAlignment,
3080  VmaSuballocationType allocType,
3081  bool canMakeOtherLost,
3082  VmaAllocationRequest* pAllocationRequest);
3083 
3084  bool MakeRequestedAllocationsLost(
3085  uint32_t currentFrameIndex,
3086  uint32_t frameInUseCount,
3087  VmaAllocationRequest* pAllocationRequest);
3088 
3089  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3090 
3091  // Makes actual allocation based on request. Request must already be checked and valid.
3092  void Alloc(
3093  const VmaAllocationRequest& request,
3094  VmaSuballocationType type,
3095  VkDeviceSize allocSize,
3096  VmaAllocation hAllocation);
3097 
3098  // Frees suballocation assigned to given memory region.
3099  void Free(const VmaAllocation allocation);
3100 
3101 private:
3102  VkDeviceSize m_Size;
3103  uint32_t m_FreeCount;
3104  VkDeviceSize m_SumFreeSize;
3105  VmaSuballocationList m_Suballocations;
3106  // Suballocations that are free and have size greater than certain threshold.
3107  // Sorted by size, ascending.
3108  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3109 
3110  bool ValidateFreeSuballocationList() const;
3111 
3112  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3113  // If yes, fills pOffset and returns true. If no, returns false.
3114  bool CheckAllocation(
3115  uint32_t currentFrameIndex,
3116  uint32_t frameInUseCount,
3117  VkDeviceSize bufferImageGranularity,
3118  VkDeviceSize allocSize,
3119  VkDeviceSize allocAlignment,
3120  VmaSuballocationType allocType,
3121  VmaSuballocationList::const_iterator suballocItem,
3122  bool canMakeOtherLost,
3123  VkDeviceSize* pOffset,
3124  size_t* itemsToMakeLostCount,
3125  VkDeviceSize* pSumFreeSize,
3126  VkDeviceSize* pSumItemSize) const;
3127  // Given free suballocation, it merges it with following one, which must also be free.
3128  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3129  // Releases given suballocation, making it free.
3130  // Merges it with adjacent free suballocations if applicable.
3131  // Returns iterator to new free suballocation at this place.
3132  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3133  // Given free suballocation, it inserts it into sorted list of
3134  // m_FreeSuballocationsBySize if it's suitable.
3135  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3136  // Given free suballocation, it removes it from sorted list of
3137  // m_FreeSuballocationsBySize if it's suitable.
3138  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3139 };
3140 
3141 /*
3142 Represents a single block of device memory (`VkDeviceMemory`) with all the
3143 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3144 
3145 Thread-safety: This class must be externally synchronized.
3146 */
3147 class VmaDeviceMemoryBlock
3148 {
3149 public:
3150  uint32_t m_MemoryTypeIndex;
3151  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3152  VkDeviceMemory m_hMemory;
3153  bool m_PersistentMap;
3154  void* m_pMappedData;
3155  VmaBlockMetadata m_Metadata;
3156 
3157  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3158 
3159  ~VmaDeviceMemoryBlock()
3160  {
3161  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3162  }
3163 
3164  // Always call after construction.
3165  void Init(
3166  uint32_t newMemoryTypeIndex,
3167  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
3168  VkDeviceMemory newMemory,
3169  VkDeviceSize newSize,
3170  bool persistentMap,
3171  void* pMappedData);
3172  // Always call before destruction.
3173  void Destroy(VmaAllocator allocator);
3174 
3175  // Validates all data structures inside this object. If not valid, returns false.
3176  bool Validate() const;
3177 };
3178 
3179 struct VmaPointerLess
3180 {
3181  bool operator()(const void* lhs, const void* rhs) const
3182  {
3183  return lhs < rhs;
3184  }
3185 };
3186 
3187 class VmaDefragmentator;
3188 
3189 /*
3190 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3191 Vulkan memory type.
3192 
3193 Synchronized internally with a mutex.
3194 */
3195 struct VmaBlockVector
3196 {
3197  VmaBlockVector(
3198  VmaAllocator hAllocator,
3199  uint32_t memoryTypeIndex,
3200  VMA_BLOCK_VECTOR_TYPE blockVectorType,
3201  VkDeviceSize preferredBlockSize,
3202  size_t minBlockCount,
3203  size_t maxBlockCount,
3204  VkDeviceSize bufferImageGranularity,
3205  uint32_t frameInUseCount,
3206  bool isCustomPool);
3207  ~VmaBlockVector();
3208 
3209  VkResult CreateMinBlocks();
3210 
3211  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3212  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3213  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3214  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3215  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; }
3216 
3217  void GetPoolStats(VmaPoolStats* pStats);
3218 
3219  bool IsEmpty() const { return m_Blocks.empty(); }
3220 
3221  VkResult Allocate(
3222  VmaPool hCurrentPool,
3223  uint32_t currentFrameIndex,
3224  const VkMemoryRequirements& vkMemReq,
3225  const VmaAllocationCreateInfo& createInfo,
3226  VmaSuballocationType suballocType,
3227  VmaAllocation* pAllocation);
3228 
3229  void Free(
3230  VmaAllocation hAllocation);
3231 
3232  // Adds statistics of this BlockVector to pStats.
3233  void AddStats(VmaStats* pStats);
3234 
3235 #if VMA_STATS_STRING_ENABLED
3236  void PrintDetailedMap(class VmaJsonWriter& json);
3237 #endif
3238 
3239  void UnmapPersistentlyMappedMemory();
3240  VkResult MapPersistentlyMappedMemory();
3241 
3242  void MakePoolAllocationsLost(
3243  uint32_t currentFrameIndex,
3244  size_t* pLostAllocationCount);
3245 
3246  VmaDefragmentator* EnsureDefragmentator(
3247  VmaAllocator hAllocator,
3248  uint32_t currentFrameIndex);
3249 
3250  VkResult Defragment(
3251  VmaDefragmentationStats* pDefragmentationStats,
3252  VkDeviceSize& maxBytesToMove,
3253  uint32_t& maxAllocationsToMove);
3254 
3255  void DestroyDefragmentator();
3256 
3257 private:
3258  friend class VmaDefragmentator;
3259 
3260  const VmaAllocator m_hAllocator;
3261  const uint32_t m_MemoryTypeIndex;
3262  const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3263  const VkDeviceSize m_PreferredBlockSize;
3264  const size_t m_MinBlockCount;
3265  const size_t m_MaxBlockCount;
3266  const VkDeviceSize m_BufferImageGranularity;
3267  const uint32_t m_FrameInUseCount;
3268  const bool m_IsCustomPool;
3269  VMA_MUTEX m_Mutex;
3270  // Incrementally sorted by sumFreeSize, ascending.
3271  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3272  /* There can be at most one allocation that is completely empty - a
3273  hysteresis to avoid pessimistic case of alternating creation and destruction
3274  of a VkDeviceMemory. */
3275  bool m_HasEmptyBlock;
3276  VmaDefragmentator* m_pDefragmentator;
3277 
3278  // Finds and removes given block from vector.
3279  void Remove(VmaDeviceMemoryBlock* pBlock);
3280 
3281  // Performs single step in sorting m_Blocks. They may not be fully sorted
3282  // after this call.
3283  void IncrementallySortBlocks();
3284 
3285  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3286 };
3287 
3288 struct VmaPool_T
3289 {
3290 public:
3291  VmaBlockVector m_BlockVector;
3292 
3293  // Takes ownership.
3294  VmaPool_T(
3295  VmaAllocator hAllocator,
3296  const VmaPoolCreateInfo& createInfo);
3297  ~VmaPool_T();
3298 
3299  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3300 
3301 #if VMA_STATS_STRING_ENABLED
3302  //void PrintDetailedMap(class VmaStringBuilder& sb);
3303 #endif
3304 };
3305 
3306 class VmaDefragmentator
3307 {
3308  const VmaAllocator m_hAllocator;
3309  VmaBlockVector* const m_pBlockVector;
3310  uint32_t m_CurrentFrameIndex;
3311  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3312  VkDeviceSize m_BytesMoved;
3313  uint32_t m_AllocationsMoved;
3314 
3315  struct AllocationInfo
3316  {
3317  VmaAllocation m_hAllocation;
3318  VkBool32* m_pChanged;
3319 
3320  AllocationInfo() :
3321  m_hAllocation(VK_NULL_HANDLE),
3322  m_pChanged(VMA_NULL)
3323  {
3324  }
3325  };
3326 
3327  struct AllocationInfoSizeGreater
3328  {
3329  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3330  {
3331  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3332  }
3333  };
3334 
3335  // Used between AddAllocation and Defragment.
3336  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3337 
3338  struct BlockInfo
3339  {
3340  VmaDeviceMemoryBlock* m_pBlock;
3341  bool m_HasNonMovableAllocations;
3342  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3343 
3344  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3345  m_pBlock(VMA_NULL),
3346  m_HasNonMovableAllocations(true),
3347  m_Allocations(pAllocationCallbacks),
3348  m_pMappedDataForDefragmentation(VMA_NULL)
3349  {
3350  }
3351 
3352  void CalcHasNonMovableAllocations()
3353  {
3354  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3355  const size_t defragmentAllocCount = m_Allocations.size();
3356  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3357  }
3358 
3359  void SortAllocationsBySizeDescecnding()
3360  {
3361  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3362  }
3363 
3364  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3365  void Unmap(VmaAllocator hAllocator);
3366 
3367  private:
3368  // Not null if mapped for defragmentation only, not persistently mapped.
3369  void* m_pMappedDataForDefragmentation;
3370  };
3371 
3372  struct BlockPointerLess
3373  {
3374  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3375  {
3376  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3377  }
3378  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3379  {
3380  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3381  }
3382  };
3383 
3384  // 1. Blocks with some non-movable allocations go first.
3385  // 2. Blocks with smaller sumFreeSize go first.
3386  struct BlockInfoCompareMoveDestination
3387  {
3388  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3389  {
3390  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3391  {
3392  return true;
3393  }
3394  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3395  {
3396  return false;
3397  }
3398  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3399  {
3400  return true;
3401  }
3402  return false;
3403  }
3404  };
3405 
3406  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3407  BlockInfoVector m_Blocks;
3408 
3409  VkResult DefragmentRound(
3410  VkDeviceSize maxBytesToMove,
3411  uint32_t maxAllocationsToMove);
3412 
3413  static bool MoveMakesSense(
3414  size_t dstBlockIndex, VkDeviceSize dstOffset,
3415  size_t srcBlockIndex, VkDeviceSize srcOffset);
3416 
3417 public:
3418  VmaDefragmentator(
3419  VmaAllocator hAllocator,
3420  VmaBlockVector* pBlockVector,
3421  uint32_t currentFrameIndex);
3422 
3423  ~VmaDefragmentator();
3424 
3425  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3426  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3427 
3428  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3429 
3430  VkResult Defragment(
3431  VkDeviceSize maxBytesToMove,
3432  uint32_t maxAllocationsToMove);
3433 };
3434 
3435 // Main allocator object.
3436 struct VmaAllocator_T
3437 {
3438  bool m_UseMutex;
3439  VkDevice m_hDevice;
3440  bool m_AllocationCallbacksSpecified;
3441  VkAllocationCallbacks m_AllocationCallbacks;
3442  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3443  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
3444  // Counter to allow nested calls to these functions.
3445  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
3446 
3447  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3448  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3449  VMA_MUTEX m_HeapSizeLimitMutex;
3450 
3451  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3452  VkPhysicalDeviceMemoryProperties m_MemProps;
3453 
3454  // Default pools.
3455  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3456 
3457  // Each vector is sorted by memory (handle value).
3458  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3459  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3460  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
3461 
3462  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
3463  ~VmaAllocator_T();
3464 
3465  const VkAllocationCallbacks* GetAllocationCallbacks() const
3466  {
3467  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
3468  }
3469  const VmaVulkanFunctions& GetVulkanFunctions() const
3470  {
3471  return m_VulkanFunctions;
3472  }
3473 
3474  VkDeviceSize GetBufferImageGranularity() const
3475  {
3476  return VMA_MAX(
3477  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
3478  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
3479  }
3480 
3481  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
3482  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
3483 
3484  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
3485  {
3486  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
3487  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3488  }
3489 
3490  // Main allocation function.
3491  VkResult AllocateMemory(
3492  const VkMemoryRequirements& vkMemReq,
3493  const VmaAllocationCreateInfo& createInfo,
3494  VmaSuballocationType suballocType,
3495  VmaAllocation* pAllocation);
3496 
3497  // Main deallocation function.
3498  void FreeMemory(const VmaAllocation allocation);
3499 
3500  void CalculateStats(VmaStats* pStats);
3501 
3502 #if VMA_STATS_STRING_ENABLED
3503  void PrintDetailedMap(class VmaJsonWriter& json);
3504 #endif
3505 
3506  void UnmapPersistentlyMappedMemory();
3507  VkResult MapPersistentlyMappedMemory();
3508 
3509  VkResult Defragment(
3510  VmaAllocation* pAllocations,
3511  size_t allocationCount,
3512  VkBool32* pAllocationsChanged,
3513  const VmaDefragmentationInfo* pDefragmentationInfo,
3514  VmaDefragmentationStats* pDefragmentationStats);
3515 
3516  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
3517 
3518  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
3519  void DestroyPool(VmaPool pool);
3520  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
3521 
3522  void SetCurrentFrameIndex(uint32_t frameIndex);
3523 
3524  void MakePoolAllocationsLost(
3525  VmaPool hPool,
3526  size_t* pLostAllocationCount);
3527 
3528  void CreateLostAllocation(VmaAllocation* pAllocation);
3529 
3530  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
3531  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
3532 
3533 private:
3534  VkDeviceSize m_PreferredLargeHeapBlockSize;
3535  VkDeviceSize m_PreferredSmallHeapBlockSize;
3536 
3537  VkPhysicalDevice m_PhysicalDevice;
3538  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
3539 
3540  VMA_MUTEX m_PoolsMutex;
3541  // Protected by m_PoolsMutex. Sorted by pointer value.
3542  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
3543 
3544  VmaVulkanFunctions m_VulkanFunctions;
3545 
3546  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
3547 
3548  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
3549 
3550  VkResult AllocateMemoryOfType(
3551  const VkMemoryRequirements& vkMemReq,
3552  const VmaAllocationCreateInfo& createInfo,
3553  uint32_t memTypeIndex,
3554  VmaSuballocationType suballocType,
3555  VmaAllocation* pAllocation);
3556 
3557  // Allocates and registers new VkDeviceMemory specifically for single allocation.
3558  VkResult AllocateOwnMemory(
3559  VkDeviceSize size,
3560  VmaSuballocationType suballocType,
3561  uint32_t memTypeIndex,
3562  bool map,
3563  void* pUserData,
3564  VmaAllocation* pAllocation);
3565 
3566  // Tries to free pMemory as Own Memory. Returns true if found and freed.
3567  void FreeOwnMemory(VmaAllocation allocation);
3568 };
3569 
3571 // Memory allocation #2 after VmaAllocator_T definition
3572 
3573 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
3574 {
3575  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
3576 }
3577 
3578 static void VmaFree(VmaAllocator hAllocator, void* ptr)
3579 {
3580  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
3581 }
3582 
3583 template<typename T>
3584 static T* VmaAllocate(VmaAllocator hAllocator)
3585 {
3586  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
3587 }
3588 
3589 template<typename T>
3590 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
3591 {
3592  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
3593 }
3594 
3595 template<typename T>
3596 static void vma_delete(VmaAllocator hAllocator, T* ptr)
3597 {
3598  if(ptr != VMA_NULL)
3599  {
3600  ptr->~T();
3601  VmaFree(hAllocator, ptr);
3602  }
3603 }
3604 
3605 template<typename T>
3606 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
3607 {
3608  if(ptr != VMA_NULL)
3609  {
3610  for(size_t i = count; i--; )
3611  ptr[i].~T();
3612  VmaFree(hAllocator, ptr);
3613  }
3614 }
3615 
3617 // VmaStringBuilder
3618 
3619 #if VMA_STATS_STRING_ENABLED
3620 
3621 class VmaStringBuilder
3622 {
3623 public:
3624  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
3625  size_t GetLength() const { return m_Data.size(); }
3626  const char* GetData() const { return m_Data.data(); }
3627 
3628  void Add(char ch) { m_Data.push_back(ch); }
3629  void Add(const char* pStr);
3630  void AddNewLine() { Add('\n'); }
3631  void AddNumber(uint32_t num);
3632  void AddNumber(uint64_t num);
3633  void AddPointer(const void* ptr);
3634 
3635 private:
3636  VmaVector< char, VmaStlAllocator<char> > m_Data;
3637 };
3638 
3639 void VmaStringBuilder::Add(const char* pStr)
3640 {
3641  const size_t strLen = strlen(pStr);
3642  if(strLen > 0)
3643  {
3644  const size_t oldCount = m_Data.size();
3645  m_Data.resize(oldCount + strLen);
3646  memcpy(m_Data.data() + oldCount, pStr, strLen);
3647  }
3648 }
3649 
3650 void VmaStringBuilder::AddNumber(uint32_t num)
3651 {
3652  char buf[11];
3653  VmaUint32ToStr(buf, sizeof(buf), num);
3654  Add(buf);
3655 }
3656 
3657 void VmaStringBuilder::AddNumber(uint64_t num)
3658 {
3659  char buf[21];
3660  VmaUint64ToStr(buf, sizeof(buf), num);
3661  Add(buf);
3662 }
3663 
3664 void VmaStringBuilder::AddPointer(const void* ptr)
3665 {
3666  char buf[21];
3667  VmaPtrToStr(buf, sizeof(buf), ptr);
3668  Add(buf);
3669 }
3670 
3671 #endif // #if VMA_STATS_STRING_ENABLED
3672 
3674 // VmaJsonWriter
3675 
3676 #if VMA_STATS_STRING_ENABLED
3677 
3678 class VmaJsonWriter
3679 {
3680 public:
3681  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
3682  ~VmaJsonWriter();
3683 
3684  void BeginObject(bool singleLine = false);
3685  void EndObject();
3686 
3687  void BeginArray(bool singleLine = false);
3688  void EndArray();
3689 
3690  void WriteString(const char* pStr);
3691  void BeginString(const char* pStr = VMA_NULL);
3692  void ContinueString(const char* pStr);
3693  void ContinueString(uint32_t n);
3694  void ContinueString(uint64_t n);
3695  void EndString(const char* pStr = VMA_NULL);
3696 
3697  void WriteNumber(uint32_t n);
3698  void WriteNumber(uint64_t n);
3699  void WriteBool(bool b);
3700  void WriteNull();
3701 
3702 private:
3703  static const char* const INDENT;
3704 
3705  enum COLLECTION_TYPE
3706  {
3707  COLLECTION_TYPE_OBJECT,
3708  COLLECTION_TYPE_ARRAY,
3709  };
3710  struct StackItem
3711  {
3712  COLLECTION_TYPE type;
3713  uint32_t valueCount;
3714  bool singleLineMode;
3715  };
3716 
3717  VmaStringBuilder& m_SB;
3718  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
3719  bool m_InsideString;
3720 
3721  void BeginValue(bool isString);
3722  void WriteIndent(bool oneLess = false);
3723 };
3724 
3725 const char* const VmaJsonWriter::INDENT = " ";
3726 
3727 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
3728  m_SB(sb),
3729  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
3730  m_InsideString(false)
3731 {
3732 }
3733 
3734 VmaJsonWriter::~VmaJsonWriter()
3735 {
3736  VMA_ASSERT(!m_InsideString);
3737  VMA_ASSERT(m_Stack.empty());
3738 }
3739 
3740 void VmaJsonWriter::BeginObject(bool singleLine)
3741 {
3742  VMA_ASSERT(!m_InsideString);
3743 
3744  BeginValue(false);
3745  m_SB.Add('{');
3746 
3747  StackItem item;
3748  item.type = COLLECTION_TYPE_OBJECT;
3749  item.valueCount = 0;
3750  item.singleLineMode = singleLine;
3751  m_Stack.push_back(item);
3752 }
3753 
3754 void VmaJsonWriter::EndObject()
3755 {
3756  VMA_ASSERT(!m_InsideString);
3757 
3758  WriteIndent(true);
3759  m_SB.Add('}');
3760 
3761  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
3762  m_Stack.pop_back();
3763 }
3764 
3765 void VmaJsonWriter::BeginArray(bool singleLine)
3766 {
3767  VMA_ASSERT(!m_InsideString);
3768 
3769  BeginValue(false);
3770  m_SB.Add('[');
3771 
3772  StackItem item;
3773  item.type = COLLECTION_TYPE_ARRAY;
3774  item.valueCount = 0;
3775  item.singleLineMode = singleLine;
3776  m_Stack.push_back(item);
3777 }
3778 
3779 void VmaJsonWriter::EndArray()
3780 {
3781  VMA_ASSERT(!m_InsideString);
3782 
3783  WriteIndent(true);
3784  m_SB.Add(']');
3785 
3786  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
3787  m_Stack.pop_back();
3788 }
3789 
3790 void VmaJsonWriter::WriteString(const char* pStr)
3791 {
3792  BeginString(pStr);
3793  EndString();
3794 }
3795 
3796 void VmaJsonWriter::BeginString(const char* pStr)
3797 {
3798  VMA_ASSERT(!m_InsideString);
3799 
3800  BeginValue(true);
3801  m_SB.Add('"');
3802  m_InsideString = true;
3803  if(pStr != VMA_NULL && pStr[0] != '\0')
3804  {
3805  ContinueString(pStr);
3806  }
3807 }
3808 
3809 void VmaJsonWriter::ContinueString(const char* pStr)
3810 {
3811  VMA_ASSERT(m_InsideString);
3812 
3813  const size_t strLen = strlen(pStr);
3814  for(size_t i = 0; i < strLen; ++i)
3815  {
3816  char ch = pStr[i];
3817  if(ch == '\'')
3818  {
3819  m_SB.Add("\\\\");
3820  }
3821  else if(ch == '"')
3822  {
3823  m_SB.Add("\\\"");
3824  }
3825  else if(ch >= 32)
3826  {
3827  m_SB.Add(ch);
3828  }
3829  else switch(ch)
3830  {
3831  case '\n':
3832  m_SB.Add("\\n");
3833  break;
3834  case '\r':
3835  m_SB.Add("\\r");
3836  break;
3837  case '\t':
3838  m_SB.Add("\\t");
3839  break;
3840  default:
3841  VMA_ASSERT(0 && "Character not currently supported.");
3842  break;
3843  }
3844  }
3845 }
3846 
3847 void VmaJsonWriter::ContinueString(uint32_t n)
3848 {
3849  VMA_ASSERT(m_InsideString);
3850  m_SB.AddNumber(n);
3851 }
3852 
3853 void VmaJsonWriter::ContinueString(uint64_t n)
3854 {
3855  VMA_ASSERT(m_InsideString);
3856  m_SB.AddNumber(n);
3857 }
3858 
3859 void VmaJsonWriter::EndString(const char* pStr)
3860 {
3861  VMA_ASSERT(m_InsideString);
3862  if(pStr != VMA_NULL && pStr[0] != '\0')
3863  {
3864  ContinueString(pStr);
3865  }
3866  m_SB.Add('"');
3867  m_InsideString = false;
3868 }
3869 
3870 void VmaJsonWriter::WriteNumber(uint32_t n)
3871 {
3872  VMA_ASSERT(!m_InsideString);
3873  BeginValue(false);
3874  m_SB.AddNumber(n);
3875 }
3876 
3877 void VmaJsonWriter::WriteNumber(uint64_t n)
3878 {
3879  VMA_ASSERT(!m_InsideString);
3880  BeginValue(false);
3881  m_SB.AddNumber(n);
3882 }
3883 
3884 void VmaJsonWriter::WriteBool(bool b)
3885 {
3886  VMA_ASSERT(!m_InsideString);
3887  BeginValue(false);
3888  m_SB.Add(b ? "true" : "false");
3889 }
3890 
3891 void VmaJsonWriter::WriteNull()
3892 {
3893  VMA_ASSERT(!m_InsideString);
3894  BeginValue(false);
3895  m_SB.Add("null");
3896 }
3897 
3898 void VmaJsonWriter::BeginValue(bool isString)
3899 {
3900  if(!m_Stack.empty())
3901  {
3902  StackItem& currItem = m_Stack.back();
3903  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3904  currItem.valueCount % 2 == 0)
3905  {
3906  VMA_ASSERT(isString);
3907  }
3908 
3909  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3910  currItem.valueCount % 2 != 0)
3911  {
3912  m_SB.Add(": ");
3913  }
3914  else if(currItem.valueCount > 0)
3915  {
3916  m_SB.Add(", ");
3917  WriteIndent();
3918  }
3919  else
3920  {
3921  WriteIndent();
3922  }
3923  ++currItem.valueCount;
3924  }
3925 }
3926 
3927 void VmaJsonWriter::WriteIndent(bool oneLess)
3928 {
3929  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
3930  {
3931  m_SB.AddNewLine();
3932 
3933  size_t count = m_Stack.size();
3934  if(count > 0 && oneLess)
3935  {
3936  --count;
3937  }
3938  for(size_t i = 0; i < count; ++i)
3939  {
3940  m_SB.Add(INDENT);
3941  }
3942  }
3943 }
3944 
3945 #endif // #if VMA_STATS_STRING_ENABLED
3946 
3948 
3949 VkDeviceSize VmaAllocation_T::GetOffset() const
3950 {
3951  switch(m_Type)
3952  {
3953  case ALLOCATION_TYPE_BLOCK:
3954  return m_BlockAllocation.m_Offset;
3955  case ALLOCATION_TYPE_OWN:
3956  return 0;
3957  default:
3958  VMA_ASSERT(0);
3959  return 0;
3960  }
3961 }
3962 
3963 VkDeviceMemory VmaAllocation_T::GetMemory() const
3964 {
3965  switch(m_Type)
3966  {
3967  case ALLOCATION_TYPE_BLOCK:
3968  return m_BlockAllocation.m_Block->m_hMemory;
3969  case ALLOCATION_TYPE_OWN:
3970  return m_OwnAllocation.m_hMemory;
3971  default:
3972  VMA_ASSERT(0);
3973  return VK_NULL_HANDLE;
3974  }
3975 }
3976 
3977 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
3978 {
3979  switch(m_Type)
3980  {
3981  case ALLOCATION_TYPE_BLOCK:
3982  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
3983  case ALLOCATION_TYPE_OWN:
3984  return m_OwnAllocation.m_MemoryTypeIndex;
3985  default:
3986  VMA_ASSERT(0);
3987  return UINT32_MAX;
3988  }
3989 }
3990 
3991 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
3992 {
3993  switch(m_Type)
3994  {
3995  case ALLOCATION_TYPE_BLOCK:
3996  return m_BlockAllocation.m_Block->m_BlockVectorType;
3997  case ALLOCATION_TYPE_OWN:
3998  return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
3999  default:
4000  VMA_ASSERT(0);
4001  return VMA_BLOCK_VECTOR_TYPE_COUNT;
4002  }
4003 }
4004 
4005 void* VmaAllocation_T::GetMappedData() const
4006 {
4007  switch(m_Type)
4008  {
4009  case ALLOCATION_TYPE_BLOCK:
4010  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
4011  {
4012  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
4013  }
4014  else
4015  {
4016  return VMA_NULL;
4017  }
4018  break;
4019  case ALLOCATION_TYPE_OWN:
4020  return m_OwnAllocation.m_pMappedData;
4021  default:
4022  VMA_ASSERT(0);
4023  return VMA_NULL;
4024  }
4025 }
4026 
4027 bool VmaAllocation_T::CanBecomeLost() const
4028 {
4029  switch(m_Type)
4030  {
4031  case ALLOCATION_TYPE_BLOCK:
4032  return m_BlockAllocation.m_CanBecomeLost;
4033  case ALLOCATION_TYPE_OWN:
4034  return false;
4035  default:
4036  VMA_ASSERT(0);
4037  return false;
4038  }
4039 }
4040 
4041 VmaPool VmaAllocation_T::GetPool() const
4042 {
4043  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4044  return m_BlockAllocation.m_hPool;
4045 }
4046 
4047 VkResult VmaAllocation_T::OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator)
4048 {
4049  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4050  if(m_OwnAllocation.m_PersistentMap)
4051  {
4052  return (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4053  hAllocator->m_hDevice,
4054  m_OwnAllocation.m_hMemory,
4055  0,
4056  VK_WHOLE_SIZE,
4057  0,
4058  &m_OwnAllocation.m_pMappedData);
4059  }
4060  return VK_SUCCESS;
4061 }
4062 void VmaAllocation_T::OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator)
4063 {
4064  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4065  if(m_OwnAllocation.m_pMappedData)
4066  {
4067  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
4068  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_OwnAllocation.m_hMemory);
4069  m_OwnAllocation.m_pMappedData = VMA_NULL;
4070  }
4071 }
4072 
4073 
4074 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4075 {
4076  VMA_ASSERT(CanBecomeLost());
4077 
4078  /*
4079  Warning: This is a carefully designed algorithm.
4080  Do not modify unless you really know what you're doing :)
4081  */
4082  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4083  for(;;)
4084  {
4085  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4086  {
4087  VMA_ASSERT(0);
4088  return false;
4089  }
4090  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4091  {
4092  return false;
4093  }
4094  else // Last use time earlier than current time.
4095  {
4096  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4097  {
4098  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4099  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4100  return true;
4101  }
4102  }
4103  }
4104 }
4105 
4106 #if VMA_STATS_STRING_ENABLED
4107 
4108 // Correspond to values of enum VmaSuballocationType.
4109 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4110  "FREE",
4111  "UNKNOWN",
4112  "BUFFER",
4113  "IMAGE_UNKNOWN",
4114  "IMAGE_LINEAR",
4115  "IMAGE_OPTIMAL",
4116 };
4117 
4118 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4119 {
4120  json.BeginObject();
4121 
4122  json.WriteString("Blocks");
4123  json.WriteNumber(stat.blockCount);
4124 
4125  json.WriteString("Allocations");
4126  json.WriteNumber(stat.allocationCount);
4127 
4128  json.WriteString("UnusedRanges");
4129  json.WriteNumber(stat.unusedRangeCount);
4130 
4131  json.WriteString("UsedBytes");
4132  json.WriteNumber(stat.usedBytes);
4133 
4134  json.WriteString("UnusedBytes");
4135  json.WriteNumber(stat.unusedBytes);
4136 
4137  if(stat.allocationCount > 1)
4138  {
4139  json.WriteString("AllocationSize");
4140  json.BeginObject(true);
4141  json.WriteString("Min");
4142  json.WriteNumber(stat.allocationSizeMin);
4143  json.WriteString("Avg");
4144  json.WriteNumber(stat.allocationSizeAvg);
4145  json.WriteString("Max");
4146  json.WriteNumber(stat.allocationSizeMax);
4147  json.EndObject();
4148  }
4149 
4150  if(stat.unusedRangeCount > 1)
4151  {
4152  json.WriteString("UnusedRangeSize");
4153  json.BeginObject(true);
4154  json.WriteString("Min");
4155  json.WriteNumber(stat.unusedRangeSizeMin);
4156  json.WriteString("Avg");
4157  json.WriteNumber(stat.unusedRangeSizeAvg);
4158  json.WriteString("Max");
4159  json.WriteNumber(stat.unusedRangeSizeMax);
4160  json.EndObject();
4161  }
4162 
4163  json.EndObject();
4164 }
4165 
4166 #endif // #if VMA_STATS_STRING_ENABLED
4167 
4168 struct VmaSuballocationItemSizeLess
4169 {
4170  bool operator()(
4171  const VmaSuballocationList::iterator lhs,
4172  const VmaSuballocationList::iterator rhs) const
4173  {
4174  return lhs->size < rhs->size;
4175  }
4176  bool operator()(
4177  const VmaSuballocationList::iterator lhs,
4178  VkDeviceSize rhsSize) const
4179  {
4180  return lhs->size < rhsSize;
4181  }
4182 };
4183 
4185 // class VmaBlockMetadata
4186 
4187 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4188  m_Size(0),
4189  m_FreeCount(0),
4190  m_SumFreeSize(0),
4191  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4192  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4193 {
4194 }
4195 
4196 VmaBlockMetadata::~VmaBlockMetadata()
4197 {
4198 }
4199 
4200 void VmaBlockMetadata::Init(VkDeviceSize size)
4201 {
4202  m_Size = size;
4203  m_FreeCount = 1;
4204  m_SumFreeSize = size;
4205 
4206  VmaSuballocation suballoc = {};
4207  suballoc.offset = 0;
4208  suballoc.size = size;
4209  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4210  suballoc.hAllocation = VK_NULL_HANDLE;
4211 
4212  m_Suballocations.push_back(suballoc);
4213  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4214  --suballocItem;
4215  m_FreeSuballocationsBySize.push_back(suballocItem);
4216 }
4217 
4218 bool VmaBlockMetadata::Validate() const
4219 {
4220  if(m_Suballocations.empty())
4221  {
4222  return false;
4223  }
4224 
4225  // Expected offset of new suballocation as calculates from previous ones.
4226  VkDeviceSize calculatedOffset = 0;
4227  // Expected number of free suballocations as calculated from traversing their list.
4228  uint32_t calculatedFreeCount = 0;
4229  // Expected sum size of free suballocations as calculated from traversing their list.
4230  VkDeviceSize calculatedSumFreeSize = 0;
4231  // Expected number of free suballocations that should be registered in
4232  // m_FreeSuballocationsBySize calculated from traversing their list.
4233  size_t freeSuballocationsToRegister = 0;
4234  // True if previous visisted suballocation was free.
4235  bool prevFree = false;
4236 
4237  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4238  suballocItem != m_Suballocations.cend();
4239  ++suballocItem)
4240  {
4241  const VmaSuballocation& subAlloc = *suballocItem;
4242 
4243  // Actual offset of this suballocation doesn't match expected one.
4244  if(subAlloc.offset != calculatedOffset)
4245  {
4246  return false;
4247  }
4248 
4249  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4250  // Two adjacent free suballocations are invalid. They should be merged.
4251  if(prevFree && currFree)
4252  {
4253  return false;
4254  }
4255  prevFree = currFree;
4256 
4257  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4258  {
4259  return false;
4260  }
4261 
4262  if(currFree)
4263  {
4264  calculatedSumFreeSize += subAlloc.size;
4265  ++calculatedFreeCount;
4266  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4267  {
4268  ++freeSuballocationsToRegister;
4269  }
4270  }
4271 
4272  calculatedOffset += subAlloc.size;
4273  }
4274 
4275  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4276  // match expected one.
4277  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4278  {
4279  return false;
4280  }
4281 
4282  VkDeviceSize lastSize = 0;
4283  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4284  {
4285  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4286 
4287  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4288  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4289  {
4290  return false;
4291  }
4292  // They must be sorted by size ascending.
4293  if(suballocItem->size < lastSize)
4294  {
4295  return false;
4296  }
4297 
4298  lastSize = suballocItem->size;
4299  }
4300 
4301  // Check if totals match calculacted values.
4302  return
4303  ValidateFreeSuballocationList() &&
4304  (calculatedOffset == m_Size) &&
4305  (calculatedSumFreeSize == m_SumFreeSize) &&
4306  (calculatedFreeCount == m_FreeCount);
4307 }
4308 
4309 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
4310 {
4311  if(!m_FreeSuballocationsBySize.empty())
4312  {
4313  return m_FreeSuballocationsBySize.back()->size;
4314  }
4315  else
4316  {
4317  return 0;
4318  }
4319 }
4320 
4321 bool VmaBlockMetadata::IsEmpty() const
4322 {
4323  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
4324 }
4325 
4326 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
4327 {
4328  outInfo.blockCount = 1;
4329 
4330  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4331  outInfo.allocationCount = rangeCount - m_FreeCount;
4332  outInfo.unusedRangeCount = m_FreeCount;
4333 
4334  outInfo.unusedBytes = m_SumFreeSize;
4335  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
4336 
4337  outInfo.allocationSizeMin = UINT64_MAX;
4338  outInfo.allocationSizeMax = 0;
4339  outInfo.unusedRangeSizeMin = UINT64_MAX;
4340  outInfo.unusedRangeSizeMax = 0;
4341 
4342  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4343  suballocItem != m_Suballocations.cend();
4344  ++suballocItem)
4345  {
4346  const VmaSuballocation& suballoc = *suballocItem;
4347  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
4348  {
4349  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
4350  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
4351  }
4352  else
4353  {
4354  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
4355  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
4356  }
4357  }
4358 }
4359 
4360 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
4361 {
4362  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4363 
4364  inoutStats.size += m_Size;
4365  inoutStats.unusedSize += m_SumFreeSize;
4366  inoutStats.allocationCount += rangeCount - m_FreeCount;
4367  inoutStats.unusedRangeCount += m_FreeCount;
4368  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
4369 }
4370 
4371 #if VMA_STATS_STRING_ENABLED
4372 
4373 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
4374 {
4375  json.BeginObject();
4376 
4377  json.WriteString("TotalBytes");
4378  json.WriteNumber(m_Size);
4379 
4380  json.WriteString("UnusedBytes");
4381  json.WriteNumber(m_SumFreeSize);
4382 
4383  json.WriteString("Allocations");
4384  json.WriteNumber(m_Suballocations.size() - m_FreeCount);
4385 
4386  json.WriteString("UnusedRanges");
4387  json.WriteNumber(m_FreeCount);
4388 
4389  json.WriteString("Suballocations");
4390  json.BeginArray();
4391  size_t i = 0;
4392  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4393  suballocItem != m_Suballocations.cend();
4394  ++suballocItem, ++i)
4395  {
4396  json.BeginObject(true);
4397 
4398  json.WriteString("Type");
4399  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
4400 
4401  json.WriteString("Size");
4402  json.WriteNumber(suballocItem->size);
4403 
4404  json.WriteString("Offset");
4405  json.WriteNumber(suballocItem->offset);
4406 
4407  json.EndObject();
4408  }
4409  json.EndArray();
4410 
4411  json.EndObject();
4412 }
4413 
4414 #endif // #if VMA_STATS_STRING_ENABLED
4415 
4416 /*
4417 How many suitable free suballocations to analyze before choosing best one.
4418 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
4419  be chosen.
4420 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
4421  suballocations will be analized and best one will be chosen.
4422 - Any other value is also acceptable.
4423 */
4424 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
4425 
4426 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
4427 {
4428  VMA_ASSERT(IsEmpty());
4429  pAllocationRequest->offset = 0;
4430  pAllocationRequest->sumFreeSize = m_SumFreeSize;
4431  pAllocationRequest->sumItemSize = 0;
4432  pAllocationRequest->item = m_Suballocations.begin();
4433  pAllocationRequest->itemsToMakeLostCount = 0;
4434 }
4435 
4436 bool VmaBlockMetadata::CreateAllocationRequest(
4437  uint32_t currentFrameIndex,
4438  uint32_t frameInUseCount,
4439  VkDeviceSize bufferImageGranularity,
4440  VkDeviceSize allocSize,
4441  VkDeviceSize allocAlignment,
4442  VmaSuballocationType allocType,
4443  bool canMakeOtherLost,
4444  VmaAllocationRequest* pAllocationRequest)
4445 {
4446  VMA_ASSERT(allocSize > 0);
4447  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4448  VMA_ASSERT(pAllocationRequest != VMA_NULL);
4449  VMA_HEAVY_ASSERT(Validate());
4450 
4451  // There is not enough total free space in this block to fullfill the request: Early return.
4452  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
4453  {
4454  return false;
4455  }
4456 
4457  // New algorithm, efficiently searching freeSuballocationsBySize.
4458  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
4459  if(freeSuballocCount > 0)
4460  {
4461  if(VMA_BEST_FIT)
4462  {
4463  // Find first free suballocation with size not less than allocSize.
4464  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
4465  m_FreeSuballocationsBySize.data(),
4466  m_FreeSuballocationsBySize.data() + freeSuballocCount,
4467  allocSize,
4468  VmaSuballocationItemSizeLess());
4469  size_t index = it - m_FreeSuballocationsBySize.data();
4470  for(; index < freeSuballocCount; ++index)
4471  {
4472  if(CheckAllocation(
4473  currentFrameIndex,
4474  frameInUseCount,
4475  bufferImageGranularity,
4476  allocSize,
4477  allocAlignment,
4478  allocType,
4479  m_FreeSuballocationsBySize[index],
4480  false, // canMakeOtherLost
4481  &pAllocationRequest->offset,
4482  &pAllocationRequest->itemsToMakeLostCount,
4483  &pAllocationRequest->sumFreeSize,
4484  &pAllocationRequest->sumItemSize))
4485  {
4486  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4487  return true;
4488  }
4489  }
4490  }
4491  else
4492  {
4493  // Search staring from biggest suballocations.
4494  for(size_t index = freeSuballocCount; index--; )
4495  {
4496  if(CheckAllocation(
4497  currentFrameIndex,
4498  frameInUseCount,
4499  bufferImageGranularity,
4500  allocSize,
4501  allocAlignment,
4502  allocType,
4503  m_FreeSuballocationsBySize[index],
4504  false, // canMakeOtherLost
4505  &pAllocationRequest->offset,
4506  &pAllocationRequest->itemsToMakeLostCount,
4507  &pAllocationRequest->sumFreeSize,
4508  &pAllocationRequest->sumItemSize))
4509  {
4510  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4511  return true;
4512  }
4513  }
4514  }
4515  }
4516 
4517  if(canMakeOtherLost)
4518  {
4519  // Brute-force algorithm. TODO: Come up with something better.
4520 
4521  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
4522  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
4523 
4524  VmaAllocationRequest tmpAllocRequest = {};
4525  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
4526  suballocIt != m_Suballocations.end();
4527  ++suballocIt)
4528  {
4529  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
4530  suballocIt->hAllocation->CanBecomeLost())
4531  {
4532  if(CheckAllocation(
4533  currentFrameIndex,
4534  frameInUseCount,
4535  bufferImageGranularity,
4536  allocSize,
4537  allocAlignment,
4538  allocType,
4539  suballocIt,
4540  canMakeOtherLost,
4541  &tmpAllocRequest.offset,
4542  &tmpAllocRequest.itemsToMakeLostCount,
4543  &tmpAllocRequest.sumFreeSize,
4544  &tmpAllocRequest.sumItemSize))
4545  {
4546  tmpAllocRequest.item = suballocIt;
4547 
4548  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
4549  {
4550  *pAllocationRequest = tmpAllocRequest;
4551  }
4552  }
4553  }
4554  }
4555 
4556  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
4557  {
4558  return true;
4559  }
4560  }
4561 
4562  return false;
4563 }
4564 
4565 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
4566  uint32_t currentFrameIndex,
4567  uint32_t frameInUseCount,
4568  VmaAllocationRequest* pAllocationRequest)
4569 {
4570  while(pAllocationRequest->itemsToMakeLostCount > 0)
4571  {
4572  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
4573  {
4574  ++pAllocationRequest->item;
4575  }
4576  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4577  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
4578  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
4579  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4580  {
4581  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
4582  --pAllocationRequest->itemsToMakeLostCount;
4583  }
4584  else
4585  {
4586  return false;
4587  }
4588  }
4589 
4590  VMA_HEAVY_ASSERT(Validate());
4591  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4592  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
4593 
4594  return true;
4595 }
4596 
4597 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4598 {
4599  uint32_t lostAllocationCount = 0;
4600  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
4601  it != m_Suballocations.end();
4602  ++it)
4603  {
4604  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
4605  it->hAllocation->CanBecomeLost() &&
4606  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4607  {
4608  it = FreeSuballocation(it);
4609  ++lostAllocationCount;
4610  }
4611  }
4612  return lostAllocationCount;
4613 }
4614 
4615 void VmaBlockMetadata::Alloc(
4616  const VmaAllocationRequest& request,
4617  VmaSuballocationType type,
4618  VkDeviceSize allocSize,
4619  VmaAllocation hAllocation)
4620 {
4621  VMA_ASSERT(request.item != m_Suballocations.end());
4622  VmaSuballocation& suballoc = *request.item;
4623  // Given suballocation is a free block.
4624  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4625  // Given offset is inside this suballocation.
4626  VMA_ASSERT(request.offset >= suballoc.offset);
4627  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
4628  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
4629  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
4630 
4631  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
4632  // it to become used.
4633  UnregisterFreeSuballocation(request.item);
4634 
4635  suballoc.offset = request.offset;
4636  suballoc.size = allocSize;
4637  suballoc.type = type;
4638  suballoc.hAllocation = hAllocation;
4639 
4640  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
4641  if(paddingEnd)
4642  {
4643  VmaSuballocation paddingSuballoc = {};
4644  paddingSuballoc.offset = request.offset + allocSize;
4645  paddingSuballoc.size = paddingEnd;
4646  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4647  VmaSuballocationList::iterator next = request.item;
4648  ++next;
4649  const VmaSuballocationList::iterator paddingEndItem =
4650  m_Suballocations.insert(next, paddingSuballoc);
4651  RegisterFreeSuballocation(paddingEndItem);
4652  }
4653 
4654  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
4655  if(paddingBegin)
4656  {
4657  VmaSuballocation paddingSuballoc = {};
4658  paddingSuballoc.offset = request.offset - paddingBegin;
4659  paddingSuballoc.size = paddingBegin;
4660  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4661  const VmaSuballocationList::iterator paddingBeginItem =
4662  m_Suballocations.insert(request.item, paddingSuballoc);
4663  RegisterFreeSuballocation(paddingBeginItem);
4664  }
4665 
4666  // Update totals.
4667  m_FreeCount = m_FreeCount - 1;
4668  if(paddingBegin > 0)
4669  {
4670  ++m_FreeCount;
4671  }
4672  if(paddingEnd > 0)
4673  {
4674  ++m_FreeCount;
4675  }
4676  m_SumFreeSize -= allocSize;
4677 }
4678 
4679 void VmaBlockMetadata::Free(const VmaAllocation allocation)
4680 {
4681  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
4682  suballocItem != m_Suballocations.end();
4683  ++suballocItem)
4684  {
4685  VmaSuballocation& suballoc = *suballocItem;
4686  if(suballoc.hAllocation == allocation)
4687  {
4688  FreeSuballocation(suballocItem);
4689  VMA_HEAVY_ASSERT(Validate());
4690  return;
4691  }
4692  }
4693  VMA_ASSERT(0 && "Not found!");
4694 }
4695 
4696 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
4697 {
4698  VkDeviceSize lastSize = 0;
4699  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
4700  {
4701  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
4702 
4703  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
4704  {
4705  VMA_ASSERT(0);
4706  return false;
4707  }
4708  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4709  {
4710  VMA_ASSERT(0);
4711  return false;
4712  }
4713  if(it->size < lastSize)
4714  {
4715  VMA_ASSERT(0);
4716  return false;
4717  }
4718 
4719  lastSize = it->size;
4720  }
4721  return true;
4722 }
4723 
4724 bool VmaBlockMetadata::CheckAllocation(
4725  uint32_t currentFrameIndex,
4726  uint32_t frameInUseCount,
4727  VkDeviceSize bufferImageGranularity,
4728  VkDeviceSize allocSize,
4729  VkDeviceSize allocAlignment,
4730  VmaSuballocationType allocType,
4731  VmaSuballocationList::const_iterator suballocItem,
4732  bool canMakeOtherLost,
4733  VkDeviceSize* pOffset,
4734  size_t* itemsToMakeLostCount,
4735  VkDeviceSize* pSumFreeSize,
4736  VkDeviceSize* pSumItemSize) const
4737 {
4738  VMA_ASSERT(allocSize > 0);
4739  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4740  VMA_ASSERT(suballocItem != m_Suballocations.cend());
4741  VMA_ASSERT(pOffset != VMA_NULL);
4742 
4743  *itemsToMakeLostCount = 0;
4744  *pSumFreeSize = 0;
4745  *pSumItemSize = 0;
4746 
4747  if(canMakeOtherLost)
4748  {
4749  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4750  {
4751  *pSumFreeSize = suballocItem->size;
4752  }
4753  else
4754  {
4755  if(suballocItem->hAllocation->CanBecomeLost() &&
4756  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4757  {
4758  ++*itemsToMakeLostCount;
4759  *pSumItemSize = suballocItem->size;
4760  }
4761  else
4762  {
4763  return false;
4764  }
4765  }
4766 
4767  // Remaining size is too small for this request: Early return.
4768  if(m_Size - suballocItem->offset < allocSize)
4769  {
4770  return false;
4771  }
4772 
4773  // Start from offset equal to beginning of this suballocation.
4774  *pOffset = suballocItem->offset;
4775 
4776  // Apply VMA_DEBUG_MARGIN at the beginning.
4777  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4778  {
4779  *pOffset += VMA_DEBUG_MARGIN;
4780  }
4781 
4782  // Apply alignment.
4783  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4784  *pOffset = VmaAlignUp(*pOffset, alignment);
4785 
4786  // Check previous suballocations for BufferImageGranularity conflicts.
4787  // Make bigger alignment if necessary.
4788  if(bufferImageGranularity > 1)
4789  {
4790  bool bufferImageGranularityConflict = false;
4791  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4792  while(prevSuballocItem != m_Suballocations.cbegin())
4793  {
4794  --prevSuballocItem;
4795  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4796  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4797  {
4798  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4799  {
4800  bufferImageGranularityConflict = true;
4801  break;
4802  }
4803  }
4804  else
4805  // Already on previous page.
4806  break;
4807  }
4808  if(bufferImageGranularityConflict)
4809  {
4810  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4811  }
4812  }
4813 
4814  // Now that we have final *pOffset, check if we are past suballocItem.
4815  // If yes, return false - this function should be called for another suballocItem as starting point.
4816  if(*pOffset >= suballocItem->offset + suballocItem->size)
4817  {
4818  return false;
4819  }
4820 
4821  // Calculate padding at the beginning based on current offset.
4822  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
4823 
4824  // Calculate required margin at the end if this is not last suballocation.
4825  VmaSuballocationList::const_iterator next = suballocItem;
4826  ++next;
4827  const VkDeviceSize requiredEndMargin =
4828  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4829 
4830  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
4831  // Another early return check.
4832  if(suballocItem->offset + totalSize > m_Size)
4833  {
4834  return false;
4835  }
4836 
4837  // Advance lastSuballocItem until desired size is reached.
4838  // Update itemsToMakeLostCount.
4839  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
4840  if(totalSize > suballocItem->size)
4841  {
4842  VkDeviceSize remainingSize = totalSize - suballocItem->size;
4843  while(remainingSize > 0)
4844  {
4845  ++lastSuballocItem;
4846  if(lastSuballocItem == m_Suballocations.cend())
4847  {
4848  return false;
4849  }
4850  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4851  {
4852  *pSumFreeSize += lastSuballocItem->size;
4853  }
4854  else
4855  {
4856  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
4857  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
4858  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4859  {
4860  ++*itemsToMakeLostCount;
4861  *pSumItemSize += lastSuballocItem->size;
4862  }
4863  else
4864  {
4865  return false;
4866  }
4867  }
4868  remainingSize = (lastSuballocItem->size < remainingSize) ?
4869  remainingSize - lastSuballocItem->size : 0;
4870  }
4871  }
4872 
4873  // Check next suballocations for BufferImageGranularity conflicts.
4874  // If conflict exists, we must mark more allocations lost or fail.
4875  if(bufferImageGranularity > 1)
4876  {
4877  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
4878  ++nextSuballocItem;
4879  while(nextSuballocItem != m_Suballocations.cend())
4880  {
4881  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4882  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4883  {
4884  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4885  {
4886  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
4887  if(nextSuballoc.hAllocation->CanBecomeLost() &&
4888  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4889  {
4890  ++*itemsToMakeLostCount;
4891  }
4892  else
4893  {
4894  return false;
4895  }
4896  }
4897  }
4898  else
4899  {
4900  // Already on next page.
4901  break;
4902  }
4903  ++nextSuballocItem;
4904  }
4905  }
4906  }
4907  else
4908  {
4909  const VmaSuballocation& suballoc = *suballocItem;
4910  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4911 
4912  *pSumFreeSize = suballoc.size;
4913 
4914  // Size of this suballocation is too small for this request: Early return.
4915  if(suballoc.size < allocSize)
4916  {
4917  return false;
4918  }
4919 
4920  // Start from offset equal to beginning of this suballocation.
4921  *pOffset = suballoc.offset;
4922 
4923  // Apply VMA_DEBUG_MARGIN at the beginning.
4924  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4925  {
4926  *pOffset += VMA_DEBUG_MARGIN;
4927  }
4928 
4929  // Apply alignment.
4930  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4931  *pOffset = VmaAlignUp(*pOffset, alignment);
4932 
4933  // Check previous suballocations for BufferImageGranularity conflicts.
4934  // Make bigger alignment if necessary.
4935  if(bufferImageGranularity > 1)
4936  {
4937  bool bufferImageGranularityConflict = false;
4938  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4939  while(prevSuballocItem != m_Suballocations.cbegin())
4940  {
4941  --prevSuballocItem;
4942  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4943  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4944  {
4945  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4946  {
4947  bufferImageGranularityConflict = true;
4948  break;
4949  }
4950  }
4951  else
4952  // Already on previous page.
4953  break;
4954  }
4955  if(bufferImageGranularityConflict)
4956  {
4957  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4958  }
4959  }
4960 
4961  // Calculate padding at the beginning based on current offset.
4962  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
4963 
4964  // Calculate required margin at the end if this is not last suballocation.
4965  VmaSuballocationList::const_iterator next = suballocItem;
4966  ++next;
4967  const VkDeviceSize requiredEndMargin =
4968  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4969 
4970  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
4971  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
4972  {
4973  return false;
4974  }
4975 
4976  // Check next suballocations for BufferImageGranularity conflicts.
4977  // If conflict exists, allocation cannot be made here.
4978  if(bufferImageGranularity > 1)
4979  {
4980  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
4981  ++nextSuballocItem;
4982  while(nextSuballocItem != m_Suballocations.cend())
4983  {
4984  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4985  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4986  {
4987  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4988  {
4989  return false;
4990  }
4991  }
4992  else
4993  {
4994  // Already on next page.
4995  break;
4996  }
4997  ++nextSuballocItem;
4998  }
4999  }
5000  }
5001 
5002  // All tests passed: Success. pOffset is already filled.
5003  return true;
5004 }
5005 
5006 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5007 {
5008  VMA_ASSERT(item != m_Suballocations.end());
5009  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5010 
5011  VmaSuballocationList::iterator nextItem = item;
5012  ++nextItem;
5013  VMA_ASSERT(nextItem != m_Suballocations.end());
5014  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5015 
5016  item->size += nextItem->size;
5017  --m_FreeCount;
5018  m_Suballocations.erase(nextItem);
5019 }
5020 
5021 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5022 {
5023  // Change this suballocation to be marked as free.
5024  VmaSuballocation& suballoc = *suballocItem;
5025  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5026  suballoc.hAllocation = VK_NULL_HANDLE;
5027 
5028  // Update totals.
5029  ++m_FreeCount;
5030  m_SumFreeSize += suballoc.size;
5031 
5032  // Merge with previous and/or next suballocation if it's also free.
5033  bool mergeWithNext = false;
5034  bool mergeWithPrev = false;
5035 
5036  VmaSuballocationList::iterator nextItem = suballocItem;
5037  ++nextItem;
5038  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5039  {
5040  mergeWithNext = true;
5041  }
5042 
5043  VmaSuballocationList::iterator prevItem = suballocItem;
5044  if(suballocItem != m_Suballocations.begin())
5045  {
5046  --prevItem;
5047  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5048  {
5049  mergeWithPrev = true;
5050  }
5051  }
5052 
5053  if(mergeWithNext)
5054  {
5055  UnregisterFreeSuballocation(nextItem);
5056  MergeFreeWithNext(suballocItem);
5057  }
5058 
5059  if(mergeWithPrev)
5060  {
5061  UnregisterFreeSuballocation(prevItem);
5062  MergeFreeWithNext(prevItem);
5063  RegisterFreeSuballocation(prevItem);
5064  return prevItem;
5065  }
5066  else
5067  {
5068  RegisterFreeSuballocation(suballocItem);
5069  return suballocItem;
5070  }
5071 }
5072 
5073 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5074 {
5075  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5076  VMA_ASSERT(item->size > 0);
5077 
5078  // You may want to enable this validation at the beginning or at the end of
5079  // this function, depending on what do you want to check.
5080  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5081 
5082  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5083  {
5084  if(m_FreeSuballocationsBySize.empty())
5085  {
5086  m_FreeSuballocationsBySize.push_back(item);
5087  }
5088  else
5089  {
5090  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5091  }
5092  }
5093 
5094  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5095 }
5096 
5097 
5098 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5099 {
5100  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5101  VMA_ASSERT(item->size > 0);
5102 
5103  // You may want to enable this validation at the beginning or at the end of
5104  // this function, depending on what do you want to check.
5105  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5106 
5107  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5108  {
5109  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5110  m_FreeSuballocationsBySize.data(),
5111  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5112  item,
5113  VmaSuballocationItemSizeLess());
5114  for(size_t index = it - m_FreeSuballocationsBySize.data();
5115  index < m_FreeSuballocationsBySize.size();
5116  ++index)
5117  {
5118  if(m_FreeSuballocationsBySize[index] == item)
5119  {
5120  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5121  return;
5122  }
5123  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5124  }
5125  VMA_ASSERT(0 && "Not found.");
5126  }
5127 
5128  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5129 }
5130 
5132 // class VmaDeviceMemoryBlock
5133 
5134 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5135  m_MemoryTypeIndex(UINT32_MAX),
5136  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
5137  m_hMemory(VK_NULL_HANDLE),
5138  m_PersistentMap(false),
5139  m_pMappedData(VMA_NULL),
5140  m_Metadata(hAllocator)
5141 {
5142 }
5143 
5144 void VmaDeviceMemoryBlock::Init(
5145  uint32_t newMemoryTypeIndex,
5146  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
5147  VkDeviceMemory newMemory,
5148  VkDeviceSize newSize,
5149  bool persistentMap,
5150  void* pMappedData)
5151 {
5152  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5153 
5154  m_MemoryTypeIndex = newMemoryTypeIndex;
5155  m_BlockVectorType = newBlockVectorType;
5156  m_hMemory = newMemory;
5157  m_PersistentMap = persistentMap;
5158  m_pMappedData = pMappedData;
5159 
5160  m_Metadata.Init(newSize);
5161 }
5162 
5163 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5164 {
5165  // This is the most important assert in the entire library.
5166  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5167  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5168 
5169  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5170  if(m_pMappedData != VMA_NULL)
5171  {
5172  (allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, m_hMemory);
5173  m_pMappedData = VMA_NULL;
5174  }
5175 
5176  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5177  m_hMemory = VK_NULL_HANDLE;
5178 }
5179 
5180 bool VmaDeviceMemoryBlock::Validate() const
5181 {
5182  if((m_hMemory == VK_NULL_HANDLE) ||
5183  (m_Metadata.GetSize() == 0))
5184  {
5185  return false;
5186  }
5187 
5188  return m_Metadata.Validate();
5189 }
5190 
5191 static void InitStatInfo(VmaStatInfo& outInfo)
5192 {
5193  memset(&outInfo, 0, sizeof(outInfo));
5194  outInfo.allocationSizeMin = UINT64_MAX;
5195  outInfo.unusedRangeSizeMin = UINT64_MAX;
5196 }
5197 
5198 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5199 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5200 {
5201  inoutInfo.blockCount += srcInfo.blockCount;
5202  inoutInfo.allocationCount += srcInfo.allocationCount;
5203  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
5204  inoutInfo.usedBytes += srcInfo.usedBytes;
5205  inoutInfo.unusedBytes += srcInfo.unusedBytes;
5206  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
5207  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
5208  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
5209  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
5210 }
5211 
5212 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
5213 {
5214  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
5215  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
5216  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
5217  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
5218 }
5219 
5220 VmaPool_T::VmaPool_T(
5221  VmaAllocator hAllocator,
5222  const VmaPoolCreateInfo& createInfo) :
5223  m_BlockVector(
5224  hAllocator,
5225  createInfo.memoryTypeIndex,
5226  (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ?
5227  VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
5228  createInfo.blockSize,
5229  createInfo.minBlockCount,
5230  createInfo.maxBlockCount,
5231  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
5232  createInfo.frameInUseCount,
5233  true) // isCustomPool
5234 {
5235 }
5236 
5237 VmaPool_T::~VmaPool_T()
5238 {
5239 }
5240 
5241 #if VMA_STATS_STRING_ENABLED
5242 
5243 #endif // #if VMA_STATS_STRING_ENABLED
5244 
5245 VmaBlockVector::VmaBlockVector(
5246  VmaAllocator hAllocator,
5247  uint32_t memoryTypeIndex,
5248  VMA_BLOCK_VECTOR_TYPE blockVectorType,
5249  VkDeviceSize preferredBlockSize,
5250  size_t minBlockCount,
5251  size_t maxBlockCount,
5252  VkDeviceSize bufferImageGranularity,
5253  uint32_t frameInUseCount,
5254  bool isCustomPool) :
5255  m_hAllocator(hAllocator),
5256  m_MemoryTypeIndex(memoryTypeIndex),
5257  m_BlockVectorType(blockVectorType),
5258  m_PreferredBlockSize(preferredBlockSize),
5259  m_MinBlockCount(minBlockCount),
5260  m_MaxBlockCount(maxBlockCount),
5261  m_BufferImageGranularity(bufferImageGranularity),
5262  m_FrameInUseCount(frameInUseCount),
5263  m_IsCustomPool(isCustomPool),
5264  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
5265  m_HasEmptyBlock(false),
5266  m_pDefragmentator(VMA_NULL)
5267 {
5268 }
5269 
5270 VmaBlockVector::~VmaBlockVector()
5271 {
5272  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
5273 
5274  for(size_t i = m_Blocks.size(); i--; )
5275  {
5276  m_Blocks[i]->Destroy(m_hAllocator);
5277  vma_delete(m_hAllocator, m_Blocks[i]);
5278  }
5279 }
5280 
5281 VkResult VmaBlockVector::CreateMinBlocks()
5282 {
5283  for(size_t i = 0; i < m_MinBlockCount; ++i)
5284  {
5285  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
5286  if(res != VK_SUCCESS)
5287  {
5288  return res;
5289  }
5290  }
5291  return VK_SUCCESS;
5292 }
5293 
5294 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
5295 {
5296  pStats->size = 0;
5297  pStats->unusedSize = 0;
5298  pStats->allocationCount = 0;
5299  pStats->unusedRangeCount = 0;
5300  pStats->unusedRangeSizeMax = 0;
5301 
5302  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5303 
5304  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5305  {
5306  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5307  VMA_ASSERT(pBlock);
5308  VMA_HEAVY_ASSERT(pBlock->Validate());
5309  pBlock->m_Metadata.AddPoolStats(*pStats);
5310  }
5311 }
5312 
5313 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
5314 
5315 VkResult VmaBlockVector::Allocate(
5316  VmaPool hCurrentPool,
5317  uint32_t currentFrameIndex,
5318  const VkMemoryRequirements& vkMemReq,
5319  const VmaAllocationCreateInfo& createInfo,
5320  VmaSuballocationType suballocType,
5321  VmaAllocation* pAllocation)
5322 {
5323  // Validate flags.
5324  if(createInfo.pool != VK_NULL_HANDLE &&
5325  ((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) != (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED))
5326  {
5327  VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT.");
5328  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5329  }
5330 
5331  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5332 
5333  // 1. Search existing allocations. Try to allocate without making other allocations lost.
5334  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5335  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5336  {
5337  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5338  VMA_ASSERT(pCurrBlock);
5339  VmaAllocationRequest currRequest = {};
5340  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5341  currentFrameIndex,
5342  m_FrameInUseCount,
5343  m_BufferImageGranularity,
5344  vkMemReq.size,
5345  vkMemReq.alignment,
5346  suballocType,
5347  false, // canMakeOtherLost
5348  &currRequest))
5349  {
5350  // Allocate from pCurrBlock.
5351  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
5352 
5353  // We no longer have an empty Allocation.
5354  if(pCurrBlock->m_Metadata.IsEmpty())
5355  {
5356  m_HasEmptyBlock = false;
5357  }
5358 
5359  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5360  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
5361  (*pAllocation)->InitBlockAllocation(
5362  hCurrentPool,
5363  pCurrBlock,
5364  currRequest.offset,
5365  vkMemReq.alignment,
5366  vkMemReq.size,
5367  suballocType,
5368  createInfo.pUserData,
5369  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5370  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
5371  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5372  return VK_SUCCESS;
5373  }
5374  }
5375 
5376  const bool canCreateNewBlock =
5377  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
5378  (m_Blocks.size() < m_MaxBlockCount);
5379 
5380  // 2. Try to create new block.
5381  if(canCreateNewBlock)
5382  {
5383  // 2.1. Start with full preferredBlockSize.
5384  VkDeviceSize blockSize = m_PreferredBlockSize;
5385  size_t newBlockIndex = 0;
5386  VkResult res = CreateBlock(blockSize, &newBlockIndex);
5387  // Allocating blocks of other sizes is allowed only in default pools.
5388  // In custom pools block size is fixed.
5389  if(res < 0 && m_IsCustomPool == false)
5390  {
5391  // 2.2. Try half the size.
5392  blockSize /= 2;
5393  if(blockSize >= vkMemReq.size)
5394  {
5395  res = CreateBlock(blockSize, &newBlockIndex);
5396  if(res < 0)
5397  {
5398  // 2.3. Try quarter the size.
5399  blockSize /= 2;
5400  if(blockSize >= vkMemReq.size)
5401  {
5402  res = CreateBlock(blockSize, &newBlockIndex);
5403  }
5404  }
5405  }
5406  }
5407  if(res == VK_SUCCESS)
5408  {
5409  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
5410  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
5411 
5412  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
5413  VmaAllocationRequest allocRequest;
5414  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
5415  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5416  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
5417  (*pAllocation)->InitBlockAllocation(
5418  hCurrentPool,
5419  pBlock,
5420  allocRequest.offset,
5421  vkMemReq.alignment,
5422  vkMemReq.size,
5423  suballocType,
5424  createInfo.pUserData,
5425  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5426  VMA_HEAVY_ASSERT(pBlock->Validate());
5427  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
5428 
5429  return VK_SUCCESS;
5430  }
5431  }
5432 
5433  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
5434 
5435  // 3. Try to allocate from existing blocks with making other allocations lost.
5436  if(canMakeOtherLost)
5437  {
5438  uint32_t tryIndex = 0;
5439  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
5440  {
5441  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
5442  VmaAllocationRequest bestRequest = {};
5443  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
5444 
5445  // 1. Search existing allocations.
5446  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5447  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5448  {
5449  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5450  VMA_ASSERT(pCurrBlock);
5451  VmaAllocationRequest currRequest = {};
5452  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5453  currentFrameIndex,
5454  m_FrameInUseCount,
5455  m_BufferImageGranularity,
5456  vkMemReq.size,
5457  vkMemReq.alignment,
5458  suballocType,
5459  canMakeOtherLost,
5460  &currRequest))
5461  {
5462  const VkDeviceSize currRequestCost = currRequest.CalcCost();
5463  if(pBestRequestBlock == VMA_NULL ||
5464  currRequestCost < bestRequestCost)
5465  {
5466  pBestRequestBlock = pCurrBlock;
5467  bestRequest = currRequest;
5468  bestRequestCost = currRequestCost;
5469 
5470  if(bestRequestCost == 0)
5471  {
5472  break;
5473  }
5474  }
5475  }
5476  }
5477 
5478  if(pBestRequestBlock != VMA_NULL)
5479  {
5480  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
5481  currentFrameIndex,
5482  m_FrameInUseCount,
5483  &bestRequest))
5484  {
5485  // We no longer have an empty Allocation.
5486  if(pBestRequestBlock->m_Metadata.IsEmpty())
5487  {
5488  m_HasEmptyBlock = false;
5489  }
5490  // Allocate from this pBlock.
5491  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5492  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
5493  (*pAllocation)->InitBlockAllocation(
5494  hCurrentPool,
5495  pBestRequestBlock,
5496  bestRequest.offset,
5497  vkMemReq.alignment,
5498  vkMemReq.size,
5499  suballocType,
5500  createInfo.pUserData,
5501  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5502  VMA_HEAVY_ASSERT(pBlock->Validate());
5503  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5504  return VK_SUCCESS;
5505  }
5506  // else: Some allocations must have been touched while we are here. Next try.
5507  }
5508  else
5509  {
5510  // Could not find place in any of the blocks - break outer loop.
5511  break;
5512  }
5513  }
5514  /* Maximum number of tries exceeded - a very unlike event when many other
5515  threads are simultaneously touching allocations making it impossible to make
5516  lost at the same time as we try to allocate. */
5517  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
5518  {
5519  return VK_ERROR_TOO_MANY_OBJECTS;
5520  }
5521  }
5522 
5523  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5524 }
5525 
5526 void VmaBlockVector::Free(
5527  VmaAllocation hAllocation)
5528 {
5529  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
5530 
5531  // Scope for lock.
5532  {
5533  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5534 
5535  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
5536 
5537  pBlock->m_Metadata.Free(hAllocation);
5538  VMA_HEAVY_ASSERT(pBlock->Validate());
5539 
5540  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
5541 
5542  // pBlock became empty after this deallocation.
5543  if(pBlock->m_Metadata.IsEmpty())
5544  {
5545  // Already has empty Allocation. We don't want to have two, so delete this one.
5546  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
5547  {
5548  pBlockToDelete = pBlock;
5549  Remove(pBlock);
5550  }
5551  // We now have first empty Allocation.
5552  else
5553  {
5554  m_HasEmptyBlock = true;
5555  }
5556  }
5557  // pBlock didn't become empty, but we have another empty block - find and free that one.
5558  // (This is optional, heuristics.)
5559  else if(m_HasEmptyBlock)
5560  {
5561  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
5562  if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
5563  {
5564  pBlockToDelete = pLastBlock;
5565  m_Blocks.pop_back();
5566  m_HasEmptyBlock = false;
5567  }
5568  }
5569 
5570  IncrementallySortBlocks();
5571  }
5572 
5573  // Destruction of a free Allocation. Deferred until this point, outside of mutex
5574  // lock, for performance reason.
5575  if(pBlockToDelete != VMA_NULL)
5576  {
5577  VMA_DEBUG_LOG(" Deleted empty allocation");
5578  pBlockToDelete->Destroy(m_hAllocator);
5579  vma_delete(m_hAllocator, pBlockToDelete);
5580  }
5581 }
5582 
5583 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
5584 {
5585  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5586  {
5587  if(m_Blocks[blockIndex] == pBlock)
5588  {
5589  VmaVectorRemove(m_Blocks, blockIndex);
5590  return;
5591  }
5592  }
5593  VMA_ASSERT(0);
5594 }
5595 
5596 void VmaBlockVector::IncrementallySortBlocks()
5597 {
5598  // Bubble sort only until first swap.
5599  for(size_t i = 1; i < m_Blocks.size(); ++i)
5600  {
5601  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
5602  {
5603  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
5604  return;
5605  }
5606  }
5607 }
5608 
5609 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
5610 {
5611  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
5612  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
5613  allocInfo.allocationSize = blockSize;
5614  VkDeviceMemory mem = VK_NULL_HANDLE;
5615  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
5616  if(res < 0)
5617  {
5618  return res;
5619  }
5620 
5621  // New VkDeviceMemory successfully created.
5622 
5623  // Map memory if needed.
5624  void* pMappedData = VMA_NULL;
5625  const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED);
5626  if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0)
5627  {
5628  res = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5629  m_hAllocator->m_hDevice,
5630  mem,
5631  0,
5632  VK_WHOLE_SIZE,
5633  0,
5634  &pMappedData);
5635  if(res < 0)
5636  {
5637  VMA_DEBUG_LOG(" vkMapMemory FAILED");
5638  m_hAllocator->FreeVulkanMemory(m_MemoryTypeIndex, blockSize, mem);
5639  return res;
5640  }
5641  }
5642 
5643  // Create new Allocation for it.
5644  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
5645  pBlock->Init(
5646  m_MemoryTypeIndex,
5647  (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType,
5648  mem,
5649  allocInfo.allocationSize,
5650  persistentMap,
5651  pMappedData);
5652 
5653  m_Blocks.push_back(pBlock);
5654  if(pNewBlockIndex != VMA_NULL)
5655  {
5656  *pNewBlockIndex = m_Blocks.size() - 1;
5657  }
5658 
5659  return VK_SUCCESS;
5660 }
5661 
5662 #if VMA_STATS_STRING_ENABLED
5663 
5664 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
5665 {
5666  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5667 
5668  json.BeginObject();
5669 
5670  if(m_IsCustomPool)
5671  {
5672  json.WriteString("MemoryTypeIndex");
5673  json.WriteNumber(m_MemoryTypeIndex);
5674 
5675  if(m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
5676  {
5677  json.WriteString("Mapped");
5678  json.WriteBool(true);
5679  }
5680 
5681  json.WriteString("BlockSize");
5682  json.WriteNumber(m_PreferredBlockSize);
5683 
5684  json.WriteString("BlockCount");
5685  json.BeginObject(true);
5686  if(m_MinBlockCount > 0)
5687  {
5688  json.WriteString("Min");
5689  json.WriteNumber(m_MinBlockCount);
5690  }
5691  if(m_MaxBlockCount < SIZE_MAX)
5692  {
5693  json.WriteString("Max");
5694  json.WriteNumber(m_MaxBlockCount);
5695  }
5696  json.WriteString("Cur");
5697  json.WriteNumber(m_Blocks.size());
5698  json.EndObject();
5699 
5700  if(m_FrameInUseCount > 0)
5701  {
5702  json.WriteString("FrameInUseCount");
5703  json.WriteNumber(m_FrameInUseCount);
5704  }
5705  }
5706  else
5707  {
5708  json.WriteString("PreferredBlockSize");
5709  json.WriteNumber(m_PreferredBlockSize);
5710  }
5711 
5712  json.WriteString("Blocks");
5713  json.BeginArray();
5714  for(size_t i = 0; i < m_Blocks.size(); ++i)
5715  {
5716  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
5717  }
5718  json.EndArray();
5719 
5720  json.EndObject();
5721 }
5722 
5723 #endif // #if VMA_STATS_STRING_ENABLED
5724 
5725 void VmaBlockVector::UnmapPersistentlyMappedMemory()
5726 {
5727  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5728 
5729  for(size_t i = m_Blocks.size(); i--; )
5730  {
5731  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5732  if(pBlock->m_pMappedData != VMA_NULL)
5733  {
5734  VMA_ASSERT(pBlock->m_PersistentMap != false);
5735  (m_hAllocator->GetVulkanFunctions().vkUnmapMemory)(m_hAllocator->m_hDevice, pBlock->m_hMemory);
5736  pBlock->m_pMappedData = VMA_NULL;
5737  }
5738  }
5739 }
5740 
5741 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
5742 {
5743  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5744 
5745  VkResult finalResult = VK_SUCCESS;
5746  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
5747  {
5748  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5749  if(pBlock->m_PersistentMap)
5750  {
5751  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
5752  VkResult localResult = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5753  m_hAllocator->m_hDevice,
5754  pBlock->m_hMemory,
5755  0,
5756  VK_WHOLE_SIZE,
5757  0,
5758  &pBlock->m_pMappedData);
5759  if(localResult != VK_SUCCESS)
5760  {
5761  finalResult = localResult;
5762  }
5763  }
5764  }
5765  return finalResult;
5766 }
5767 
5768 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
5769  VmaAllocator hAllocator,
5770  uint32_t currentFrameIndex)
5771 {
5772  if(m_pDefragmentator == VMA_NULL)
5773  {
5774  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
5775  hAllocator,
5776  this,
5777  currentFrameIndex);
5778  }
5779 
5780  return m_pDefragmentator;
5781 }
5782 
5783 VkResult VmaBlockVector::Defragment(
5784  VmaDefragmentationStats* pDefragmentationStats,
5785  VkDeviceSize& maxBytesToMove,
5786  uint32_t& maxAllocationsToMove)
5787 {
5788  if(m_pDefragmentator == VMA_NULL)
5789  {
5790  return VK_SUCCESS;
5791  }
5792 
5793  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5794 
5795  // Defragment.
5796  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
5797 
5798  // Accumulate statistics.
5799  if(pDefragmentationStats != VMA_NULL)
5800  {
5801  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
5802  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
5803  pDefragmentationStats->bytesMoved += bytesMoved;
5804  pDefragmentationStats->allocationsMoved += allocationsMoved;
5805  VMA_ASSERT(bytesMoved <= maxBytesToMove);
5806  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
5807  maxBytesToMove -= bytesMoved;
5808  maxAllocationsToMove -= allocationsMoved;
5809  }
5810 
5811  // Free empty blocks.
5812  m_HasEmptyBlock = false;
5813  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
5814  {
5815  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
5816  if(pBlock->m_Metadata.IsEmpty())
5817  {
5818  if(m_Blocks.size() > m_MinBlockCount)
5819  {
5820  if(pDefragmentationStats != VMA_NULL)
5821  {
5822  ++pDefragmentationStats->deviceMemoryBlocksFreed;
5823  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
5824  }
5825 
5826  VmaVectorRemove(m_Blocks, blockIndex);
5827  pBlock->Destroy(m_hAllocator);
5828  vma_delete(m_hAllocator, pBlock);
5829  }
5830  else
5831  {
5832  m_HasEmptyBlock = true;
5833  }
5834  }
5835  }
5836 
5837  return result;
5838 }
5839 
5840 void VmaBlockVector::DestroyDefragmentator()
5841 {
5842  if(m_pDefragmentator != VMA_NULL)
5843  {
5844  vma_delete(m_hAllocator, m_pDefragmentator);
5845  m_pDefragmentator = VMA_NULL;
5846  }
5847 }
5848 
5849 void VmaBlockVector::MakePoolAllocationsLost(
5850  uint32_t currentFrameIndex,
5851  size_t* pLostAllocationCount)
5852 {
5853  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5854 
5855  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5856  {
5857  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5858  VMA_ASSERT(pBlock);
5859  pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
5860  }
5861 }
5862 
5863 void VmaBlockVector::AddStats(VmaStats* pStats)
5864 {
5865  const uint32_t memTypeIndex = m_MemoryTypeIndex;
5866  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
5867 
5868  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5869 
5870  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5871  {
5872  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5873  VMA_ASSERT(pBlock);
5874  VMA_HEAVY_ASSERT(pBlock->Validate());
5875  VmaStatInfo allocationStatInfo;
5876  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
5877  VmaAddStatInfo(pStats->total, allocationStatInfo);
5878  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
5879  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
5880  }
5881 }
5882 
5884 // VmaDefragmentator members definition
5885 
5886 VmaDefragmentator::VmaDefragmentator(
5887  VmaAllocator hAllocator,
5888  VmaBlockVector* pBlockVector,
5889  uint32_t currentFrameIndex) :
5890  m_hAllocator(hAllocator),
5891  m_pBlockVector(pBlockVector),
5892  m_CurrentFrameIndex(currentFrameIndex),
5893  m_BytesMoved(0),
5894  m_AllocationsMoved(0),
5895  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
5896  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
5897 {
5898 }
5899 
5900 VmaDefragmentator::~VmaDefragmentator()
5901 {
5902  for(size_t i = m_Blocks.size(); i--; )
5903  {
5904  vma_delete(m_hAllocator, m_Blocks[i]);
5905  }
5906 }
5907 
5908 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
5909 {
5910  AllocationInfo allocInfo;
5911  allocInfo.m_hAllocation = hAlloc;
5912  allocInfo.m_pChanged = pChanged;
5913  m_Allocations.push_back(allocInfo);
5914 }
5915 
5916 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
5917 {
5918  // It has already been mapped for defragmentation.
5919  if(m_pMappedDataForDefragmentation)
5920  {
5921  *ppMappedData = m_pMappedDataForDefragmentation;
5922  return VK_SUCCESS;
5923  }
5924 
5925  // It is persistently mapped.
5926  if(m_pBlock->m_PersistentMap)
5927  {
5928  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
5929  *ppMappedData = m_pBlock->m_pMappedData;
5930  return VK_SUCCESS;
5931  }
5932 
5933  // Map on first usage.
5934  VkResult res = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5935  hAllocator->m_hDevice,
5936  m_pBlock->m_hMemory,
5937  0,
5938  VK_WHOLE_SIZE,
5939  0,
5940  &m_pMappedDataForDefragmentation);
5941  *ppMappedData = m_pMappedDataForDefragmentation;
5942  return res;
5943 }
5944 
5945 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
5946 {
5947  if(m_pMappedDataForDefragmentation != VMA_NULL)
5948  {
5949  (hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_pBlock->m_hMemory);
5950  }
5951 }
5952 
5953 VkResult VmaDefragmentator::DefragmentRound(
5954  VkDeviceSize maxBytesToMove,
5955  uint32_t maxAllocationsToMove)
5956 {
5957  if(m_Blocks.empty())
5958  {
5959  return VK_SUCCESS;
5960  }
5961 
5962  size_t srcBlockIndex = m_Blocks.size() - 1;
5963  size_t srcAllocIndex = SIZE_MAX;
5964  for(;;)
5965  {
5966  // 1. Find next allocation to move.
5967  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
5968  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
5969  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
5970  {
5971  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
5972  {
5973  // Finished: no more allocations to process.
5974  if(srcBlockIndex == 0)
5975  {
5976  return VK_SUCCESS;
5977  }
5978  else
5979  {
5980  --srcBlockIndex;
5981  srcAllocIndex = SIZE_MAX;
5982  }
5983  }
5984  else
5985  {
5986  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
5987  }
5988  }
5989 
5990  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
5991  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
5992 
5993  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
5994  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
5995  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
5996  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
5997 
5998  // 2. Try to find new place for this allocation in preceding or current block.
5999  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
6000  {
6001  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
6002  VmaAllocationRequest dstAllocRequest;
6003  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
6004  m_CurrentFrameIndex,
6005  m_pBlockVector->GetFrameInUseCount(),
6006  m_pBlockVector->GetBufferImageGranularity(),
6007  size,
6008  alignment,
6009  suballocType,
6010  false, // canMakeOtherLost
6011  &dstAllocRequest) &&
6012  MoveMakesSense(
6013  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
6014  {
6015  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
6016 
6017  // Reached limit on number of allocations or bytes to move.
6018  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6019  (m_BytesMoved + size > maxBytesToMove))
6020  {
6021  return VK_INCOMPLETE;
6022  }
6023 
6024  void* pDstMappedData = VMA_NULL;
6025  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6026  if(res != VK_SUCCESS)
6027  {
6028  return res;
6029  }
6030 
6031  void* pSrcMappedData = VMA_NULL;
6032  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6033  if(res != VK_SUCCESS)
6034  {
6035  return res;
6036  }
6037 
6038  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6039  memcpy(
6040  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6041  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6042  static_cast<size_t>(size));
6043 
6044  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6045  pSrcBlockInfo->m_pBlock->m_Metadata.Free(allocInfo.m_hAllocation);
6046 
6047  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6048 
6049  if(allocInfo.m_pChanged != VMA_NULL)
6050  {
6051  *allocInfo.m_pChanged = VK_TRUE;
6052  }
6053 
6054  ++m_AllocationsMoved;
6055  m_BytesMoved += size;
6056 
6057  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6058 
6059  break;
6060  }
6061  }
6062 
6063  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6064 
6065  if(srcAllocIndex > 0)
6066  {
6067  --srcAllocIndex;
6068  }
6069  else
6070  {
6071  if(srcBlockIndex > 0)
6072  {
6073  --srcBlockIndex;
6074  srcAllocIndex = SIZE_MAX;
6075  }
6076  else
6077  {
6078  return VK_SUCCESS;
6079  }
6080  }
6081  }
6082 }
6083 
6084 VkResult VmaDefragmentator::Defragment(
6085  VkDeviceSize maxBytesToMove,
6086  uint32_t maxAllocationsToMove)
6087 {
6088  if(m_Allocations.empty())
6089  {
6090  return VK_SUCCESS;
6091  }
6092 
6093  // Create block info for each block.
6094  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6095  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6096  {
6097  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6098  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6099  m_Blocks.push_back(pBlockInfo);
6100  }
6101 
6102  // Sort them by m_pBlock pointer value.
6103  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6104 
6105  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6106  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6107  {
6108  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6109  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6110  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6111  {
6112  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6113  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6114  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6115  {
6116  (*it)->m_Allocations.push_back(allocInfo);
6117  }
6118  else
6119  {
6120  VMA_ASSERT(0);
6121  }
6122  }
6123  }
6124  m_Allocations.clear();
6125 
6126  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6127  {
6128  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6129  pBlockInfo->CalcHasNonMovableAllocations();
6130  pBlockInfo->SortAllocationsBySizeDescecnding();
6131  }
6132 
6133  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6134  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6135 
6136  // Execute defragmentation rounds (the main part).
6137  VkResult result = VK_SUCCESS;
6138  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6139  {
6140  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6141  }
6142 
6143  // Unmap blocks that were mapped for defragmentation.
6144  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6145  {
6146  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6147  }
6148 
6149  return result;
6150 }
6151 
6152 bool VmaDefragmentator::MoveMakesSense(
6153  size_t dstBlockIndex, VkDeviceSize dstOffset,
6154  size_t srcBlockIndex, VkDeviceSize srcOffset)
6155 {
6156  if(dstBlockIndex < srcBlockIndex)
6157  {
6158  return true;
6159  }
6160  if(dstBlockIndex > srcBlockIndex)
6161  {
6162  return false;
6163  }
6164  if(dstOffset < srcOffset)
6165  {
6166  return true;
6167  }
6168  return false;
6169 }
6170 
6172 // VmaAllocator_T
6173 
6174 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6175  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6176  m_PhysicalDevice(pCreateInfo->physicalDevice),
6177  m_hDevice(pCreateInfo->device),
6178  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6179  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6180  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6181  m_UnmapPersistentlyMappedMemoryCounter(0),
6182  m_PreferredLargeHeapBlockSize(0),
6183  m_PreferredSmallHeapBlockSize(0),
6184  m_CurrentFrameIndex(0),
6185  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6186 {
6187  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6188 
6189  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6190  memset(&m_MemProps, 0, sizeof(m_MemProps));
6191  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6192 
6193  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6194  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
6195 
6196  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6197  {
6198  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6199  }
6200 
6201  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6202  {
6203  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6204  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6205  }
6206 
6207  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6208 
6209  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6210  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6211 
6212  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6213  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6214  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
6215  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
6216 
6217  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
6218  {
6219  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
6220  {
6221  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
6222  if(limit != VK_WHOLE_SIZE)
6223  {
6224  m_HeapSizeLimit[heapIndex] = limit;
6225  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
6226  {
6227  m_MemProps.memoryHeaps[heapIndex].size = limit;
6228  }
6229  }
6230  }
6231  }
6232 
6233  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6234  {
6235  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
6236 
6237  for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex)
6238  {
6239  m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)(
6240  this,
6241  memTypeIndex,
6242  static_cast<VMA_BLOCK_VECTOR_TYPE>(blockVectorTypeIndex),
6243  preferredBlockSize,
6244  0,
6245  SIZE_MAX,
6246  GetBufferImageGranularity(),
6247  pCreateInfo->frameInUseCount,
6248  false); // isCustomPool
6249  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
6250  // becase minBlockCount is 0.
6251  m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
6252  }
6253  }
6254 }
6255 
6256 VmaAllocator_T::~VmaAllocator_T()
6257 {
6258  VMA_ASSERT(m_Pools.empty());
6259 
6260  for(size_t i = GetMemoryTypeCount(); i--; )
6261  {
6262  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
6263  {
6264  vma_delete(this, m_pOwnAllocations[i][j]);
6265  vma_delete(this, m_pBlockVectors[i][j]);
6266  }
6267  }
6268 }
6269 
6270 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
6271 {
6272 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6273  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
6274  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
6275  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
6276  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
6277  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
6278  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
6279  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
6280  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
6281  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
6282  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
6283  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
6284  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
6285  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
6286  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
6287 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6288 
6289  if(pVulkanFunctions != VMA_NULL)
6290  {
6291  m_VulkanFunctions = *pVulkanFunctions;
6292  }
6293 
6294  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
6295  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
6296  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
6297  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
6298  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
6299  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
6300  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
6301  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
6302  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
6303  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
6304  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
6305  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
6306  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
6307  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
6308  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
6309  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
6310 }
6311 
6312 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
6313 {
6314  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6315  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
6316  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
6317  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
6318 }
6319 
6320 VkResult VmaAllocator_T::AllocateMemoryOfType(
6321  const VkMemoryRequirements& vkMemReq,
6322  const VmaAllocationCreateInfo& createInfo,
6323  uint32_t memTypeIndex,
6324  VmaSuballocationType suballocType,
6325  VmaAllocation* pAllocation)
6326 {
6327  VMA_ASSERT(pAllocation != VMA_NULL);
6328  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
6329 
6330  uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags);
6331  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6332  VMA_ASSERT(blockVector);
6333 
6334  VmaAllocationCreateInfo finalCreateInfo = createInfo;
6335 
6336  if(VMA_DEBUG_ALWAYS_OWN_MEMORY)
6337  {
6338  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT;
6339  }
6340 
6341  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
6342  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
6343  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
6344  vkMemReq.size > preferredBlockSize / 2)
6345  {
6346  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT;
6347  }
6348 
6349  // If memory type is not HOST_VISIBLE, disable PERSISTENT_MAP.
6350  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 &&
6351  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
6352  {
6353  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT;
6354  }
6355 
6356  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0)
6357  {
6358  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6359  {
6360  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6361  }
6362  else
6363  {
6364  return AllocateOwnMemory(
6365  vkMemReq.size,
6366  suballocType,
6367  memTypeIndex,
6368  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6369  finalCreateInfo.pUserData,
6370  pAllocation);
6371  }
6372  }
6373  else
6374  {
6375  VkResult res = blockVector->Allocate(
6376  VK_NULL_HANDLE, // hCurrentPool
6377  m_CurrentFrameIndex.load(),
6378  vkMemReq,
6379  finalCreateInfo,
6380  suballocType,
6381  pAllocation);
6382  if(res == VK_SUCCESS)
6383  {
6384  return res;
6385  }
6386 
6387  // 5. Try own memory.
6388  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6389  {
6390  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6391  }
6392  else
6393  {
6394  res = AllocateOwnMemory(
6395  vkMemReq.size,
6396  suballocType,
6397  memTypeIndex,
6398  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6399  finalCreateInfo.pUserData,
6400  pAllocation);
6401  if(res == VK_SUCCESS)
6402  {
6403  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
6404  VMA_DEBUG_LOG(" Allocated as OwnMemory");
6405  return VK_SUCCESS;
6406  }
6407  else
6408  {
6409  // Everything failed: Return error code.
6410  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6411  return res;
6412  }
6413  }
6414  }
6415 }
6416 
6417 VkResult VmaAllocator_T::AllocateOwnMemory(
6418  VkDeviceSize size,
6419  VmaSuballocationType suballocType,
6420  uint32_t memTypeIndex,
6421  bool map,
6422  void* pUserData,
6423  VmaAllocation* pAllocation)
6424 {
6425  VMA_ASSERT(pAllocation);
6426 
6427  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6428  allocInfo.memoryTypeIndex = memTypeIndex;
6429  allocInfo.allocationSize = size;
6430 
6431  // Allocate VkDeviceMemory.
6432  VkDeviceMemory hMemory = VK_NULL_HANDLE;
6433  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
6434  if(res < 0)
6435  {
6436  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6437  return res;
6438  }
6439 
6440  void* pMappedData = nullptr;
6441  if(map)
6442  {
6443  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
6444  {
6445  res = (*m_VulkanFunctions.vkMapMemory)(
6446  m_hDevice,
6447  hMemory,
6448  0,
6449  VK_WHOLE_SIZE,
6450  0,
6451  &pMappedData);
6452  if(res < 0)
6453  {
6454  VMA_DEBUG_LOG(" vkMapMemory FAILED");
6455  FreeVulkanMemory(memTypeIndex, size, hMemory);
6456  return res;
6457  }
6458  }
6459  }
6460 
6461  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load());
6462  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
6463 
6464  // Register it in m_pOwnAllocations.
6465  {
6466  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6467  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
6468  VMA_ASSERT(pOwnAllocations);
6469  VmaVectorInsertSorted<VmaPointerLess>(*pOwnAllocations, *pAllocation);
6470  }
6471 
6472  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
6473 
6474  return VK_SUCCESS;
6475 }
6476 
6477 VkResult VmaAllocator_T::AllocateMemory(
6478  const VkMemoryRequirements& vkMemReq,
6479  const VmaAllocationCreateInfo& createInfo,
6480  VmaSuballocationType suballocType,
6481  VmaAllocation* pAllocation)
6482 {
6483  if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 &&
6484  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6485  {
6486  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
6487  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6488  }
6489  if((createInfo.pool != VK_NULL_HANDLE) &&
6490  ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0))
6491  {
6492  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid.");
6493  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6494  }
6495 
6496  if(createInfo.pool != VK_NULL_HANDLE)
6497  {
6498  return createInfo.pool->m_BlockVector.Allocate(
6499  createInfo.pool,
6500  m_CurrentFrameIndex.load(),
6501  vkMemReq,
6502  createInfo,
6503  suballocType,
6504  pAllocation);
6505  }
6506  else
6507  {
6508  // Bit mask of memory Vulkan types acceptable for this allocation.
6509  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
6510  uint32_t memTypeIndex = UINT32_MAX;
6511  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6512  if(res == VK_SUCCESS)
6513  {
6514  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6515  // Succeeded on first try.
6516  if(res == VK_SUCCESS)
6517  {
6518  return res;
6519  }
6520  // Allocation from this memory type failed. Try other compatible memory types.
6521  else
6522  {
6523  for(;;)
6524  {
6525  // Remove old memTypeIndex from list of possibilities.
6526  memoryTypeBits &= ~(1u << memTypeIndex);
6527  // Find alternative memTypeIndex.
6528  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6529  if(res == VK_SUCCESS)
6530  {
6531  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6532  // Allocation from this alternative memory type succeeded.
6533  if(res == VK_SUCCESS)
6534  {
6535  return res;
6536  }
6537  // else: Allocation from this memory type failed. Try next one - next loop iteration.
6538  }
6539  // No other matching memory type index could be found.
6540  else
6541  {
6542  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
6543  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6544  }
6545  }
6546  }
6547  }
6548  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
6549  else
6550  return res;
6551  }
6552 }
6553 
6554 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
6555 {
6556  VMA_ASSERT(allocation);
6557 
6558  if(allocation->CanBecomeLost() == false ||
6559  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6560  {
6561  switch(allocation->GetType())
6562  {
6563  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
6564  {
6565  VmaBlockVector* pBlockVector = VMA_NULL;
6566  VmaPool hPool = allocation->GetPool();
6567  if(hPool != VK_NULL_HANDLE)
6568  {
6569  pBlockVector = &hPool->m_BlockVector;
6570  }
6571  else
6572  {
6573  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
6574  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
6575  pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6576  }
6577  pBlockVector->Free(allocation);
6578  }
6579  break;
6580  case VmaAllocation_T::ALLOCATION_TYPE_OWN:
6581  FreeOwnMemory(allocation);
6582  break;
6583  default:
6584  VMA_ASSERT(0);
6585  }
6586  }
6587 
6588  vma_delete(this, allocation);
6589 }
6590 
6591 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
6592 {
6593  // Initialize.
6594  InitStatInfo(pStats->total);
6595  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
6596  InitStatInfo(pStats->memoryType[i]);
6597  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6598  InitStatInfo(pStats->memoryHeap[i]);
6599 
6600  // Process default pools.
6601  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6602  {
6603  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6604  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6605  {
6606  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6607  VMA_ASSERT(pBlockVector);
6608  pBlockVector->AddStats(pStats);
6609  }
6610  }
6611 
6612  // Process custom pools.
6613  {
6614  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6615  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6616  {
6617  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
6618  }
6619  }
6620 
6621  // Process own allocations.
6622  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6623  {
6624  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6625  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6626  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6627  {
6628  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
6629  VMA_ASSERT(pOwnAllocVector);
6630  for(size_t allocIndex = 0, allocCount = pOwnAllocVector->size(); allocIndex < allocCount; ++allocIndex)
6631  {
6632  VmaStatInfo allocationStatInfo;
6633  (*pOwnAllocVector)[allocIndex]->OwnAllocCalcStatsInfo(allocationStatInfo);
6634  VmaAddStatInfo(pStats->total, allocationStatInfo);
6635  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6636  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6637  }
6638  }
6639  }
6640 
6641  // Postprocess.
6642  VmaPostprocessCalcStatInfo(pStats->total);
6643  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
6644  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
6645  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
6646  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
6647 }
6648 
6649 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
6650 
6651 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
6652 {
6653  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
6654  {
6655  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6656  {
6657  for(uint32_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
6658  {
6659  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6660  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6661  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6662  {
6663  // Process OwnAllocations.
6664  {
6665  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6666  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6667  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
6668  {
6669  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
6670  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(this);
6671  }
6672  }
6673 
6674  // Process normal Allocations.
6675  {
6676  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6677  pBlockVector->UnmapPersistentlyMappedMemory();
6678  }
6679  }
6680  }
6681 
6682  // Process custom pools.
6683  {
6684  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6685  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6686  {
6687  m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory();
6688  }
6689  }
6690  }
6691  }
6692 }
6693 
6694 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
6695 {
6696  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
6697  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
6698  {
6699  VkResult finalResult = VK_SUCCESS;
6700  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6701  {
6702  // Process custom pools.
6703  {
6704  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6705  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6706  {
6707  m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory();
6708  }
6709  }
6710 
6711  for(uint32_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
6712  {
6713  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6714  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6715  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6716  {
6717  // Process OwnAllocations.
6718  {
6719  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6720  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6721  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
6722  {
6723  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
6724  hAlloc->OwnAllocMapPersistentlyMappedMemory(this);
6725  }
6726  }
6727 
6728  // Process normal Allocations.
6729  {
6730  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6731  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
6732  if(localResult != VK_SUCCESS)
6733  {
6734  finalResult = localResult;
6735  }
6736  }
6737  }
6738  }
6739  }
6740  return finalResult;
6741  }
6742  else
6743  return VK_SUCCESS;
6744 }
6745 
6746 VkResult VmaAllocator_T::Defragment(
6747  VmaAllocation* pAllocations,
6748  size_t allocationCount,
6749  VkBool32* pAllocationsChanged,
6750  const VmaDefragmentationInfo* pDefragmentationInfo,
6751  VmaDefragmentationStats* pDefragmentationStats)
6752 {
6753  if(pAllocationsChanged != VMA_NULL)
6754  {
6755  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
6756  }
6757  if(pDefragmentationStats != VMA_NULL)
6758  {
6759  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
6760  }
6761 
6762  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
6763  {
6764  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
6765  return VK_ERROR_MEMORY_MAP_FAILED;
6766  }
6767 
6768  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
6769 
6770  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
6771 
6772  const size_t poolCount = m_Pools.size();
6773 
6774  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
6775  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
6776  {
6777  VmaAllocation hAlloc = pAllocations[allocIndex];
6778  VMA_ASSERT(hAlloc);
6779  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
6780  // OwnAlloc cannot be defragmented.
6781  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
6782  // Only HOST_VISIBLE memory types can be defragmented.
6783  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
6784  // Lost allocation cannot be defragmented.
6785  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
6786  {
6787  VmaBlockVector* pAllocBlockVector = nullptr;
6788 
6789  const VmaPool hAllocPool = hAlloc->GetPool();
6790  // This allocation belongs to custom pool.
6791  if(hAllocPool != VK_NULL_HANDLE)
6792  {
6793  pAllocBlockVector = &hAllocPool->GetBlockVector();
6794  }
6795  // This allocation belongs to general pool.
6796  else
6797  {
6798  pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()];
6799  }
6800 
6801  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
6802 
6803  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
6804  &pAllocationsChanged[allocIndex] : VMA_NULL;
6805  pDefragmentator->AddAllocation(hAlloc, pChanged);
6806  }
6807  }
6808 
6809  VkResult result = VK_SUCCESS;
6810 
6811  // ======== Main processing.
6812 
6813  VkDeviceSize maxBytesToMove = SIZE_MAX;
6814  uint32_t maxAllocationsToMove = UINT32_MAX;
6815  if(pDefragmentationInfo != VMA_NULL)
6816  {
6817  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
6818  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
6819  }
6820 
6821  // Process standard memory.
6822  for(uint32_t memTypeIndex = 0;
6823  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
6824  ++memTypeIndex)
6825  {
6826  // Only HOST_VISIBLE memory types can be defragmented.
6827  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6828  {
6829  for(uint32_t blockVectorType = 0;
6830  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
6831  ++blockVectorType)
6832  {
6833  result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment(
6834  pDefragmentationStats,
6835  maxBytesToMove,
6836  maxAllocationsToMove);
6837  }
6838  }
6839  }
6840 
6841  // Process custom pools.
6842  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
6843  {
6844  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
6845  pDefragmentationStats,
6846  maxBytesToMove,
6847  maxAllocationsToMove);
6848  }
6849 
6850  // ======== Destroy defragmentators.
6851 
6852  // Process custom pools.
6853  for(size_t poolIndex = poolCount; poolIndex--; )
6854  {
6855  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
6856  }
6857 
6858  // Process standard memory.
6859  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
6860  {
6861  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6862  {
6863  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
6864  {
6865  m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator();
6866  }
6867  }
6868  }
6869 
6870  return result;
6871 }
6872 
6873 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
6874 {
6875  if(hAllocation->CanBecomeLost())
6876  {
6877  /*
6878  Warning: This is a carefully designed algorithm.
6879  Do not modify unless you really know what you're doing :)
6880  */
6881  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
6882  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
6883  for(;;)
6884  {
6885  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6886  {
6887  pAllocationInfo->memoryType = UINT32_MAX;
6888  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
6889  pAllocationInfo->offset = 0;
6890  pAllocationInfo->size = hAllocation->GetSize();
6891  pAllocationInfo->pMappedData = VMA_NULL;
6892  pAllocationInfo->pUserData = hAllocation->GetUserData();
6893  return;
6894  }
6895  else if(localLastUseFrameIndex == localCurrFrameIndex)
6896  {
6897  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6898  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6899  pAllocationInfo->offset = hAllocation->GetOffset();
6900  pAllocationInfo->size = hAllocation->GetSize();
6901  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6902  pAllocationInfo->pUserData = hAllocation->GetUserData();
6903  return;
6904  }
6905  else // Last use time earlier than current time.
6906  {
6907  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
6908  {
6909  localLastUseFrameIndex = localCurrFrameIndex;
6910  }
6911  }
6912  }
6913  }
6914  // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic.
6915  else
6916  {
6917  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6918  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6919  pAllocationInfo->offset = hAllocation->GetOffset();
6920  pAllocationInfo->size = hAllocation->GetSize();
6921  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6922  pAllocationInfo->pUserData = hAllocation->GetUserData();
6923  }
6924 }
6925 
6926 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
6927 {
6928  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
6929 
6930  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
6931 
6932  if(newCreateInfo.maxBlockCount == 0)
6933  {
6934  newCreateInfo.maxBlockCount = SIZE_MAX;
6935  }
6936  if(newCreateInfo.blockSize == 0)
6937  {
6938  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
6939  }
6940 
6941  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
6942 
6943  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
6944  if(res != VK_SUCCESS)
6945  {
6946  vma_delete(this, *pPool);
6947  *pPool = VMA_NULL;
6948  return res;
6949  }
6950 
6951  // Add to m_Pools.
6952  {
6953  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6954  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
6955  }
6956 
6957  return VK_SUCCESS;
6958 }
6959 
6960 void VmaAllocator_T::DestroyPool(VmaPool pool)
6961 {
6962  // Remove from m_Pools.
6963  {
6964  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6965  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
6966  VMA_ASSERT(success && "Pool not found in Allocator.");
6967  }
6968 
6969  vma_delete(this, pool);
6970 }
6971 
6972 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
6973 {
6974  pool->m_BlockVector.GetPoolStats(pPoolStats);
6975 }
6976 
6977 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
6978 {
6979  m_CurrentFrameIndex.store(frameIndex);
6980 }
6981 
6982 void VmaAllocator_T::MakePoolAllocationsLost(
6983  VmaPool hPool,
6984  size_t* pLostAllocationCount)
6985 {
6986  hPool->m_BlockVector.MakePoolAllocationsLost(
6987  m_CurrentFrameIndex.load(),
6988  pLostAllocationCount);
6989 }
6990 
6991 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
6992 {
6993  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST);
6994  (*pAllocation)->InitLost();
6995 }
6996 
6997 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
6998 {
6999  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
7000 
7001  VkResult res;
7002  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7003  {
7004  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7005  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
7006  {
7007  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7008  if(res == VK_SUCCESS)
7009  {
7010  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
7011  }
7012  }
7013  else
7014  {
7015  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
7016  }
7017  }
7018  else
7019  {
7020  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
7021  }
7022 
7023  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
7024  {
7025  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
7026  }
7027 
7028  return res;
7029 }
7030 
7031 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
7032 {
7033  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
7034  {
7035  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
7036  }
7037 
7038  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7039 
7040  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7041  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7042  {
7043  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7044  m_HeapSizeLimit[heapIndex] += size;
7045  }
7046 }
7047 
7048 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
7049 {
7050  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
7051 
7052  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7053  {
7054  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7055  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
7056  VMA_ASSERT(pOwnAllocations);
7057  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pOwnAllocations, allocation);
7058  VMA_ASSERT(success);
7059  }
7060 
7061  VkDeviceMemory hMemory = allocation->GetMemory();
7062 
7063  if(allocation->GetMappedData() != VMA_NULL)
7064  {
7065  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7066  }
7067 
7068  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7069 
7070  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
7071 }
7072 
7073 #if VMA_STATS_STRING_ENABLED
7074 
7075 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7076 {
7077  bool ownAllocationsStarted = false;
7078  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7079  {
7080  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7081  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7082  {
7083  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
7084  VMA_ASSERT(pOwnAllocVector);
7085  if(pOwnAllocVector->empty() == false)
7086  {
7087  if(ownAllocationsStarted == false)
7088  {
7089  ownAllocationsStarted = true;
7090  json.WriteString("OwnAllocations");
7091  json.BeginObject();
7092  }
7093 
7094  json.BeginString("Type ");
7095  json.ContinueString(memTypeIndex);
7096  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7097  {
7098  json.ContinueString(" Mapped");
7099  }
7100  json.EndString();
7101 
7102  json.BeginArray();
7103 
7104  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
7105  {
7106  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
7107  json.BeginObject(true);
7108 
7109  json.WriteString("Size");
7110  json.WriteNumber(hAlloc->GetSize());
7111 
7112  json.WriteString("Type");
7113  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7114 
7115  json.EndObject();
7116  }
7117 
7118  json.EndArray();
7119  }
7120  }
7121  }
7122  if(ownAllocationsStarted)
7123  {
7124  json.EndObject();
7125  }
7126 
7127  {
7128  bool allocationsStarted = false;
7129  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7130  {
7131  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7132  {
7133  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
7134  {
7135  if(allocationsStarted == false)
7136  {
7137  allocationsStarted = true;
7138  json.WriteString("DefaultPools");
7139  json.BeginObject();
7140  }
7141 
7142  json.BeginString("Type ");
7143  json.ContinueString(memTypeIndex);
7144  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7145  {
7146  json.ContinueString(" Mapped");
7147  }
7148  json.EndString();
7149 
7150  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json);
7151  }
7152  }
7153  }
7154  if(allocationsStarted)
7155  {
7156  json.EndObject();
7157  }
7158  }
7159 
7160  {
7161  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7162  const size_t poolCount = m_Pools.size();
7163  if(poolCount > 0)
7164  {
7165  json.WriteString("Pools");
7166  json.BeginArray();
7167  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
7168  {
7169  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
7170  }
7171  json.EndArray();
7172  }
7173  }
7174 }
7175 
7176 #endif // #if VMA_STATS_STRING_ENABLED
7177 
7178 static VkResult AllocateMemoryForImage(
7179  VmaAllocator allocator,
7180  VkImage image,
7181  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7182  VmaSuballocationType suballocType,
7183  VmaAllocation* pAllocation)
7184 {
7185  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
7186 
7187  VkMemoryRequirements vkMemReq = {};
7188  (*allocator->GetVulkanFunctions().vkGetImageMemoryRequirements)(allocator->m_hDevice, image, &vkMemReq);
7189 
7190  return allocator->AllocateMemory(
7191  vkMemReq,
7192  *pAllocationCreateInfo,
7193  suballocType,
7194  pAllocation);
7195 }
7196 
7198 // Public interface
7199 
7200 VkResult vmaCreateAllocator(
7201  const VmaAllocatorCreateInfo* pCreateInfo,
7202  VmaAllocator* pAllocator)
7203 {
7204  VMA_ASSERT(pCreateInfo && pAllocator);
7205  VMA_DEBUG_LOG("vmaCreateAllocator");
7206  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
7207  return VK_SUCCESS;
7208 }
7209 
7210 void vmaDestroyAllocator(
7211  VmaAllocator allocator)
7212 {
7213  if(allocator != VK_NULL_HANDLE)
7214  {
7215  VMA_DEBUG_LOG("vmaDestroyAllocator");
7216  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
7217  vma_delete(&allocationCallbacks, allocator);
7218  }
7219 }
7220 
7221 void vmaGetPhysicalDeviceProperties(
7222  VmaAllocator allocator,
7223  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
7224 {
7225  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
7226  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
7227 }
7228 
7229 void vmaGetMemoryProperties(
7230  VmaAllocator allocator,
7231  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
7232 {
7233  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
7234  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
7235 }
7236 
7237 void vmaGetMemoryTypeProperties(
7238  VmaAllocator allocator,
7239  uint32_t memoryTypeIndex,
7240  VkMemoryPropertyFlags* pFlags)
7241 {
7242  VMA_ASSERT(allocator && pFlags);
7243  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
7244  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
7245 }
7246 
7247 void vmaSetCurrentFrameIndex(
7248  VmaAllocator allocator,
7249  uint32_t frameIndex)
7250 {
7251  VMA_ASSERT(allocator);
7252  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
7253 
7254  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7255 
7256  allocator->SetCurrentFrameIndex(frameIndex);
7257 }
7258 
7259 void vmaCalculateStats(
7260  VmaAllocator allocator,
7261  VmaStats* pStats)
7262 {
7263  VMA_ASSERT(allocator && pStats);
7264  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7265  allocator->CalculateStats(pStats);
7266 }
7267 
7268 #if VMA_STATS_STRING_ENABLED
7269 
7270 void vmaBuildStatsString(
7271  VmaAllocator allocator,
7272  char** ppStatsString,
7273  VkBool32 detailedMap)
7274 {
7275  VMA_ASSERT(allocator && ppStatsString);
7276  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7277 
7278  VmaStringBuilder sb(allocator);
7279  {
7280  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
7281  json.BeginObject();
7282 
7283  VmaStats stats;
7284  allocator->CalculateStats(&stats);
7285 
7286  json.WriteString("Total");
7287  VmaPrintStatInfo(json, stats.total);
7288 
7289  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
7290  {
7291  json.BeginString("Heap ");
7292  json.ContinueString(heapIndex);
7293  json.EndString();
7294  json.BeginObject();
7295 
7296  json.WriteString("Size");
7297  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
7298 
7299  json.WriteString("Flags");
7300  json.BeginArray(true);
7301  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
7302  {
7303  json.WriteString("DEVICE_LOCAL");
7304  }
7305  json.EndArray();
7306 
7307  if(stats.memoryHeap[heapIndex].blockCount > 0)
7308  {
7309  json.WriteString("Stats");
7310  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
7311  }
7312 
7313  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
7314  {
7315  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
7316  {
7317  json.BeginString("Type ");
7318  json.ContinueString(typeIndex);
7319  json.EndString();
7320 
7321  json.BeginObject();
7322 
7323  json.WriteString("Flags");
7324  json.BeginArray(true);
7325  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
7326  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
7327  {
7328  json.WriteString("DEVICE_LOCAL");
7329  }
7330  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7331  {
7332  json.WriteString("HOST_VISIBLE");
7333  }
7334  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
7335  {
7336  json.WriteString("HOST_COHERENT");
7337  }
7338  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
7339  {
7340  json.WriteString("HOST_CACHED");
7341  }
7342  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
7343  {
7344  json.WriteString("LAZILY_ALLOCATED");
7345  }
7346  json.EndArray();
7347 
7348  if(stats.memoryType[typeIndex].blockCount > 0)
7349  {
7350  json.WriteString("Stats");
7351  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
7352  }
7353 
7354  json.EndObject();
7355  }
7356  }
7357 
7358  json.EndObject();
7359  }
7360  if(detailedMap == VK_TRUE)
7361  {
7362  allocator->PrintDetailedMap(json);
7363  }
7364 
7365  json.EndObject();
7366  }
7367 
7368  const size_t len = sb.GetLength();
7369  char* const pChars = vma_new_array(allocator, char, len + 1);
7370  if(len > 0)
7371  {
7372  memcpy(pChars, sb.GetData(), len);
7373  }
7374  pChars[len] = '\0';
7375  *ppStatsString = pChars;
7376 }
7377 
7378 void vmaFreeStatsString(
7379  VmaAllocator allocator,
7380  char* pStatsString)
7381 {
7382  if(pStatsString != VMA_NULL)
7383  {
7384  VMA_ASSERT(allocator);
7385  size_t len = strlen(pStatsString);
7386  vma_delete_array(allocator, pStatsString, len + 1);
7387  }
7388 }
7389 
7390 #endif // #if VMA_STATS_STRING_ENABLED
7391 
7394 VkResult vmaFindMemoryTypeIndex(
7395  VmaAllocator allocator,
7396  uint32_t memoryTypeBits,
7397  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7398  uint32_t* pMemoryTypeIndex)
7399 {
7400  VMA_ASSERT(allocator != VK_NULL_HANDLE);
7401  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
7402  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
7403 
7404  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
7405  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
7406  if(preferredFlags == 0)
7407  {
7408  preferredFlags = requiredFlags;
7409  }
7410  // preferredFlags, if not 0, must be a superset of requiredFlags.
7411  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
7412 
7413  // Convert usage to requiredFlags and preferredFlags.
7414  switch(pAllocationCreateInfo->usage)
7415  {
7416  case VMA_MEMORY_USAGE_UNKNOWN:
7417  break;
7418  case VMA_MEMORY_USAGE_GPU_ONLY:
7419  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7420  break;
7421  case VMA_MEMORY_USAGE_CPU_ONLY:
7422  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7423  break;
7424  case VMA_MEMORY_USAGE_CPU_TO_GPU:
7425  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7426  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7427  break;
7428  case VMA_MEMORY_USAGE_GPU_TO_CPU:
7429  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7430  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
7431  break;
7432  default:
7433  break;
7434  }
7435 
7436  *pMemoryTypeIndex = UINT32_MAX;
7437  uint32_t minCost = UINT32_MAX;
7438  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
7439  memTypeIndex < allocator->GetMemoryTypeCount();
7440  ++memTypeIndex, memTypeBit <<= 1)
7441  {
7442  // This memory type is acceptable according to memoryTypeBits bitmask.
7443  if((memTypeBit & memoryTypeBits) != 0)
7444  {
7445  const VkMemoryPropertyFlags currFlags =
7446  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
7447  // This memory type contains requiredFlags.
7448  if((requiredFlags & ~currFlags) == 0)
7449  {
7450  // Calculate cost as number of bits from preferredFlags not present in this memory type.
7451  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
7452  // Remember memory type with lowest cost.
7453  if(currCost < minCost)
7454  {
7455  *pMemoryTypeIndex = memTypeIndex;
7456  if(currCost == 0)
7457  {
7458  return VK_SUCCESS;
7459  }
7460  minCost = currCost;
7461  }
7462  }
7463  }
7464  }
7465  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
7466 }
7467 
7468 VkResult vmaCreatePool(
7469  VmaAllocator allocator,
7470  const VmaPoolCreateInfo* pCreateInfo,
7471  VmaPool* pPool)
7472 {
7473  VMA_ASSERT(allocator && pCreateInfo && pPool);
7474 
7475  VMA_DEBUG_LOG("vmaCreatePool");
7476 
7477  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7478 
7479  return allocator->CreatePool(pCreateInfo, pPool);
7480 }
7481 
7482 void vmaDestroyPool(
7483  VmaAllocator allocator,
7484  VmaPool pool)
7485 {
7486  VMA_ASSERT(allocator && pool);
7487 
7488  VMA_DEBUG_LOG("vmaDestroyPool");
7489 
7490  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7491 
7492  allocator->DestroyPool(pool);
7493 }
7494 
7495 void vmaGetPoolStats(
7496  VmaAllocator allocator,
7497  VmaPool pool,
7498  VmaPoolStats* pPoolStats)
7499 {
7500  VMA_ASSERT(allocator && pool && pPoolStats);
7501 
7502  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7503 
7504  allocator->GetPoolStats(pool, pPoolStats);
7505 }
7506 
7507 void vmaMakePoolAllocationsLost(
7508  VmaAllocator allocator,
7509  VmaPool pool,
7510  size_t* pLostAllocationCount)
7511 {
7512  VMA_ASSERT(allocator && pool);
7513 
7514  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7515 
7516  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
7517 }
7518 
7519 VkResult vmaAllocateMemory(
7520  VmaAllocator allocator,
7521  const VkMemoryRequirements* pVkMemoryRequirements,
7522  const VmaAllocationCreateInfo* pCreateInfo,
7523  VmaAllocation* pAllocation,
7524  VmaAllocationInfo* pAllocationInfo)
7525 {
7526  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
7527 
7528  VMA_DEBUG_LOG("vmaAllocateMemory");
7529 
7530  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7531 
7532  VkResult result = allocator->AllocateMemory(
7533  *pVkMemoryRequirements,
7534  *pCreateInfo,
7535  VMA_SUBALLOCATION_TYPE_UNKNOWN,
7536  pAllocation);
7537 
7538  if(pAllocationInfo && result == VK_SUCCESS)
7539  {
7540  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7541  }
7542 
7543  return result;
7544 }
7545 
7546 VkResult vmaAllocateMemoryForBuffer(
7547  VmaAllocator allocator,
7548  VkBuffer buffer,
7549  const VmaAllocationCreateInfo* pCreateInfo,
7550  VmaAllocation* pAllocation,
7551  VmaAllocationInfo* pAllocationInfo)
7552 {
7553  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7554 
7555  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
7556 
7557  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7558 
7559  VkMemoryRequirements vkMemReq = {};
7560  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, buffer, &vkMemReq);
7561 
7562  VkResult result = allocator->AllocateMemory(
7563  vkMemReq,
7564  *pCreateInfo,
7565  VMA_SUBALLOCATION_TYPE_BUFFER,
7566  pAllocation);
7567 
7568  if(pAllocationInfo && result == VK_SUCCESS)
7569  {
7570  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7571  }
7572 
7573  return result;
7574 }
7575 
7576 VkResult vmaAllocateMemoryForImage(
7577  VmaAllocator allocator,
7578  VkImage image,
7579  const VmaAllocationCreateInfo* pCreateInfo,
7580  VmaAllocation* pAllocation,
7581  VmaAllocationInfo* pAllocationInfo)
7582 {
7583  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7584 
7585  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
7586 
7587  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7588 
7589  VkResult result = AllocateMemoryForImage(
7590  allocator,
7591  image,
7592  pCreateInfo,
7593  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
7594  pAllocation);
7595 
7596  if(pAllocationInfo && result == VK_SUCCESS)
7597  {
7598  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7599  }
7600 
7601  return result;
7602 }
7603 
7604 void vmaFreeMemory(
7605  VmaAllocator allocator,
7606  VmaAllocation allocation)
7607 {
7608  VMA_ASSERT(allocator && allocation);
7609 
7610  VMA_DEBUG_LOG("vmaFreeMemory");
7611 
7612  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7613 
7614  allocator->FreeMemory(allocation);
7615 }
7616 
7617 void vmaGetAllocationInfo(
7618  VmaAllocator allocator,
7619  VmaAllocation allocation,
7620  VmaAllocationInfo* pAllocationInfo)
7621 {
7622  VMA_ASSERT(allocator && allocation && pAllocationInfo);
7623 
7624  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7625 
7626  allocator->GetAllocationInfo(allocation, pAllocationInfo);
7627 }
7628 
7629 void vmaSetAllocationUserData(
7630  VmaAllocator allocator,
7631  VmaAllocation allocation,
7632  void* pUserData)
7633 {
7634  VMA_ASSERT(allocator && allocation);
7635 
7636  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7637 
7638  allocation->SetUserData(pUserData);
7639 }
7640 
7641 void vmaCreateLostAllocation(
7642  VmaAllocator allocator,
7643  VmaAllocation* pAllocation)
7644 {
7645  VMA_ASSERT(allocator && pAllocation);
7646 
7647  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
7648 
7649  allocator->CreateLostAllocation(pAllocation);
7650 }
7651 
7652 VkResult vmaMapMemory(
7653  VmaAllocator allocator,
7654  VmaAllocation allocation,
7655  void** ppData)
7656 {
7657  VMA_ASSERT(allocator && allocation && ppData);
7658 
7659  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7660 
7661  return (*allocator->GetVulkanFunctions().vkMapMemory)(
7662  allocator->m_hDevice,
7663  allocation->GetMemory(),
7664  allocation->GetOffset(),
7665  allocation->GetSize(),
7666  0,
7667  ppData);
7668 }
7669 
7670 void vmaUnmapMemory(
7671  VmaAllocator allocator,
7672  VmaAllocation allocation)
7673 {
7674  VMA_ASSERT(allocator && allocation);
7675 
7676  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7677 
7678  (*allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, allocation->GetMemory());
7679 }
7680 
7681 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
7682 {
7683  VMA_ASSERT(allocator);
7684 
7685  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7686 
7687  allocator->UnmapPersistentlyMappedMemory();
7688 }
7689 
7690 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
7691 {
7692  VMA_ASSERT(allocator);
7693 
7694  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7695 
7696  return allocator->MapPersistentlyMappedMemory();
7697 }
7698 
7699 VkResult vmaDefragment(
7700  VmaAllocator allocator,
7701  VmaAllocation* pAllocations,
7702  size_t allocationCount,
7703  VkBool32* pAllocationsChanged,
7704  const VmaDefragmentationInfo *pDefragmentationInfo,
7705  VmaDefragmentationStats* pDefragmentationStats)
7706 {
7707  VMA_ASSERT(allocator && pAllocations);
7708 
7709  VMA_DEBUG_LOG("vmaDefragment");
7710 
7711  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7712 
7713  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
7714 }
7715 
7716 VkResult vmaCreateBuffer(
7717  VmaAllocator allocator,
7718  const VkBufferCreateInfo* pBufferCreateInfo,
7719  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7720  VkBuffer* pBuffer,
7721  VmaAllocation* pAllocation,
7722  VmaAllocationInfo* pAllocationInfo)
7723 {
7724  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
7725 
7726  VMA_DEBUG_LOG("vmaCreateBuffer");
7727 
7728  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7729 
7730  *pBuffer = VK_NULL_HANDLE;
7731  *pAllocation = VK_NULL_HANDLE;
7732 
7733  // 1. Create VkBuffer.
7734  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
7735  allocator->m_hDevice,
7736  pBufferCreateInfo,
7737  allocator->GetAllocationCallbacks(),
7738  pBuffer);
7739  if(res >= 0)
7740  {
7741  // 2. vkGetBufferMemoryRequirements.
7742  VkMemoryRequirements vkMemReq = {};
7743  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, *pBuffer, &vkMemReq);
7744 
7745  // 3. Allocate memory using allocator.
7746  res = allocator->AllocateMemory(
7747  vkMemReq,
7748  *pAllocationCreateInfo,
7749  VMA_SUBALLOCATION_TYPE_BUFFER,
7750  pAllocation);
7751  if(res >= 0)
7752  {
7753  // 3. Bind buffer with memory.
7754  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
7755  allocator->m_hDevice,
7756  *pBuffer,
7757  (*pAllocation)->GetMemory(),
7758  (*pAllocation)->GetOffset());
7759  if(res >= 0)
7760  {
7761  // All steps succeeded.
7762  if(pAllocationInfo != VMA_NULL)
7763  {
7764  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7765  }
7766  return VK_SUCCESS;
7767  }
7768  allocator->FreeMemory(*pAllocation);
7769  *pAllocation = VK_NULL_HANDLE;
7770  return res;
7771  }
7772  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
7773  *pBuffer = VK_NULL_HANDLE;
7774  return res;
7775  }
7776  return res;
7777 }
7778 
7779 void vmaDestroyBuffer(
7780  VmaAllocator allocator,
7781  VkBuffer buffer,
7782  VmaAllocation allocation)
7783 {
7784  if(buffer != VK_NULL_HANDLE)
7785  {
7786  VMA_ASSERT(allocator);
7787 
7788  VMA_DEBUG_LOG("vmaDestroyBuffer");
7789 
7790  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7791 
7792  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
7793 
7794  allocator->FreeMemory(allocation);
7795  }
7796 }
7797 
7798 VkResult vmaCreateImage(
7799  VmaAllocator allocator,
7800  const VkImageCreateInfo* pImageCreateInfo,
7801  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7802  VkImage* pImage,
7803  VmaAllocation* pAllocation,
7804  VmaAllocationInfo* pAllocationInfo)
7805 {
7806  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
7807 
7808  VMA_DEBUG_LOG("vmaCreateImage");
7809 
7810  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7811 
7812  *pImage = VK_NULL_HANDLE;
7813  *pAllocation = VK_NULL_HANDLE;
7814 
7815  // 1. Create VkImage.
7816  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
7817  allocator->m_hDevice,
7818  pImageCreateInfo,
7819  allocator->GetAllocationCallbacks(),
7820  pImage);
7821  if(res >= 0)
7822  {
7823  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
7824  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
7825  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
7826 
7827  // 2. Allocate memory using allocator.
7828  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
7829  if(res >= 0)
7830  {
7831  // 3. Bind image with memory.
7832  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
7833  allocator->m_hDevice,
7834  *pImage,
7835  (*pAllocation)->GetMemory(),
7836  (*pAllocation)->GetOffset());
7837  if(res >= 0)
7838  {
7839  // All steps succeeded.
7840  if(pAllocationInfo != VMA_NULL)
7841  {
7842  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7843  }
7844  return VK_SUCCESS;
7845  }
7846  allocator->FreeMemory(*pAllocation);
7847  *pAllocation = VK_NULL_HANDLE;
7848  return res;
7849  }
7850  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
7851  *pImage = VK_NULL_HANDLE;
7852  return res;
7853  }
7854  return res;
7855 }
7856 
7857 void vmaDestroyImage(
7858  VmaAllocator allocator,
7859  VkImage image,
7860  VmaAllocation allocation)
7861 {
7862  if(image != VK_NULL_HANDLE)
7863  {
7864  VMA_ASSERT(allocator);
7865 
7866  VMA_DEBUG_LOG("vmaDestroyImage");
7867 
7868  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7869 
7870  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
7871 
7872  allocator->FreeMemory(allocation);
7873  }
7874 }
7875 
7876 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:446
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:469
-
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:800
+
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:806
vmaGetPoolStats
void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
Retrieves statistics of existing VmaPool object.
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:456
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for frequent writing on device and readback on host (download).
Definition: vk_mem_alloc.h:651
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:450
-
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:928
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1081
+
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:934
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1087
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaCreateBuffer().
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation.
vmaUnmapPersistentlyMappedMemory
void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
Destroys Vulkan image and frees allocated memory.
-
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:852
+
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:858
VmaDefragmentationInfo
struct VmaDefragmentationInfo VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
-
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:700
-
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:733
+
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:706
+
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:739
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:409
vmaMakePoolAllocationsLost
void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:481
-
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:802
+
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:808
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:528
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:463
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from small heaps <= 512 MB...
Definition: vk_mem_alloc.h:478
@@ -91,66 +91,66 @@ $(function() {
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:453
VmaAllocatorFlags
VkFlags VmaAllocatorFlags
Definition: vk_mem_alloc.h:439
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:592
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1085
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1091
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:498
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:602
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1093
-
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:716
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1076
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1099
+
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:722
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1082
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:454
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:472
-
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:806
+
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:812
VmaPoolStats
struct VmaPoolStats VmaPoolStats
Describes parameter of existing VmaPool.
-
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:938
+
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:944
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:451
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
General purpose memory allocation.
-
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:735
-
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:822
-
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:858
-
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:809
+
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:741
+
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:828
+
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:864
+
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:815
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
VmaVulkanFunctions
struct VmaVulkanFunctions VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
-
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:709
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1071
+
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:715
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1077
vmaCreatePool
VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
Allocates Vulkan device memory and creates VmaPool object.
-
VMA_POOL_CREATE_PERSISTENT_MAP_BIT
Definition: vk_mem_alloc.h:780
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1089
+
VMA_POOL_CREATE_PERSISTENT_MAP_BIT
Definition: vk_mem_alloc.h:786
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1095
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:452
vmaSetCurrentFrameIndex
void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
Sets index of the current frame.
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:598
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaAllocateMemoryForBuffer().
-
VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:689
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1091
+
VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:695
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1097
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:637
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
-
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:727
+
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:733
VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:435
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
VmaAllocatorFlagBits
VmaAllocatorFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:430
vmaSetAllocationUserData
void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
Sets pUserData in given allocation to new value.
-
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:868
+
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:874
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:447
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:581
-
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:817
+
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:823
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:422
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:594
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:426
vmaMapPersistentlyMappedMemory
VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
-
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:812
+
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:818
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:403
-
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:722
-
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:713
+
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:728
+
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:719
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:584
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:449
-
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:830
+
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:836
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:484
-
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:861
-
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:740
+
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:867
+
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:746
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:516
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:600
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:593
@@ -158,38 +158,38 @@ $(function() {
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:458
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:424
-
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:707
+
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:713
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:457
-
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:844
+
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:850
VmaAllocatorCreateInfo::flags
VmaAllocatorFlags flags
Flags for created allocator. Use VmaAllocatorFlagBits enum.
Definition: vk_mem_alloc.h:466
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:949
+
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:955
VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:668
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps. ...
Definition: vk_mem_alloc.h:475
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:593
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:590
-
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:849
+
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:855
VMA_MEMORY_USAGE_CPU_ONLY
Memory will be mapped on host. Could be used for transfer to/from device.
Definition: vk_mem_alloc.h:645
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
VmaStats
struct VmaStats VmaStats
General statistics from current state of Allocator.
-
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:933
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1087
+
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:939
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1093
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:445
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:588
-
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:711
+
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:717
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:586
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:455
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:459
-
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:771
-
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
Definition: vk_mem_alloc.h:944
+
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:777
+
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
Definition: vk_mem_alloc.h:950
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
VMA_MEMORY_USAGE_UNKNOWN
No intended memory usage specified.
Definition: vk_mem_alloc.h:640
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:448
vmaCreateLostAllocation
void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
Creates new allocation that is in lost state from the beginning.
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:652
-
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:914
+
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:920
VMA_MEMORY_USAGE_CPU_TO_GPU
Memory will be used for frequent (dynamic) updates from host and reads on device (upload).
Definition: vk_mem_alloc.h:648
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:656
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:594
@@ -203,11 +203,11 @@ $(function() {
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:601
VmaDefragmentationStats
struct VmaDefragmentationStats VmaDefragmentationStats
Statistics returned by function vmaDefragment().
vmaDestroyPool
void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
Destroys VmaPool object and frees Vulkan device memory.
-
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:855
+
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:861
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:594
-
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:798
+
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:804
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
Destroys Vulkan buffer and frees allocated memory.
-
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:919
+
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:925
VmaPoolCreateInfo
struct VmaPoolCreateInfo VmaPoolCreateInfo
Describes parameter of created VmaPool.
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 3ce8d66..3bb5f7e 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -685,6 +685,12 @@ typedef enum VmaAllocationCreateFlagBits { If VmaAllocationCreateInfo::pool is not null, usage of this flag must match usage of flag `VMA_POOL_CREATE_PERSISTENT_MAP_BIT` used during pool creation. + + Is it valid to use this flag for allocation made from memory type that is not + `HOST_VISIBLE`. This flag is then ignored and memory is not mapped. This is + useful if you need an allocation that is efficient to use on GPU + (`DEVICE_LOCAL`) and still want to map it directly if possible on platforms that + support it (e.g. Intel GPU). */ VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004, /** Allocation created with this flag can become lost as a result of another @@ -5315,8 +5321,8 @@ VkResult VmaBlockVector::Allocate( VmaAllocation* pAllocation) { // Validate flags. - if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) != - (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)) + if(createInfo.pool != VK_NULL_HANDLE && + ((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) != (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)) { VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT."); return VK_ERROR_OUT_OF_DEVICE_MEMORY; @@ -5548,7 +5554,19 @@ void VmaBlockVector::Free( m_HasEmptyBlock = true; } } - // Must be called after srcBlockIndex is used, because later it may become invalid! + // pBlock didn't become empty, but we have another empty block - find and free that one. + // (This is optional, heuristics.) + else if(m_HasEmptyBlock) + { + VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back(); + if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount) + { + pBlockToDelete = pLastBlock; + m_Blocks.pop_back(); + m_HasEmptyBlock = false; + } + } + IncrementallySortBlocks(); } @@ -6313,17 +6331,31 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; VMA_ASSERT(blockVector); - const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize(); - // Heuristics: Allocate own memory if requested size if greater than half of preferred block size. - const bool ownMemory = - (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 || - VMA_DEBUG_ALWAYS_OWN_MEMORY || - ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 && - vkMemReq.size > preferredBlockSize / 2); + VmaAllocationCreateInfo finalCreateInfo = createInfo; - if(ownMemory) + if(VMA_DEBUG_ALWAYS_OWN_MEMORY) { - if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) + finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT; + } + + // Heuristics: Allocate own memory if requested size if greater than half of preferred block size. + const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize(); + if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 && + vkMemReq.size > preferredBlockSize / 2) + { + finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT; + } + + // If memory type is not HOST_VISIBLE, disable PERSISTENT_MAP. + if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 && + (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) + { + finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT; + } + + if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0) + { + if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) { return VK_ERROR_OUT_OF_DEVICE_MEMORY; } @@ -6333,8 +6365,8 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( vkMemReq.size, suballocType, memTypeIndex, - (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, - createInfo.pUserData, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, + finalCreateInfo.pUserData, pAllocation); } } @@ -6344,7 +6376,7 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VK_NULL_HANDLE, // hCurrentPool m_CurrentFrameIndex.load(), vkMemReq, - createInfo, + finalCreateInfo, suballocType, pAllocation); if(res == VK_SUCCESS) @@ -6353,24 +6385,31 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( } // 5. Try own memory. - res = AllocateOwnMemory( - vkMemReq.size, - suballocType, - memTypeIndex, - (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, - createInfo.pUserData, - pAllocation); - if(res == VK_SUCCESS) + if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) { - // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here. - VMA_DEBUG_LOG(" Allocated as OwnMemory"); - return VK_SUCCESS; + return VK_ERROR_OUT_OF_DEVICE_MEMORY; } else { - // Everything failed: Return error code. - VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); - return res; + res = AllocateOwnMemory( + vkMemReq.size, + suballocType, + memTypeIndex, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, + finalCreateInfo.pUserData, + pAllocation); + if(res == VK_SUCCESS) + { + // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here. + VMA_DEBUG_LOG(" Allocated as OwnMemory"); + return VK_SUCCESS; + } + else + { + // Everything failed: Return error code. + VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); + return res; + } } } } @@ -7394,11 +7433,6 @@ VkResult vmaFindMemoryTypeIndex( break; } - if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) - { - requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; - } - *pMemoryTypeIndex = UINT32_MAX; uint32_t minCost = UINT32_MAX; for(uint32_t memTypeIndex = 0, memTypeBit = 1;