From c7b3e7c85db40a626afdd5e21de60778913ded9e Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Wed, 27 Sep 2017 13:40:58 +0200 Subject: [PATCH] Updated documents - added links to GitHub repository and GPUOpen product page. --- README.md | 8 +- docs/html/index.html | 6 +- docs/html/vk__mem__alloc_8h_source.html | 212 ++++++++++++------------ src/vk_mem_alloc.h | 6 +- 4 files changed, 119 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index c1889cd..f87a36d 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ Easy to integrate Vulkan memory allocation library. -**Documentation:** See [Vulkan Memory Allocator](https://gpuopen-librariesandsdks.github.io/VulkanMemoryAllocator/html/) (generated from Doxygen-style comments in [src/vk_mem_alloc.h](src/vk_mem_alloc.h)). +**Documentation:** See [Vulkan Memory Allocator](https://gpuopen-librariesandsdks.github.io/VulkanMemoryAllocator/html/) (generated from Doxygen-style comments in [src/vk_mem_alloc.h](src/vk_mem_alloc.h)) -**License:** MIT. See [LICENSE.txt](LICENSE.txt). +**License:** MIT. See [LICENSE.txt](LICENSE.txt) -**Changelog:** See [CHANGELOG.md](CHANGELOG.md). +**Changelog:** See [CHANGELOG.md](CHANGELOG.md) + +**Product page:** [Vulkan Memory Allocator on GPUOpen](https://gpuopen.com/gaming-product/vulkan-memory-allocator/) **Build status:** diff --git a/docs/html/index.html b/docs/html/index.html index 31a1ef8..09b11cb 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -80,8 +80,10 @@ $(function() {

Version 2.0.0-alpha.3 (2017-09-12)

-

Members grouped: see Modules.

-

All members: see vk_mem_alloc.h.

+

Source repository: VulkanMemoryAllocator project on GitHub
+Product page: Vulkan Memory Allocator on GPUOpen

+

Documentation of members grouped: Modules
+Documentation of all members: vk_mem_alloc.h

User guide

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index e6626c8..b189ea8 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,152 +62,152 @@ $(function() {
vk_mem_alloc.h

-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
387 #include <vulkan/vulkan.h>
388 
390 
394 VK_DEFINE_HANDLE(VmaAllocator)
395 
396 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
398  VmaAllocator allocator,
399  uint32_t memoryType,
400  VkDeviceMemory memory,
401  VkDeviceSize size);
403 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
404  VmaAllocator allocator,
405  uint32_t memoryType,
406  VkDeviceMemory memory,
407  VkDeviceSize size);
408 
416 typedef struct VmaDeviceMemoryCallbacks {
418  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
420  PFN_vmaFreeDeviceMemoryFunction pfnFree;
421 } VmaDeviceMemoryCallbacks;
422 
424 typedef enum VmaAllocatorFlagBits {
429  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
430 
431  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
432 } VmaAllocatorFlagBits;
433 typedef VkFlags VmaAllocatorFlags;
434 
439 typedef struct VmaVulkanFunctions {
440  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
441  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
442  PFN_vkAllocateMemory vkAllocateMemory;
443  PFN_vkFreeMemory vkFreeMemory;
444  PFN_vkMapMemory vkMapMemory;
445  PFN_vkUnmapMemory vkUnmapMemory;
446  PFN_vkBindBufferMemory vkBindBufferMemory;
447  PFN_vkBindImageMemory vkBindImageMemory;
448  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
449  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
450  PFN_vkCreateBuffer vkCreateBuffer;
451  PFN_vkDestroyBuffer vkDestroyBuffer;
452  PFN_vkCreateImage vkCreateImage;
453  PFN_vkDestroyImage vkDestroyImage;
454 } VmaVulkanFunctions;
455 
457 typedef struct VmaAllocatorCreateInfo
458 {
460  VmaAllocatorFlags flags;
462 
463  VkPhysicalDevice physicalDevice;
465 
466  VkDevice device;
468 
469  VkDeviceSize preferredLargeHeapBlockSize;
471 
472  VkDeviceSize preferredSmallHeapBlockSize;
474 
475  const VkAllocationCallbacks* pAllocationCallbacks;
477 
478  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
492  uint32_t frameInUseCount;
510  const VkDeviceSize* pHeapSizeLimit;
522  const VmaVulkanFunctions* pVulkanFunctions;
523 } VmaAllocatorCreateInfo;
524 
526 VkResult vmaCreateAllocator(
527  const VmaAllocatorCreateInfo* pCreateInfo,
528  VmaAllocator* pAllocator);
529 
531 void vmaDestroyAllocator(
532  VmaAllocator allocator);
533 
538 void vmaGetPhysicalDeviceProperties(
539  VmaAllocator allocator,
540  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
541 
546 void vmaGetMemoryProperties(
547  VmaAllocator allocator,
548  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
549 
556 void vmaGetMemoryTypeProperties(
557  VmaAllocator allocator,
558  uint32_t memoryTypeIndex,
559  VkMemoryPropertyFlags* pFlags);
560 
569 void vmaSetCurrentFrameIndex(
570  VmaAllocator allocator,
571  uint32_t frameIndex);
572 
575 typedef struct VmaStatInfo
576 {
578  uint32_t blockCount;
580  uint32_t allocationCount;
582  uint32_t unusedRangeCount;
584  VkDeviceSize usedBytes;
586  VkDeviceSize unusedBytes;
587  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
588  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
589 } VmaStatInfo;
590 
592 typedef struct VmaStats
593 {
594  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
595  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
596  VmaStatInfo total;
597 } VmaStats;
598 
600 void vmaCalculateStats(
601  VmaAllocator allocator,
602  VmaStats* pStats);
603 
604 #define VMA_STATS_STRING_ENABLED 1
605 
606 #if VMA_STATS_STRING_ENABLED
607 
609 
611 void vmaBuildStatsString(
612  VmaAllocator allocator,
613  char** ppStatsString,
614  VkBool32 detailedMap);
615 
616 void vmaFreeStatsString(
617  VmaAllocator allocator,
618  char* pStatsString);
619 
620 #endif // #if VMA_STATS_STRING_ENABLED
621 
624 
629 VK_DEFINE_HANDLE(VmaPool)
630 
631 typedef enum VmaMemoryUsage
632 {
634  VMA_MEMORY_USAGE_UNKNOWN = 0,
636  VMA_MEMORY_USAGE_GPU_ONLY = 1,
638 
639  VMA_MEMORY_USAGE_CPU_ONLY = 2,
641 
642  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
644 
645  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
646  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
647 } VmaMemoryUsage;
648 
650 typedef enum VmaAllocationCreateFlagBits {
662  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001,
663 
673  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
683  VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004,
694  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
701  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
702 
703  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
704 } VmaAllocationCreateFlagBits;
705 typedef VkFlags VmaAllocationCreateFlags;
706 
707 typedef struct VmaAllocationCreateInfo
708 {
710  VmaAllocationCreateFlags flags;
716  VmaMemoryUsage usage;
721  VkMemoryPropertyFlags requiredFlags;
727  VkMemoryPropertyFlags preferredFlags;
729  void* pUserData;
734  VmaPool pool;
735 } VmaAllocationCreateInfo;
736 
751 VkResult vmaFindMemoryTypeIndex(
752  VmaAllocator allocator,
753  uint32_t memoryTypeBits,
754  const VmaAllocationCreateInfo* pAllocationCreateInfo,
755  uint32_t* pMemoryTypeIndex);
756 
759 
764 typedef enum VmaPoolCreateFlagBits {
774  VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001,
792  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
793 
794  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
795 } VmaPoolCreateFlagBits;
796 typedef VkFlags VmaPoolCreateFlags;
797 
800 typedef struct VmaPoolCreateInfo {
803  uint32_t memoryTypeIndex;
806  VmaPoolCreateFlags flags;
811  VkDeviceSize blockSize;
816  size_t minBlockCount;
824  size_t maxBlockCount;
838  uint32_t frameInUseCount;
839 } VmaPoolCreateInfo;
840 
843 typedef struct VmaPoolStats {
846  VkDeviceSize size;
849  VkDeviceSize unusedSize;
852  size_t allocationCount;
855  size_t unusedRangeCount;
862  VkDeviceSize unusedRangeSizeMax;
863 } VmaPoolStats;
864 
871 VkResult vmaCreatePool(
872  VmaAllocator allocator,
873  const VmaPoolCreateInfo* pCreateInfo,
874  VmaPool* pPool);
875 
878 void vmaDestroyPool(
879  VmaAllocator allocator,
880  VmaPool pool);
881 
888 void vmaGetPoolStats(
889  VmaAllocator allocator,
890  VmaPool pool,
891  VmaPoolStats* pPoolStats);
892 
899 void vmaMakePoolAllocationsLost(
900  VmaAllocator allocator,
901  VmaPool pool,
902  size_t* pLostAllocationCount);
903 
904 VK_DEFINE_HANDLE(VmaAllocation)
905 
906 
908 typedef struct VmaAllocationInfo {
913  uint32_t memoryType;
922  VkDeviceMemory deviceMemory;
927  VkDeviceSize offset;
932  VkDeviceSize size;
938  void* pMappedData;
943  void* pUserData;
944 } VmaAllocationInfo;
945 
956 VkResult vmaAllocateMemory(
957  VmaAllocator allocator,
958  const VkMemoryRequirements* pVkMemoryRequirements,
959  const VmaAllocationCreateInfo* pCreateInfo,
960  VmaAllocation* pAllocation,
961  VmaAllocationInfo* pAllocationInfo);
962 
969 VkResult vmaAllocateMemoryForBuffer(
970  VmaAllocator allocator,
971  VkBuffer buffer,
972  const VmaAllocationCreateInfo* pCreateInfo,
973  VmaAllocation* pAllocation,
974  VmaAllocationInfo* pAllocationInfo);
975 
977 VkResult vmaAllocateMemoryForImage(
978  VmaAllocator allocator,
979  VkImage image,
980  const VmaAllocationCreateInfo* pCreateInfo,
981  VmaAllocation* pAllocation,
982  VmaAllocationInfo* pAllocationInfo);
983 
985 void vmaFreeMemory(
986  VmaAllocator allocator,
987  VmaAllocation allocation);
988 
990 void vmaGetAllocationInfo(
991  VmaAllocator allocator,
992  VmaAllocation allocation,
993  VmaAllocationInfo* pAllocationInfo);
994 
996 void vmaSetAllocationUserData(
997  VmaAllocator allocator,
998  VmaAllocation allocation,
999  void* pUserData);
1000 
1011 void vmaCreateLostAllocation(
1012  VmaAllocator allocator,
1013  VmaAllocation* pAllocation);
1014 
1023 VkResult vmaMapMemory(
1024  VmaAllocator allocator,
1025  VmaAllocation allocation,
1026  void** ppData);
1027 
1028 void vmaUnmapMemory(
1029  VmaAllocator allocator,
1030  VmaAllocation allocation);
1031 
1053 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
1054 
1062 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
1063 
1065 typedef struct VmaDefragmentationInfo {
1070  VkDeviceSize maxBytesToMove;
1075  uint32_t maxAllocationsToMove;
1076 } VmaDefragmentationInfo;
1077 
1079 typedef struct VmaDefragmentationStats {
1081  VkDeviceSize bytesMoved;
1083  VkDeviceSize bytesFreed;
1085  uint32_t allocationsMoved;
1087  uint32_t deviceMemoryBlocksFreed;
1088 } VmaDefragmentationStats;
1089 
1160 VkResult vmaDefragment(
1161  VmaAllocator allocator,
1162  VmaAllocation* pAllocations,
1163  size_t allocationCount,
1164  VkBool32* pAllocationsChanged,
1165  const VmaDefragmentationInfo *pDefragmentationInfo,
1166  VmaDefragmentationStats* pDefragmentationStats);
1167 
1170 
1193 VkResult vmaCreateBuffer(
1194  VmaAllocator allocator,
1195  const VkBufferCreateInfo* pBufferCreateInfo,
1196  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1197  VkBuffer* pBuffer,
1198  VmaAllocation* pAllocation,
1199  VmaAllocationInfo* pAllocationInfo);
1200 
1209 void vmaDestroyBuffer(
1210  VmaAllocator allocator,
1211  VkBuffer buffer,
1212  VmaAllocation allocation);
1213 
1215 VkResult vmaCreateImage(
1216  VmaAllocator allocator,
1217  const VkImageCreateInfo* pImageCreateInfo,
1218  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1219  VkImage* pImage,
1220  VmaAllocation* pAllocation,
1221  VmaAllocationInfo* pAllocationInfo);
1222 
1231 void vmaDestroyImage(
1232  VmaAllocator allocator,
1233  VkImage image,
1234  VmaAllocation allocation);
1235 
1238 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1239 
1240 // For Visual Studio IntelliSense.
1241 #ifdef __INTELLISENSE__
1242 #define VMA_IMPLEMENTATION
1243 #endif
1244 
1245 #ifdef VMA_IMPLEMENTATION
1246 #undef VMA_IMPLEMENTATION
1247 
1248 #include <cstdint>
1249 #include <cstdlib>
1250 #include <cstring>
1251 
1252 /*******************************************************************************
1253 CONFIGURATION SECTION
1254 
1255 Define some of these macros before each #include of this header or change them
1256 here if you need other then default behavior depending on your environment.
1257 */
1258 
1259 /*
1260 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1261 internally, like:
1262 
1263  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1264 
1265 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1266 VmaAllocatorCreateInfo::pVulkanFunctions.
1267 */
1268 #ifndef VMA_STATIC_VULKAN_FUNCTIONS
1269 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1270 #endif
1271 
1272 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1273 //#define VMA_USE_STL_CONTAINERS 1
1274 
1275 /* Set this macro to 1 to make the library including and using STL containers:
1276 std::pair, std::vector, std::list, std::unordered_map.
1277 
1278 Set it to 0 or undefined to make the library using its own implementation of
1279 the containers.
1280 */
1281 #if VMA_USE_STL_CONTAINERS
1282  #define VMA_USE_STL_VECTOR 1
1283  #define VMA_USE_STL_UNORDERED_MAP 1
1284  #define VMA_USE_STL_LIST 1
1285 #endif
1286 
1287 #if VMA_USE_STL_VECTOR
1288  #include <vector>
1289 #endif
1290 
1291 #if VMA_USE_STL_UNORDERED_MAP
1292  #include <unordered_map>
1293 #endif
1294 
1295 #if VMA_USE_STL_LIST
1296  #include <list>
1297 #endif
1298 
1299 /*
1300 Following headers are used in this CONFIGURATION section only, so feel free to
1301 remove them if not needed.
1302 */
1303 #include <cassert> // for assert
1304 #include <algorithm> // for min, max
1305 #include <mutex> // for std::mutex
1306 #include <atomic> // for std::atomic
1307 
1308 #if !defined(_WIN32)
1309  #include <malloc.h> // for aligned_alloc()
1310 #endif
1311 
1312 // Normal assert to check for programmer's errors, especially in Debug configuration.
1313 #ifndef VMA_ASSERT
1314  #ifdef _DEBUG
1315  #define VMA_ASSERT(expr) assert(expr)
1316  #else
1317  #define VMA_ASSERT(expr)
1318  #endif
1319 #endif
1320 
1321 // Assert that will be called very often, like inside data structures e.g. operator[].
1322 // Making it non-empty can make program slow.
1323 #ifndef VMA_HEAVY_ASSERT
1324  #ifdef _DEBUG
1325  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1326  #else
1327  #define VMA_HEAVY_ASSERT(expr)
1328  #endif
1329 #endif
1330 
1331 #ifndef VMA_NULL
1332  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1333  #define VMA_NULL nullptr
1334 #endif
1335 
1336 #ifndef VMA_ALIGN_OF
1337  #define VMA_ALIGN_OF(type) (__alignof(type))
1338 #endif
1339 
1340 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1341  #if defined(_WIN32)
1342  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1343  #else
1344  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1345  #endif
1346 #endif
1347 
1348 #ifndef VMA_SYSTEM_FREE
1349  #if defined(_WIN32)
1350  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1351  #else
1352  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1353  #endif
1354 #endif
1355 
1356 #ifndef VMA_MIN
1357  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1358 #endif
1359 
1360 #ifndef VMA_MAX
1361  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1362 #endif
1363 
1364 #ifndef VMA_SWAP
1365  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1366 #endif
1367 
1368 #ifndef VMA_SORT
1369  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1370 #endif
1371 
1372 #ifndef VMA_DEBUG_LOG
1373  #define VMA_DEBUG_LOG(format, ...)
1374  /*
1375  #define VMA_DEBUG_LOG(format, ...) do { \
1376  printf(format, __VA_ARGS__); \
1377  printf("\n"); \
1378  } while(false)
1379  */
1380 #endif
1381 
1382 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1383 #if VMA_STATS_STRING_ENABLED
1384  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1385  {
1386  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1387  }
1388  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1389  {
1390  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1391  }
1392  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1393  {
1394  snprintf(outStr, strLen, "%p", ptr);
1395  }
1396 #endif
1397 
1398 #ifndef VMA_MUTEX
1399  class VmaMutex
1400  {
1401  public:
1402  VmaMutex() { }
1403  ~VmaMutex() { }
1404  void Lock() { m_Mutex.lock(); }
1405  void Unlock() { m_Mutex.unlock(); }
1406  private:
1407  std::mutex m_Mutex;
1408  };
1409  #define VMA_MUTEX VmaMutex
1410 #endif
1411 
1412 /*
1413 If providing your own implementation, you need to implement a subset of std::atomic:
1414 
1415 - Constructor(uint32_t desired)
1416 - uint32_t load() const
1417 - void store(uint32_t desired)
1418 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1419 */
1420 #ifndef VMA_ATOMIC_UINT32
1421  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1422 #endif
1423 
1424 #ifndef VMA_BEST_FIT
1425 
1437  #define VMA_BEST_FIT (1)
1438 #endif
1439 
1440 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
1441 
1445  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
1446 #endif
1447 
1448 #ifndef VMA_DEBUG_ALIGNMENT
1449 
1453  #define VMA_DEBUG_ALIGNMENT (1)
1454 #endif
1455 
1456 #ifndef VMA_DEBUG_MARGIN
1457 
1461  #define VMA_DEBUG_MARGIN (0)
1462 #endif
1463 
1464 #ifndef VMA_DEBUG_GLOBAL_MUTEX
1465 
1469  #define VMA_DEBUG_GLOBAL_MUTEX (0)
1470 #endif
1471 
1472 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
1473 
1477  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
1478 #endif
1479 
1480 #ifndef VMA_SMALL_HEAP_MAX_SIZE
1481  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
1483 #endif
1484 
1485 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
1486  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
1488 #endif
1489 
1490 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
1491  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
1493 #endif
1494 
1495 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
1496 
1497 /*******************************************************************************
1498 END OF CONFIGURATION
1499 */
1500 
1501 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
1502  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
1503 
1504 // Returns number of bits set to 1 in (v).
1505 static inline uint32_t CountBitsSet(uint32_t v)
1506 {
1507  uint32_t c = v - ((v >> 1) & 0x55555555);
1508  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
1509  c = ((c >> 4) + c) & 0x0F0F0F0F;
1510  c = ((c >> 8) + c) & 0x00FF00FF;
1511  c = ((c >> 16) + c) & 0x0000FFFF;
1512  return c;
1513 }
1514 
1515 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
1516 // Use types like uint32_t, uint64_t as T.
1517 template <typename T>
1518 static inline T VmaAlignUp(T val, T align)
1519 {
1520  return (val + align - 1) / align * align;
1521 }
1522 
1523 // Division with mathematical rounding to nearest number.
1524 template <typename T>
1525 inline T VmaRoundDiv(T x, T y)
1526 {
1527  return (x + (y / (T)2)) / y;
1528 }
1529 
1530 #ifndef VMA_SORT
1531 
1532 template<typename Iterator, typename Compare>
1533 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
1534 {
1535  Iterator centerValue = end; --centerValue;
1536  Iterator insertIndex = beg;
1537  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
1538  {
1539  if(cmp(*memTypeIndex, *centerValue))
1540  {
1541  if(insertIndex != memTypeIndex)
1542  {
1543  VMA_SWAP(*memTypeIndex, *insertIndex);
1544  }
1545  ++insertIndex;
1546  }
1547  }
1548  if(insertIndex != centerValue)
1549  {
1550  VMA_SWAP(*insertIndex, *centerValue);
1551  }
1552  return insertIndex;
1553 }
1554 
1555 template<typename Iterator, typename Compare>
1556 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
1557 {
1558  if(beg < end)
1559  {
1560  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
1561  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
1562  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
1563  }
1564 }
1565 
1566 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
1567 
1568 #endif // #ifndef VMA_SORT
1569 
1570 /*
1571 Returns true if two memory blocks occupy overlapping pages.
1572 ResourceA must be in less memory offset than ResourceB.
1573 
1574 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1575 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1576 */
1577 static inline bool VmaBlocksOnSamePage(
1578  VkDeviceSize resourceAOffset,
1579  VkDeviceSize resourceASize,
1580  VkDeviceSize resourceBOffset,
1581  VkDeviceSize pageSize)
1582 {
1583  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1584  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1585  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1586  VkDeviceSize resourceBStart = resourceBOffset;
1587  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1588  return resourceAEndPage == resourceBStartPage;
1589 }
1590 
1591 enum VmaSuballocationType
1592 {
1593  VMA_SUBALLOCATION_TYPE_FREE = 0,
1594  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1595  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1596  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1597  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1598  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1599  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1600 };
1601 
1602 /*
1603 Returns true if given suballocation types could conflict and must respect
1604 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1605 or linear image and another one is optimal image. If type is unknown, behave
1606 conservatively.
1607 */
1608 static inline bool VmaIsBufferImageGranularityConflict(
1609  VmaSuballocationType suballocType1,
1610  VmaSuballocationType suballocType2)
1611 {
1612  if(suballocType1 > suballocType2)
1613  {
1614  VMA_SWAP(suballocType1, suballocType2);
1615  }
1616 
1617  switch(suballocType1)
1618  {
1619  case VMA_SUBALLOCATION_TYPE_FREE:
1620  return false;
1621  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1622  return true;
1623  case VMA_SUBALLOCATION_TYPE_BUFFER:
1624  return
1625  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1626  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1627  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1628  return
1629  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1630  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1631  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1632  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1633  return
1634  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1635  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1636  return false;
1637  default:
1638  VMA_ASSERT(0);
1639  return true;
1640  }
1641 }
1642 
1643 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1644 struct VmaMutexLock
1645 {
1646 public:
1647  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1648  m_pMutex(useMutex ? &mutex : VMA_NULL)
1649  {
1650  if(m_pMutex)
1651  {
1652  m_pMutex->Lock();
1653  }
1654  }
1655 
1656  ~VmaMutexLock()
1657  {
1658  if(m_pMutex)
1659  {
1660  m_pMutex->Unlock();
1661  }
1662  }
1663 
1664 private:
1665  VMA_MUTEX* m_pMutex;
1666 };
1667 
1668 #if VMA_DEBUG_GLOBAL_MUTEX
1669  static VMA_MUTEX gDebugGlobalMutex;
1670  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
1671 #else
1672  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1673 #endif
1674 
1675 // Minimum size of a free suballocation to register it in the free suballocation collection.
1676 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1677 
1678 /*
1679 Performs binary search and returns iterator to first element that is greater or
1680 equal to (key), according to comparison (cmp).
1681 
1682 Cmp should return true if first argument is less than second argument.
1683 
1684 Returned value is the found element, if present in the collection or place where
1685 new element with value (key) should be inserted.
1686 */
1687 template <typename IterT, typename KeyT, typename CmpT>
1688 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1689 {
1690  size_t down = 0, up = (end - beg);
1691  while(down < up)
1692  {
1693  const size_t mid = (down + up) / 2;
1694  if(cmp(*(beg+mid), key))
1695  {
1696  down = mid + 1;
1697  }
1698  else
1699  {
1700  up = mid;
1701  }
1702  }
1703  return beg + down;
1704 }
1705 
1707 // Memory allocation
1708 
1709 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1710 {
1711  if((pAllocationCallbacks != VMA_NULL) &&
1712  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1713  {
1714  return (*pAllocationCallbacks->pfnAllocation)(
1715  pAllocationCallbacks->pUserData,
1716  size,
1717  alignment,
1718  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1719  }
1720  else
1721  {
1722  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1723  }
1724 }
1725 
1726 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1727 {
1728  if((pAllocationCallbacks != VMA_NULL) &&
1729  (pAllocationCallbacks->pfnFree != VMA_NULL))
1730  {
1731  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1732  }
1733  else
1734  {
1735  VMA_SYSTEM_FREE(ptr);
1736  }
1737 }
1738 
1739 template<typename T>
1740 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1741 {
1742  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1743 }
1744 
1745 template<typename T>
1746 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1747 {
1748  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1749 }
1750 
1751 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1752 
1753 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1754 
1755 template<typename T>
1756 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1757 {
1758  ptr->~T();
1759  VmaFree(pAllocationCallbacks, ptr);
1760 }
1761 
1762 template<typename T>
1763 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1764 {
1765  if(ptr != VMA_NULL)
1766  {
1767  for(size_t i = count; i--; )
1768  {
1769  ptr[i].~T();
1770  }
1771  VmaFree(pAllocationCallbacks, ptr);
1772  }
1773 }
1774 
1775 // STL-compatible allocator.
1776 template<typename T>
1777 class VmaStlAllocator
1778 {
1779 public:
1780  const VkAllocationCallbacks* const m_pCallbacks;
1781  typedef T value_type;
1782 
1783  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1784  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1785 
1786  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1787  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1788 
1789  template<typename U>
1790  bool operator==(const VmaStlAllocator<U>& rhs) const
1791  {
1792  return m_pCallbacks == rhs.m_pCallbacks;
1793  }
1794  template<typename U>
1795  bool operator!=(const VmaStlAllocator<U>& rhs) const
1796  {
1797  return m_pCallbacks != rhs.m_pCallbacks;
1798  }
1799 
1800  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1801 };
1802 
1803 #if VMA_USE_STL_VECTOR
1804 
1805 #define VmaVector std::vector
1806 
1807 template<typename T, typename allocatorT>
1808 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1809 {
1810  vec.insert(vec.begin() + index, item);
1811 }
1812 
1813 template<typename T, typename allocatorT>
1814 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1815 {
1816  vec.erase(vec.begin() + index);
1817 }
1818 
1819 #else // #if VMA_USE_STL_VECTOR
1820 
1821 /* Class with interface compatible with subset of std::vector.
1822 T must be POD because constructors and destructors are not called and memcpy is
1823 used for these objects. */
1824 template<typename T, typename AllocatorT>
1825 class VmaVector
1826 {
1827 public:
1828  typedef T value_type;
1829 
1830  VmaVector(const AllocatorT& allocator) :
1831  m_Allocator(allocator),
1832  m_pArray(VMA_NULL),
1833  m_Count(0),
1834  m_Capacity(0)
1835  {
1836  }
1837 
1838  VmaVector(size_t count, const AllocatorT& allocator) :
1839  m_Allocator(allocator),
1840  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
1841  m_Count(count),
1842  m_Capacity(count)
1843  {
1844  }
1845 
1846  VmaVector(const VmaVector<T, AllocatorT>& src) :
1847  m_Allocator(src.m_Allocator),
1848  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
1849  m_Count(src.m_Count),
1850  m_Capacity(src.m_Count)
1851  {
1852  if(m_Count != 0)
1853  {
1854  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1855  }
1856  }
1857 
1858  ~VmaVector()
1859  {
1860  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1861  }
1862 
1863  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1864  {
1865  if(&rhs != this)
1866  {
1867  resize(rhs.m_Count);
1868  if(m_Count != 0)
1869  {
1870  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1871  }
1872  }
1873  return *this;
1874  }
1875 
1876  bool empty() const { return m_Count == 0; }
1877  size_t size() const { return m_Count; }
1878  T* data() { return m_pArray; }
1879  const T* data() const { return m_pArray; }
1880 
1881  T& operator[](size_t index)
1882  {
1883  VMA_HEAVY_ASSERT(index < m_Count);
1884  return m_pArray[index];
1885  }
1886  const T& operator[](size_t index) const
1887  {
1888  VMA_HEAVY_ASSERT(index < m_Count);
1889  return m_pArray[index];
1890  }
1891 
1892  T& front()
1893  {
1894  VMA_HEAVY_ASSERT(m_Count > 0);
1895  return m_pArray[0];
1896  }
1897  const T& front() const
1898  {
1899  VMA_HEAVY_ASSERT(m_Count > 0);
1900  return m_pArray[0];
1901  }
1902  T& back()
1903  {
1904  VMA_HEAVY_ASSERT(m_Count > 0);
1905  return m_pArray[m_Count - 1];
1906  }
1907  const T& back() const
1908  {
1909  VMA_HEAVY_ASSERT(m_Count > 0);
1910  return m_pArray[m_Count - 1];
1911  }
1912 
1913  void reserve(size_t newCapacity, bool freeMemory = false)
1914  {
1915  newCapacity = VMA_MAX(newCapacity, m_Count);
1916 
1917  if((newCapacity < m_Capacity) && !freeMemory)
1918  {
1919  newCapacity = m_Capacity;
1920  }
1921 
1922  if(newCapacity != m_Capacity)
1923  {
1924  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1925  if(m_Count != 0)
1926  {
1927  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1928  }
1929  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1930  m_Capacity = newCapacity;
1931  m_pArray = newArray;
1932  }
1933  }
1934 
1935  void resize(size_t newCount, bool freeMemory = false)
1936  {
1937  size_t newCapacity = m_Capacity;
1938  if(newCount > m_Capacity)
1939  {
1940  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1941  }
1942  else if(freeMemory)
1943  {
1944  newCapacity = newCount;
1945  }
1946 
1947  if(newCapacity != m_Capacity)
1948  {
1949  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1950  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1951  if(elementsToCopy != 0)
1952  {
1953  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1954  }
1955  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1956  m_Capacity = newCapacity;
1957  m_pArray = newArray;
1958  }
1959 
1960  m_Count = newCount;
1961  }
1962 
1963  void clear(bool freeMemory = false)
1964  {
1965  resize(0, freeMemory);
1966  }
1967 
1968  void insert(size_t index, const T& src)
1969  {
1970  VMA_HEAVY_ASSERT(index <= m_Count);
1971  const size_t oldCount = size();
1972  resize(oldCount + 1);
1973  if(index < oldCount)
1974  {
1975  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1976  }
1977  m_pArray[index] = src;
1978  }
1979 
1980  void remove(size_t index)
1981  {
1982  VMA_HEAVY_ASSERT(index < m_Count);
1983  const size_t oldCount = size();
1984  if(index < oldCount - 1)
1985  {
1986  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1987  }
1988  resize(oldCount - 1);
1989  }
1990 
1991  void push_back(const T& src)
1992  {
1993  const size_t newIndex = size();
1994  resize(newIndex + 1);
1995  m_pArray[newIndex] = src;
1996  }
1997 
1998  void pop_back()
1999  {
2000  VMA_HEAVY_ASSERT(m_Count > 0);
2001  resize(size() - 1);
2002  }
2003 
2004  void push_front(const T& src)
2005  {
2006  insert(0, src);
2007  }
2008 
2009  void pop_front()
2010  {
2011  VMA_HEAVY_ASSERT(m_Count > 0);
2012  remove(0);
2013  }
2014 
2015  typedef T* iterator;
2016 
2017  iterator begin() { return m_pArray; }
2018  iterator end() { return m_pArray + m_Count; }
2019 
2020 private:
2021  AllocatorT m_Allocator;
2022  T* m_pArray;
2023  size_t m_Count;
2024  size_t m_Capacity;
2025 };
2026 
2027 template<typename T, typename allocatorT>
2028 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2029 {
2030  vec.insert(index, item);
2031 }
2032 
2033 template<typename T, typename allocatorT>
2034 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2035 {
2036  vec.remove(index);
2037 }
2038 
2039 #endif // #if VMA_USE_STL_VECTOR
2040 
2041 template<typename CmpLess, typename VectorT>
2042 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2043 {
2044  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2045  vector.data(),
2046  vector.data() + vector.size(),
2047  value,
2048  CmpLess()) - vector.data();
2049  VmaVectorInsert(vector, indexToInsert, value);
2050  return indexToInsert;
2051 }
2052 
2053 template<typename CmpLess, typename VectorT>
2054 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2055 {
2056  CmpLess comparator;
2057  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2058  vector.begin(),
2059  vector.end(),
2060  value,
2061  comparator);
2062  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2063  {
2064  size_t indexToRemove = it - vector.begin();
2065  VmaVectorRemove(vector, indexToRemove);
2066  return true;
2067  }
2068  return false;
2069 }
2070 
2071 template<typename CmpLess, typename VectorT>
2072 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2073 {
2074  CmpLess comparator;
2075  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2076  vector.data(),
2077  vector.data() + vector.size(),
2078  value,
2079  comparator);
2080  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2081  {
2082  return it - vector.begin();
2083  }
2084  else
2085  {
2086  return vector.size();
2087  }
2088 }
2089 
2091 // class VmaPoolAllocator
2092 
2093 /*
2094 Allocator for objects of type T using a list of arrays (pools) to speed up
2095 allocation. Number of elements that can be allocated is not bounded because
2096 allocator can create multiple blocks.
2097 */
2098 template<typename T>
2099 class VmaPoolAllocator
2100 {
2101 public:
2102  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2103  ~VmaPoolAllocator();
2104  void Clear();
2105  T* Alloc();
2106  void Free(T* ptr);
2107 
2108 private:
2109  union Item
2110  {
2111  uint32_t NextFreeIndex;
2112  T Value;
2113  };
2114 
2115  struct ItemBlock
2116  {
2117  Item* pItems;
2118  uint32_t FirstFreeIndex;
2119  };
2120 
2121  const VkAllocationCallbacks* m_pAllocationCallbacks;
2122  size_t m_ItemsPerBlock;
2123  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2124 
2125  ItemBlock& CreateNewBlock();
2126 };
2127 
2128 template<typename T>
2129 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2130  m_pAllocationCallbacks(pAllocationCallbacks),
2131  m_ItemsPerBlock(itemsPerBlock),
2132  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2133 {
2134  VMA_ASSERT(itemsPerBlock > 0);
2135 }
2136 
2137 template<typename T>
2138 VmaPoolAllocator<T>::~VmaPoolAllocator()
2139 {
2140  Clear();
2141 }
2142 
2143 template<typename T>
2144 void VmaPoolAllocator<T>::Clear()
2145 {
2146  for(size_t i = m_ItemBlocks.size(); i--; )
2147  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2148  m_ItemBlocks.clear();
2149 }
2150 
2151 template<typename T>
2152 T* VmaPoolAllocator<T>::Alloc()
2153 {
2154  for(size_t i = m_ItemBlocks.size(); i--; )
2155  {
2156  ItemBlock& block = m_ItemBlocks[i];
2157  // This block has some free items: Use first one.
2158  if(block.FirstFreeIndex != UINT32_MAX)
2159  {
2160  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2161  block.FirstFreeIndex = pItem->NextFreeIndex;
2162  return &pItem->Value;
2163  }
2164  }
2165 
2166  // No block has free item: Create new one and use it.
2167  ItemBlock& newBlock = CreateNewBlock();
2168  Item* const pItem = &newBlock.pItems[0];
2169  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2170  return &pItem->Value;
2171 }
2172 
2173 template<typename T>
2174 void VmaPoolAllocator<T>::Free(T* ptr)
2175 {
2176  // Search all memory blocks to find ptr.
2177  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2178  {
2179  ItemBlock& block = m_ItemBlocks[i];
2180 
2181  // Casting to union.
2182  Item* pItemPtr;
2183  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2184 
2185  // Check if pItemPtr is in address range of this block.
2186  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2187  {
2188  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2189  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2190  block.FirstFreeIndex = index;
2191  return;
2192  }
2193  }
2194  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2195 }
2196 
2197 template<typename T>
2198 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2199 {
2200  ItemBlock newBlock = {
2201  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2202 
2203  m_ItemBlocks.push_back(newBlock);
2204 
2205  // Setup singly-linked list of all free items in this block.
2206  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2207  newBlock.pItems[i].NextFreeIndex = i + 1;
2208  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2209  return m_ItemBlocks.back();
2210 }
2211 
2213 // class VmaRawList, VmaList
2214 
2215 #if VMA_USE_STL_LIST
2216 
2217 #define VmaList std::list
2218 
2219 #else // #if VMA_USE_STL_LIST
2220 
2221 template<typename T>
2222 struct VmaListItem
2223 {
2224  VmaListItem* pPrev;
2225  VmaListItem* pNext;
2226  T Value;
2227 };
2228 
2229 // Doubly linked list.
2230 template<typename T>
2231 class VmaRawList
2232 {
2233 public:
2234  typedef VmaListItem<T> ItemType;
2235 
2236  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2237  ~VmaRawList();
2238  void Clear();
2239 
2240  size_t GetCount() const { return m_Count; }
2241  bool IsEmpty() const { return m_Count == 0; }
2242 
2243  ItemType* Front() { return m_pFront; }
2244  const ItemType* Front() const { return m_pFront; }
2245  ItemType* Back() { return m_pBack; }
2246  const ItemType* Back() const { return m_pBack; }
2247 
2248  ItemType* PushBack();
2249  ItemType* PushFront();
2250  ItemType* PushBack(const T& value);
2251  ItemType* PushFront(const T& value);
2252  void PopBack();
2253  void PopFront();
2254 
2255  // Item can be null - it means PushBack.
2256  ItemType* InsertBefore(ItemType* pItem);
2257  // Item can be null - it means PushFront.
2258  ItemType* InsertAfter(ItemType* pItem);
2259 
2260  ItemType* InsertBefore(ItemType* pItem, const T& value);
2261  ItemType* InsertAfter(ItemType* pItem, const T& value);
2262 
2263  void Remove(ItemType* pItem);
2264 
2265 private:
2266  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2267  VmaPoolAllocator<ItemType> m_ItemAllocator;
2268  ItemType* m_pFront;
2269  ItemType* m_pBack;
2270  size_t m_Count;
2271 
2272  // Declared not defined, to block copy constructor and assignment operator.
2273  VmaRawList(const VmaRawList<T>& src);
2274  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2275 };
2276 
2277 template<typename T>
2278 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2279  m_pAllocationCallbacks(pAllocationCallbacks),
2280  m_ItemAllocator(pAllocationCallbacks, 128),
2281  m_pFront(VMA_NULL),
2282  m_pBack(VMA_NULL),
2283  m_Count(0)
2284 {
2285 }
2286 
2287 template<typename T>
2288 VmaRawList<T>::~VmaRawList()
2289 {
2290  // Intentionally not calling Clear, because that would be unnecessary
2291  // computations to return all items to m_ItemAllocator as free.
2292 }
2293 
2294 template<typename T>
2295 void VmaRawList<T>::Clear()
2296 {
2297  if(IsEmpty() == false)
2298  {
2299  ItemType* pItem = m_pBack;
2300  while(pItem != VMA_NULL)
2301  {
2302  ItemType* const pPrevItem = pItem->pPrev;
2303  m_ItemAllocator.Free(pItem);
2304  pItem = pPrevItem;
2305  }
2306  m_pFront = VMA_NULL;
2307  m_pBack = VMA_NULL;
2308  m_Count = 0;
2309  }
2310 }
2311 
2312 template<typename T>
2313 VmaListItem<T>* VmaRawList<T>::PushBack()
2314 {
2315  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2316  pNewItem->pNext = VMA_NULL;
2317  if(IsEmpty())
2318  {
2319  pNewItem->pPrev = VMA_NULL;
2320  m_pFront = pNewItem;
2321  m_pBack = pNewItem;
2322  m_Count = 1;
2323  }
2324  else
2325  {
2326  pNewItem->pPrev = m_pBack;
2327  m_pBack->pNext = pNewItem;
2328  m_pBack = pNewItem;
2329  ++m_Count;
2330  }
2331  return pNewItem;
2332 }
2333 
2334 template<typename T>
2335 VmaListItem<T>* VmaRawList<T>::PushFront()
2336 {
2337  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2338  pNewItem->pPrev = VMA_NULL;
2339  if(IsEmpty())
2340  {
2341  pNewItem->pNext = VMA_NULL;
2342  m_pFront = pNewItem;
2343  m_pBack = pNewItem;
2344  m_Count = 1;
2345  }
2346  else
2347  {
2348  pNewItem->pNext = m_pFront;
2349  m_pFront->pPrev = pNewItem;
2350  m_pFront = pNewItem;
2351  ++m_Count;
2352  }
2353  return pNewItem;
2354 }
2355 
2356 template<typename T>
2357 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2358 {
2359  ItemType* const pNewItem = PushBack();
2360  pNewItem->Value = value;
2361  return pNewItem;
2362 }
2363 
2364 template<typename T>
2365 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2366 {
2367  ItemType* const pNewItem = PushFront();
2368  pNewItem->Value = value;
2369  return pNewItem;
2370 }
2371 
2372 template<typename T>
2373 void VmaRawList<T>::PopBack()
2374 {
2375  VMA_HEAVY_ASSERT(m_Count > 0);
2376  ItemType* const pBackItem = m_pBack;
2377  ItemType* const pPrevItem = pBackItem->pPrev;
2378  if(pPrevItem != VMA_NULL)
2379  {
2380  pPrevItem->pNext = VMA_NULL;
2381  }
2382  m_pBack = pPrevItem;
2383  m_ItemAllocator.Free(pBackItem);
2384  --m_Count;
2385 }
2386 
2387 template<typename T>
2388 void VmaRawList<T>::PopFront()
2389 {
2390  VMA_HEAVY_ASSERT(m_Count > 0);
2391  ItemType* const pFrontItem = m_pFront;
2392  ItemType* const pNextItem = pFrontItem->pNext;
2393  if(pNextItem != VMA_NULL)
2394  {
2395  pNextItem->pPrev = VMA_NULL;
2396  }
2397  m_pFront = pNextItem;
2398  m_ItemAllocator.Free(pFrontItem);
2399  --m_Count;
2400 }
2401 
2402 template<typename T>
2403 void VmaRawList<T>::Remove(ItemType* pItem)
2404 {
2405  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2406  VMA_HEAVY_ASSERT(m_Count > 0);
2407 
2408  if(pItem->pPrev != VMA_NULL)
2409  {
2410  pItem->pPrev->pNext = pItem->pNext;
2411  }
2412  else
2413  {
2414  VMA_HEAVY_ASSERT(m_pFront == pItem);
2415  m_pFront = pItem->pNext;
2416  }
2417 
2418  if(pItem->pNext != VMA_NULL)
2419  {
2420  pItem->pNext->pPrev = pItem->pPrev;
2421  }
2422  else
2423  {
2424  VMA_HEAVY_ASSERT(m_pBack == pItem);
2425  m_pBack = pItem->pPrev;
2426  }
2427 
2428  m_ItemAllocator.Free(pItem);
2429  --m_Count;
2430 }
2431 
2432 template<typename T>
2433 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2434 {
2435  if(pItem != VMA_NULL)
2436  {
2437  ItemType* const prevItem = pItem->pPrev;
2438  ItemType* const newItem = m_ItemAllocator.Alloc();
2439  newItem->pPrev = prevItem;
2440  newItem->pNext = pItem;
2441  pItem->pPrev = newItem;
2442  if(prevItem != VMA_NULL)
2443  {
2444  prevItem->pNext = newItem;
2445  }
2446  else
2447  {
2448  VMA_HEAVY_ASSERT(m_pFront == pItem);
2449  m_pFront = newItem;
2450  }
2451  ++m_Count;
2452  return newItem;
2453  }
2454  else
2455  return PushBack();
2456 }
2457 
2458 template<typename T>
2459 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2460 {
2461  if(pItem != VMA_NULL)
2462  {
2463  ItemType* const nextItem = pItem->pNext;
2464  ItemType* const newItem = m_ItemAllocator.Alloc();
2465  newItem->pNext = nextItem;
2466  newItem->pPrev = pItem;
2467  pItem->pNext = newItem;
2468  if(nextItem != VMA_NULL)
2469  {
2470  nextItem->pPrev = newItem;
2471  }
2472  else
2473  {
2474  VMA_HEAVY_ASSERT(m_pBack == pItem);
2475  m_pBack = newItem;
2476  }
2477  ++m_Count;
2478  return newItem;
2479  }
2480  else
2481  return PushFront();
2482 }
2483 
2484 template<typename T>
2485 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
2486 {
2487  ItemType* const newItem = InsertBefore(pItem);
2488  newItem->Value = value;
2489  return newItem;
2490 }
2491 
2492 template<typename T>
2493 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
2494 {
2495  ItemType* const newItem = InsertAfter(pItem);
2496  newItem->Value = value;
2497  return newItem;
2498 }
2499 
2500 template<typename T, typename AllocatorT>
2501 class VmaList
2502 {
2503 public:
2504  class iterator
2505  {
2506  public:
2507  iterator() :
2508  m_pList(VMA_NULL),
2509  m_pItem(VMA_NULL)
2510  {
2511  }
2512 
2513  T& operator*() const
2514  {
2515  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2516  return m_pItem->Value;
2517  }
2518  T* operator->() const
2519  {
2520  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2521  return &m_pItem->Value;
2522  }
2523 
2524  iterator& operator++()
2525  {
2526  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2527  m_pItem = m_pItem->pNext;
2528  return *this;
2529  }
2530  iterator& operator--()
2531  {
2532  if(m_pItem != VMA_NULL)
2533  {
2534  m_pItem = m_pItem->pPrev;
2535  }
2536  else
2537  {
2538  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
2539  m_pItem = m_pList->Back();
2540  }
2541  return *this;
2542  }
2543 
2544  iterator operator++(int)
2545  {
2546  iterator result = *this;
2547  ++*this;
2548  return result;
2549  }
2550  iterator operator--(int)
2551  {
2552  iterator result = *this;
2553  --*this;
2554  return result;
2555  }
2556 
2557  bool operator==(const iterator& rhs) const
2558  {
2559  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2560  return m_pItem == rhs.m_pItem;
2561  }
2562  bool operator!=(const iterator& rhs) const
2563  {
2564  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2565  return m_pItem != rhs.m_pItem;
2566  }
2567 
2568  private:
2569  VmaRawList<T>* m_pList;
2570  VmaListItem<T>* m_pItem;
2571 
2572  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
2573  m_pList(pList),
2574  m_pItem(pItem)
2575  {
2576  }
2577 
2578  friend class VmaList<T, AllocatorT>;
2579  };
2580 
2581  class const_iterator
2582  {
2583  public:
2584  const_iterator() :
2585  m_pList(VMA_NULL),
2586  m_pItem(VMA_NULL)
2587  {
2588  }
2589 
2590  const_iterator(const iterator& src) :
2591  m_pList(src.m_pList),
2592  m_pItem(src.m_pItem)
2593  {
2594  }
2595 
2596  const T& operator*() const
2597  {
2598  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2599  return m_pItem->Value;
2600  }
2601  const T* operator->() const
2602  {
2603  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2604  return &m_pItem->Value;
2605  }
2606 
2607  const_iterator& operator++()
2608  {
2609  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2610  m_pItem = m_pItem->pNext;
2611  return *this;
2612  }
2613  const_iterator& operator--()
2614  {
2615  if(m_pItem != VMA_NULL)
2616  {
2617  m_pItem = m_pItem->pPrev;
2618  }
2619  else
2620  {
2621  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2622  m_pItem = m_pList->Back();
2623  }
2624  return *this;
2625  }
2626 
2627  const_iterator operator++(int)
2628  {
2629  const_iterator result = *this;
2630  ++*this;
2631  return result;
2632  }
2633  const_iterator operator--(int)
2634  {
2635  const_iterator result = *this;
2636  --*this;
2637  return result;
2638  }
2639 
2640  bool operator==(const const_iterator& rhs) const
2641  {
2642  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2643  return m_pItem == rhs.m_pItem;
2644  }
2645  bool operator!=(const const_iterator& rhs) const
2646  {
2647  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2648  return m_pItem != rhs.m_pItem;
2649  }
2650 
2651  private:
2652  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2653  m_pList(pList),
2654  m_pItem(pItem)
2655  {
2656  }
2657 
2658  const VmaRawList<T>* m_pList;
2659  const VmaListItem<T>* m_pItem;
2660 
2661  friend class VmaList<T, AllocatorT>;
2662  };
2663 
2664  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2665 
2666  bool empty() const { return m_RawList.IsEmpty(); }
2667  size_t size() const { return m_RawList.GetCount(); }
2668 
2669  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2670  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2671 
2672  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2673  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2674 
2675  void clear() { m_RawList.Clear(); }
2676  void push_back(const T& value) { m_RawList.PushBack(value); }
2677  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2678  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2679 
2680 private:
2681  VmaRawList<T> m_RawList;
2682 };
2683 
2684 #endif // #if VMA_USE_STL_LIST
2685 
2687 // class VmaMap
2688 
2689 // Unused in this version.
2690 #if 0
2691 
2692 #if VMA_USE_STL_UNORDERED_MAP
2693 
2694 #define VmaPair std::pair
2695 
2696 #define VMA_MAP_TYPE(KeyT, ValueT) \
2697  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2698 
2699 #else // #if VMA_USE_STL_UNORDERED_MAP
2700 
2701 template<typename T1, typename T2>
2702 struct VmaPair
2703 {
2704  T1 first;
2705  T2 second;
2706 
2707  VmaPair() : first(), second() { }
2708  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2709 };
2710 
2711 /* Class compatible with subset of interface of std::unordered_map.
2712 KeyT, ValueT must be POD because they will be stored in VmaVector.
2713 */
2714 template<typename KeyT, typename ValueT>
2715 class VmaMap
2716 {
2717 public:
2718  typedef VmaPair<KeyT, ValueT> PairType;
2719  typedef PairType* iterator;
2720 
2721  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2722 
2723  iterator begin() { return m_Vector.begin(); }
2724  iterator end() { return m_Vector.end(); }
2725 
2726  void insert(const PairType& pair);
2727  iterator find(const KeyT& key);
2728  void erase(iterator it);
2729 
2730 private:
2731  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2732 };
2733 
2734 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2735 
2736 template<typename FirstT, typename SecondT>
2737 struct VmaPairFirstLess
2738 {
2739  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2740  {
2741  return lhs.first < rhs.first;
2742  }
2743  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2744  {
2745  return lhs.first < rhsFirst;
2746  }
2747 };
2748 
2749 template<typename KeyT, typename ValueT>
2750 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2751 {
2752  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2753  m_Vector.data(),
2754  m_Vector.data() + m_Vector.size(),
2755  pair,
2756  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2757  VmaVectorInsert(m_Vector, indexToInsert, pair);
2758 }
2759 
2760 template<typename KeyT, typename ValueT>
2761 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2762 {
2763  PairType* it = VmaBinaryFindFirstNotLess(
2764  m_Vector.data(),
2765  m_Vector.data() + m_Vector.size(),
2766  key,
2767  VmaPairFirstLess<KeyT, ValueT>());
2768  if((it != m_Vector.end()) && (it->first == key))
2769  {
2770  return it;
2771  }
2772  else
2773  {
2774  return m_Vector.end();
2775  }
2776 }
2777 
2778 template<typename KeyT, typename ValueT>
2779 void VmaMap<KeyT, ValueT>::erase(iterator it)
2780 {
2781  VmaVectorRemove(m_Vector, it - m_Vector.begin());
2782 }
2783 
2784 #endif // #if VMA_USE_STL_UNORDERED_MAP
2785 
2786 #endif // #if 0
2787 
2789 
2790 class VmaDeviceMemoryBlock;
2791 
2792 enum VMA_BLOCK_VECTOR_TYPE
2793 {
2794  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2795  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2796  VMA_BLOCK_VECTOR_TYPE_COUNT
2797 };
2798 
2799 static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags)
2800 {
2801  return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ?
2802  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2803  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2804 }
2805 
2806 struct VmaAllocation_T
2807 {
2808 public:
2809  enum ALLOCATION_TYPE
2810  {
2811  ALLOCATION_TYPE_NONE,
2812  ALLOCATION_TYPE_BLOCK,
2813  ALLOCATION_TYPE_OWN,
2814  };
2815 
2816  VmaAllocation_T(uint32_t currentFrameIndex) :
2817  m_Alignment(1),
2818  m_Size(0),
2819  m_pUserData(VMA_NULL),
2820  m_Type(ALLOCATION_TYPE_NONE),
2821  m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN),
2822  m_LastUseFrameIndex(currentFrameIndex)
2823  {
2824  }
2825 
2826  void InitBlockAllocation(
2827  VmaPool hPool,
2828  VmaDeviceMemoryBlock* block,
2829  VkDeviceSize offset,
2830  VkDeviceSize alignment,
2831  VkDeviceSize size,
2832  VmaSuballocationType suballocationType,
2833  void* pUserData,
2834  bool canBecomeLost)
2835  {
2836  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2837  VMA_ASSERT(block != VMA_NULL);
2838  m_Type = ALLOCATION_TYPE_BLOCK;
2839  m_Alignment = alignment;
2840  m_Size = size;
2841  m_pUserData = pUserData;
2842  m_SuballocationType = suballocationType;
2843  m_BlockAllocation.m_hPool = hPool;
2844  m_BlockAllocation.m_Block = block;
2845  m_BlockAllocation.m_Offset = offset;
2846  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
2847  }
2848 
2849  void InitLost()
2850  {
2851  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2852  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
2853  m_Type = ALLOCATION_TYPE_BLOCK;
2854  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
2855  m_BlockAllocation.m_Block = VMA_NULL;
2856  m_BlockAllocation.m_Offset = 0;
2857  m_BlockAllocation.m_CanBecomeLost = true;
2858  }
2859 
2860  void ChangeBlockAllocation(
2861  VmaDeviceMemoryBlock* block,
2862  VkDeviceSize offset)
2863  {
2864  VMA_ASSERT(block != VMA_NULL);
2865  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2866  m_BlockAllocation.m_Block = block;
2867  m_BlockAllocation.m_Offset = offset;
2868  }
2869 
2870  void InitOwnAllocation(
2871  uint32_t memoryTypeIndex,
2872  VkDeviceMemory hMemory,
2873  VmaSuballocationType suballocationType,
2874  bool persistentMap,
2875  void* pMappedData,
2876  VkDeviceSize size,
2877  void* pUserData)
2878  {
2879  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2880  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2881  m_Type = ALLOCATION_TYPE_OWN;
2882  m_Alignment = 0;
2883  m_Size = size;
2884  m_pUserData = pUserData;
2885  m_SuballocationType = suballocationType;
2886  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2887  m_OwnAllocation.m_hMemory = hMemory;
2888  m_OwnAllocation.m_PersistentMap = persistentMap;
2889  m_OwnAllocation.m_pMappedData = pMappedData;
2890  }
2891 
2892  ALLOCATION_TYPE GetType() const { return m_Type; }
2893  VkDeviceSize GetAlignment() const { return m_Alignment; }
2894  VkDeviceSize GetSize() const { return m_Size; }
2895  void* GetUserData() const { return m_pUserData; }
2896  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2897  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2898 
2899  VmaDeviceMemoryBlock* GetBlock() const
2900  {
2901  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2902  return m_BlockAllocation.m_Block;
2903  }
2904  VkDeviceSize GetOffset() const;
2905  VkDeviceMemory GetMemory() const;
2906  uint32_t GetMemoryTypeIndex() const;
2907  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2908  void* GetMappedData() const;
2909  bool CanBecomeLost() const;
2910  VmaPool GetPool() const;
2911 
2912  VkResult OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator);
2913  void OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator);
2914 
2915  uint32_t GetLastUseFrameIndex() const
2916  {
2917  return m_LastUseFrameIndex.load();
2918  }
2919  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
2920  {
2921  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
2922  }
2923  /*
2924  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
2925  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
2926  - Else, returns false.
2927 
2928  If hAllocation is already lost, assert - you should not call it then.
2929  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
2930  */
2931  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
2932 
2933  void OwnAllocCalcStatsInfo(VmaStatInfo& outInfo)
2934  {
2935  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2936  outInfo.blockCount = 1;
2937  outInfo.allocationCount = 1;
2938  outInfo.unusedRangeCount = 0;
2939  outInfo.usedBytes = m_Size;
2940  outInfo.unusedBytes = 0;
2941  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
2942  outInfo.unusedRangeSizeMin = UINT64_MAX;
2943  outInfo.unusedRangeSizeMax = 0;
2944  }
2945 
2946 private:
2947  VkDeviceSize m_Alignment;
2948  VkDeviceSize m_Size;
2949  void* m_pUserData;
2950  ALLOCATION_TYPE m_Type;
2951  VmaSuballocationType m_SuballocationType;
2952  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
2953 
2954  // Allocation out of VmaDeviceMemoryBlock.
2955  struct BlockAllocation
2956  {
2957  VmaPool m_hPool; // Null if belongs to general memory.
2958  VmaDeviceMemoryBlock* m_Block;
2959  VkDeviceSize m_Offset;
2960  bool m_CanBecomeLost;
2961  };
2962 
2963  // Allocation for an object that has its own private VkDeviceMemory.
2964  struct OwnAllocation
2965  {
2966  uint32_t m_MemoryTypeIndex;
2967  VkDeviceMemory m_hMemory;
2968  bool m_PersistentMap;
2969  void* m_pMappedData;
2970  };
2971 
2972  union
2973  {
2974  // Allocation out of VmaDeviceMemoryBlock.
2975  BlockAllocation m_BlockAllocation;
2976  // Allocation for an object that has its own private VkDeviceMemory.
2977  OwnAllocation m_OwnAllocation;
2978  };
2979 };
2980 
2981 /*
2982 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
2983 allocated memory block or free.
2984 */
2985 struct VmaSuballocation
2986 {
2987  VkDeviceSize offset;
2988  VkDeviceSize size;
2989  VmaAllocation hAllocation;
2990  VmaSuballocationType type;
2991 };
2992 
2993 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
2994 
2995 // Cost of one additional allocation lost, as equivalent in bytes.
2996 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
2997 
2998 /*
2999 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3000 
3001 If canMakeOtherLost was false:
3002 - item points to a FREE suballocation.
3003 - itemsToMakeLostCount is 0.
3004 
3005 If canMakeOtherLost was true:
3006 - item points to first of sequence of suballocations, which are either FREE,
3007  or point to VmaAllocations that can become lost.
3008 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3009  the requested allocation to succeed.
3010 */
3011 struct VmaAllocationRequest
3012 {
3013  VkDeviceSize offset;
3014  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3015  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3016  VmaSuballocationList::iterator item;
3017  size_t itemsToMakeLostCount;
3018 
3019  VkDeviceSize CalcCost() const
3020  {
3021  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3022  }
3023 };
3024 
3025 /*
3026 Data structure used for bookkeeping of allocations and unused ranges of memory
3027 in a single VkDeviceMemory block.
3028 */
3029 class VmaBlockMetadata
3030 {
3031 public:
3032  VmaBlockMetadata(VmaAllocator hAllocator);
3033  ~VmaBlockMetadata();
3034  void Init(VkDeviceSize size);
3035 
3036  // Validates all data structures inside this object. If not valid, returns false.
3037  bool Validate() const;
3038  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3039  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3040  VkDeviceSize GetUnusedRangeSizeMax() const;
3041  // Returns true if this block is empty - contains only single free suballocation.
3042  bool IsEmpty() const;
3043 
3044  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3045  void AddPoolStats(VmaPoolStats& inoutStats) const;
3046 
3047 #if VMA_STATS_STRING_ENABLED
3048  void PrintDetailedMap(class VmaJsonWriter& json) const;
3049 #endif
3050 
3051  // Creates trivial request for case when block is empty.
3052  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3053 
3054  // Tries to find a place for suballocation with given parameters inside this block.
3055  // If succeeded, fills pAllocationRequest and returns true.
3056  // If failed, returns false.
3057  bool CreateAllocationRequest(
3058  uint32_t currentFrameIndex,
3059  uint32_t frameInUseCount,
3060  VkDeviceSize bufferImageGranularity,
3061  VkDeviceSize allocSize,
3062  VkDeviceSize allocAlignment,
3063  VmaSuballocationType allocType,
3064  bool canMakeOtherLost,
3065  VmaAllocationRequest* pAllocationRequest);
3066 
3067  bool MakeRequestedAllocationsLost(
3068  uint32_t currentFrameIndex,
3069  uint32_t frameInUseCount,
3070  VmaAllocationRequest* pAllocationRequest);
3071 
3072  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3073 
3074  // Makes actual allocation based on request. Request must already be checked and valid.
3075  void Alloc(
3076  const VmaAllocationRequest& request,
3077  VmaSuballocationType type,
3078  VkDeviceSize allocSize,
3079  VmaAllocation hAllocation);
3080 
3081  // Frees suballocation assigned to given memory region.
3082  void Free(const VmaAllocation allocation);
3083 
3084 private:
3085  VkDeviceSize m_Size;
3086  uint32_t m_FreeCount;
3087  VkDeviceSize m_SumFreeSize;
3088  VmaSuballocationList m_Suballocations;
3089  // Suballocations that are free and have size greater than certain threshold.
3090  // Sorted by size, ascending.
3091  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3092 
3093  bool ValidateFreeSuballocationList() const;
3094 
3095  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3096  // If yes, fills pOffset and returns true. If no, returns false.
3097  bool CheckAllocation(
3098  uint32_t currentFrameIndex,
3099  uint32_t frameInUseCount,
3100  VkDeviceSize bufferImageGranularity,
3101  VkDeviceSize allocSize,
3102  VkDeviceSize allocAlignment,
3103  VmaSuballocationType allocType,
3104  VmaSuballocationList::const_iterator suballocItem,
3105  bool canMakeOtherLost,
3106  VkDeviceSize* pOffset,
3107  size_t* itemsToMakeLostCount,
3108  VkDeviceSize* pSumFreeSize,
3109  VkDeviceSize* pSumItemSize) const;
3110  // Given free suballocation, it merges it with following one, which must also be free.
3111  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3112  // Releases given suballocation, making it free.
3113  // Merges it with adjacent free suballocations if applicable.
3114  // Returns iterator to new free suballocation at this place.
3115  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3116  // Given free suballocation, it inserts it into sorted list of
3117  // m_FreeSuballocationsBySize if it's suitable.
3118  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3119  // Given free suballocation, it removes it from sorted list of
3120  // m_FreeSuballocationsBySize if it's suitable.
3121  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3122 };
3123 
3124 /*
3125 Represents a single block of device memory (`VkDeviceMemory`) with all the
3126 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3127 
3128 Thread-safety: This class must be externally synchronized.
3129 */
3130 class VmaDeviceMemoryBlock
3131 {
3132 public:
3133  uint32_t m_MemoryTypeIndex;
3134  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3135  VkDeviceMemory m_hMemory;
3136  VkDeviceSize m_Size;
3137  bool m_PersistentMap;
3138  void* m_pMappedData;
3139  VmaBlockMetadata m_Metadata;
3140 
3141  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3142 
3143  ~VmaDeviceMemoryBlock()
3144  {
3145  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3146  }
3147 
3148  // Always call after construction.
3149  void Init(
3150  uint32_t newMemoryTypeIndex,
3151  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
3152  VkDeviceMemory newMemory,
3153  VkDeviceSize newSize,
3154  bool persistentMap,
3155  void* pMappedData);
3156  // Always call before destruction.
3157  void Destroy(VmaAllocator allocator);
3158 
3159  // Validates all data structures inside this object. If not valid, returns false.
3160  bool Validate() const;
3161 };
3162 
3163 struct VmaPointerLess
3164 {
3165  bool operator()(const void* lhs, const void* rhs) const
3166  {
3167  return lhs < rhs;
3168  }
3169 };
3170 
3171 class VmaDefragmentator;
3172 
3173 /*
3174 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3175 Vulkan memory type.
3176 
3177 Synchronized internally with a mutex.
3178 */
3179 struct VmaBlockVector
3180 {
3181  VmaBlockVector(
3182  VmaAllocator hAllocator,
3183  uint32_t memoryTypeIndex,
3184  VMA_BLOCK_VECTOR_TYPE blockVectorType,
3185  VkDeviceSize preferredBlockSize,
3186  size_t minBlockCount,
3187  size_t maxBlockCount,
3188  VkDeviceSize bufferImageGranularity,
3189  uint32_t frameInUseCount,
3190  bool isCustomPool);
3191  ~VmaBlockVector();
3192 
3193  VkResult CreateMinBlocks();
3194 
3195  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3196  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3197  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3198  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3199  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; }
3200 
3201  void GetPoolStats(VmaPoolStats* pStats);
3202 
3203  bool IsEmpty() const { return m_Blocks.empty(); }
3204 
3205  VkResult Allocate(
3206  VmaPool hCurrentPool,
3207  uint32_t currentFrameIndex,
3208  const VkMemoryRequirements& vkMemReq,
3209  const VmaAllocationCreateInfo& createInfo,
3210  VmaSuballocationType suballocType,
3211  VmaAllocation* pAllocation);
3212 
3213  void Free(
3214  VmaAllocation hAllocation);
3215 
3216  // Adds statistics of this BlockVector to pStats.
3217  void AddStats(VmaStats* pStats);
3218 
3219 #if VMA_STATS_STRING_ENABLED
3220  void PrintDetailedMap(class VmaJsonWriter& json);
3221 #endif
3222 
3223  void UnmapPersistentlyMappedMemory();
3224  VkResult MapPersistentlyMappedMemory();
3225 
3226  void MakePoolAllocationsLost(
3227  uint32_t currentFrameIndex,
3228  size_t* pLostAllocationCount);
3229 
3230  VmaDefragmentator* EnsureDefragmentator(
3231  VmaAllocator hAllocator,
3232  uint32_t currentFrameIndex);
3233 
3234  VkResult Defragment(
3235  VmaDefragmentationStats* pDefragmentationStats,
3236  VkDeviceSize& maxBytesToMove,
3237  uint32_t& maxAllocationsToMove);
3238 
3239  void DestroyDefragmentator();
3240 
3241 private:
3242  friend class VmaDefragmentator;
3243 
3244  const VmaAllocator m_hAllocator;
3245  const uint32_t m_MemoryTypeIndex;
3246  const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3247  const VkDeviceSize m_PreferredBlockSize;
3248  const size_t m_MinBlockCount;
3249  const size_t m_MaxBlockCount;
3250  const VkDeviceSize m_BufferImageGranularity;
3251  const uint32_t m_FrameInUseCount;
3252  const bool m_IsCustomPool;
3253  VMA_MUTEX m_Mutex;
3254  // Incrementally sorted by sumFreeSize, ascending.
3255  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3256  /* There can be at most one allocation that is completely empty - a
3257  hysteresis to avoid pessimistic case of alternating creation and destruction
3258  of a VkDeviceMemory. */
3259  bool m_HasEmptyBlock;
3260  VmaDefragmentator* m_pDefragmentator;
3261 
3262  // Finds and removes given block from vector.
3263  void Remove(VmaDeviceMemoryBlock* pBlock);
3264 
3265  // Performs single step in sorting m_Blocks. They may not be fully sorted
3266  // after this call.
3267  void IncrementallySortBlocks();
3268 
3269  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3270 };
3271 
3272 struct VmaPool_T
3273 {
3274 public:
3275  VmaBlockVector m_BlockVector;
3276 
3277  // Takes ownership.
3278  VmaPool_T(
3279  VmaAllocator hAllocator,
3280  const VmaPoolCreateInfo& createInfo);
3281  ~VmaPool_T();
3282 
3283  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3284 
3285 #if VMA_STATS_STRING_ENABLED
3286  //void PrintDetailedMap(class VmaStringBuilder& sb);
3287 #endif
3288 };
3289 
3290 class VmaDefragmentator
3291 {
3292  const VmaAllocator m_hAllocator;
3293  VmaBlockVector* const m_pBlockVector;
3294  uint32_t m_CurrentFrameIndex;
3295  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3296  VkDeviceSize m_BytesMoved;
3297  uint32_t m_AllocationsMoved;
3298 
3299  struct AllocationInfo
3300  {
3301  VmaAllocation m_hAllocation;
3302  VkBool32* m_pChanged;
3303 
3304  AllocationInfo() :
3305  m_hAllocation(VK_NULL_HANDLE),
3306  m_pChanged(VMA_NULL)
3307  {
3308  }
3309  };
3310 
3311  struct AllocationInfoSizeGreater
3312  {
3313  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3314  {
3315  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3316  }
3317  };
3318 
3319  // Used between AddAllocation and Defragment.
3320  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3321 
3322  struct BlockInfo
3323  {
3324  VmaDeviceMemoryBlock* m_pBlock;
3325  bool m_HasNonMovableAllocations;
3326  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3327 
3328  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3329  m_pBlock(VMA_NULL),
3330  m_HasNonMovableAllocations(true),
3331  m_Allocations(pAllocationCallbacks),
3332  m_pMappedDataForDefragmentation(VMA_NULL)
3333  {
3334  }
3335 
3336  void CalcHasNonMovableAllocations()
3337  {
3338  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3339  const size_t defragmentAllocCount = m_Allocations.size();
3340  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3341  }
3342 
3343  void SortAllocationsBySizeDescecnding()
3344  {
3345  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3346  }
3347 
3348  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3349  void Unmap(VmaAllocator hAllocator);
3350 
3351  private:
3352  // Not null if mapped for defragmentation only, not persistently mapped.
3353  void* m_pMappedDataForDefragmentation;
3354  };
3355 
3356  struct BlockPointerLess
3357  {
3358  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3359  {
3360  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3361  }
3362  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3363  {
3364  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3365  }
3366  };
3367 
3368  // 1. Blocks with some non-movable allocations go first.
3369  // 2. Blocks with smaller sumFreeSize go first.
3370  struct BlockInfoCompareMoveDestination
3371  {
3372  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3373  {
3374  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3375  {
3376  return true;
3377  }
3378  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3379  {
3380  return false;
3381  }
3382  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3383  {
3384  return true;
3385  }
3386  return false;
3387  }
3388  };
3389 
3390  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3391  BlockInfoVector m_Blocks;
3392 
3393  VkResult DefragmentRound(
3394  VkDeviceSize maxBytesToMove,
3395  uint32_t maxAllocationsToMove);
3396 
3397  static bool MoveMakesSense(
3398  size_t dstBlockIndex, VkDeviceSize dstOffset,
3399  size_t srcBlockIndex, VkDeviceSize srcOffset);
3400 
3401 public:
3402  VmaDefragmentator(
3403  VmaAllocator hAllocator,
3404  VmaBlockVector* pBlockVector,
3405  uint32_t currentFrameIndex);
3406 
3407  ~VmaDefragmentator();
3408 
3409  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3410  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3411 
3412  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3413 
3414  VkResult Defragment(
3415  VkDeviceSize maxBytesToMove,
3416  uint32_t maxAllocationsToMove);
3417 };
3418 
3419 // Main allocator object.
3420 struct VmaAllocator_T
3421 {
3422  bool m_UseMutex;
3423  VkDevice m_hDevice;
3424  bool m_AllocationCallbacksSpecified;
3425  VkAllocationCallbacks m_AllocationCallbacks;
3426  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3427  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
3428  // Counter to allow nested calls to these functions.
3429  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
3430 
3431  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3432  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3433  VMA_MUTEX m_HeapSizeLimitMutex;
3434 
3435  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3436  VkPhysicalDeviceMemoryProperties m_MemProps;
3437 
3438  // Default pools.
3439  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3440 
3441  // Each vector is sorted by memory (handle value).
3442  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3443  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3444  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
3445 
3446  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
3447  ~VmaAllocator_T();
3448 
3449  const VkAllocationCallbacks* GetAllocationCallbacks() const
3450  {
3451  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
3452  }
3453  const VmaVulkanFunctions& GetVulkanFunctions() const
3454  {
3455  return m_VulkanFunctions;
3456  }
3457 
3458  VkDeviceSize GetBufferImageGranularity() const
3459  {
3460  return VMA_MAX(
3461  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
3462  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
3463  }
3464 
3465  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
3466  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
3467 
3468  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
3469  {
3470  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
3471  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3472  }
3473 
3474  // Main allocation function.
3475  VkResult AllocateMemory(
3476  const VkMemoryRequirements& vkMemReq,
3477  const VmaAllocationCreateInfo& createInfo,
3478  VmaSuballocationType suballocType,
3479  VmaAllocation* pAllocation);
3480 
3481  // Main deallocation function.
3482  void FreeMemory(const VmaAllocation allocation);
3483 
3484  void CalculateStats(VmaStats* pStats);
3485 
3486 #if VMA_STATS_STRING_ENABLED
3487  void PrintDetailedMap(class VmaJsonWriter& json);
3488 #endif
3489 
3490  void UnmapPersistentlyMappedMemory();
3491  VkResult MapPersistentlyMappedMemory();
3492 
3493  VkResult Defragment(
3494  VmaAllocation* pAllocations,
3495  size_t allocationCount,
3496  VkBool32* pAllocationsChanged,
3497  const VmaDefragmentationInfo* pDefragmentationInfo,
3498  VmaDefragmentationStats* pDefragmentationStats);
3499 
3500  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
3501 
3502  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
3503  void DestroyPool(VmaPool pool);
3504  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
3505 
3506  void SetCurrentFrameIndex(uint32_t frameIndex);
3507 
3508  void MakePoolAllocationsLost(
3509  VmaPool hPool,
3510  size_t* pLostAllocationCount);
3511 
3512  void CreateLostAllocation(VmaAllocation* pAllocation);
3513 
3514  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
3515  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
3516 
3517 private:
3518  VkDeviceSize m_PreferredLargeHeapBlockSize;
3519  VkDeviceSize m_PreferredSmallHeapBlockSize;
3520 
3521  VkPhysicalDevice m_PhysicalDevice;
3522  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
3523 
3524  VMA_MUTEX m_PoolsMutex;
3525  // Protected by m_PoolsMutex. Sorted by pointer value.
3526  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
3527 
3528  VmaVulkanFunctions m_VulkanFunctions;
3529 
3530  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
3531 
3532  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
3533 
3534  VkResult AllocateMemoryOfType(
3535  const VkMemoryRequirements& vkMemReq,
3536  const VmaAllocationCreateInfo& createInfo,
3537  uint32_t memTypeIndex,
3538  VmaSuballocationType suballocType,
3539  VmaAllocation* pAllocation);
3540 
3541  // Allocates and registers new VkDeviceMemory specifically for single allocation.
3542  VkResult AllocateOwnMemory(
3543  VkDeviceSize size,
3544  VmaSuballocationType suballocType,
3545  uint32_t memTypeIndex,
3546  bool map,
3547  void* pUserData,
3548  VmaAllocation* pAllocation);
3549 
3550  // Tries to free pMemory as Own Memory. Returns true if found and freed.
3551  void FreeOwnMemory(VmaAllocation allocation);
3552 };
3553 
3555 // Memory allocation #2 after VmaAllocator_T definition
3556 
3557 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
3558 {
3559  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
3560 }
3561 
3562 static void VmaFree(VmaAllocator hAllocator, void* ptr)
3563 {
3564  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
3565 }
3566 
3567 template<typename T>
3568 static T* VmaAllocate(VmaAllocator hAllocator)
3569 {
3570  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
3571 }
3572 
3573 template<typename T>
3574 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
3575 {
3576  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
3577 }
3578 
3579 template<typename T>
3580 static void vma_delete(VmaAllocator hAllocator, T* ptr)
3581 {
3582  if(ptr != VMA_NULL)
3583  {
3584  ptr->~T();
3585  VmaFree(hAllocator, ptr);
3586  }
3587 }
3588 
3589 template<typename T>
3590 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
3591 {
3592  if(ptr != VMA_NULL)
3593  {
3594  for(size_t i = count; i--; )
3595  ptr[i].~T();
3596  VmaFree(hAllocator, ptr);
3597  }
3598 }
3599 
3601 // VmaStringBuilder
3602 
3603 #if VMA_STATS_STRING_ENABLED
3604 
3605 class VmaStringBuilder
3606 {
3607 public:
3608  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
3609  size_t GetLength() const { return m_Data.size(); }
3610  const char* GetData() const { return m_Data.data(); }
3611 
3612  void Add(char ch) { m_Data.push_back(ch); }
3613  void Add(const char* pStr);
3614  void AddNewLine() { Add('\n'); }
3615  void AddNumber(uint32_t num);
3616  void AddNumber(uint64_t num);
3617  void AddPointer(const void* ptr);
3618 
3619 private:
3620  VmaVector< char, VmaStlAllocator<char> > m_Data;
3621 };
3622 
3623 void VmaStringBuilder::Add(const char* pStr)
3624 {
3625  const size_t strLen = strlen(pStr);
3626  if(strLen > 0)
3627  {
3628  const size_t oldCount = m_Data.size();
3629  m_Data.resize(oldCount + strLen);
3630  memcpy(m_Data.data() + oldCount, pStr, strLen);
3631  }
3632 }
3633 
3634 void VmaStringBuilder::AddNumber(uint32_t num)
3635 {
3636  char buf[11];
3637  VmaUint32ToStr(buf, sizeof(buf), num);
3638  Add(buf);
3639 }
3640 
3641 void VmaStringBuilder::AddNumber(uint64_t num)
3642 {
3643  char buf[21];
3644  VmaUint64ToStr(buf, sizeof(buf), num);
3645  Add(buf);
3646 }
3647 
3648 void VmaStringBuilder::AddPointer(const void* ptr)
3649 {
3650  char buf[21];
3651  VmaPtrToStr(buf, sizeof(buf), ptr);
3652  Add(buf);
3653 }
3654 
3655 #endif // #if VMA_STATS_STRING_ENABLED
3656 
3658 // VmaJsonWriter
3659 
3660 #if VMA_STATS_STRING_ENABLED
3661 
3662 class VmaJsonWriter
3663 {
3664 public:
3665  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
3666  ~VmaJsonWriter();
3667 
3668  void BeginObject(bool singleLine = false);
3669  void EndObject();
3670 
3671  void BeginArray(bool singleLine = false);
3672  void EndArray();
3673 
3674  void WriteString(const char* pStr);
3675  void BeginString(const char* pStr = VMA_NULL);
3676  void ContinueString(const char* pStr);
3677  void ContinueString(uint32_t n);
3678  void ContinueString(uint64_t n);
3679  void EndString(const char* pStr = VMA_NULL);
3680 
3681  void WriteNumber(uint32_t n);
3682  void WriteNumber(uint64_t n);
3683  void WriteBool(bool b);
3684  void WriteNull();
3685 
3686 private:
3687  static const char* const INDENT;
3688 
3689  enum COLLECTION_TYPE
3690  {
3691  COLLECTION_TYPE_OBJECT,
3692  COLLECTION_TYPE_ARRAY,
3693  };
3694  struct StackItem
3695  {
3696  COLLECTION_TYPE type;
3697  uint32_t valueCount;
3698  bool singleLineMode;
3699  };
3700 
3701  VmaStringBuilder& m_SB;
3702  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
3703  bool m_InsideString;
3704 
3705  void BeginValue(bool isString);
3706  void WriteIndent(bool oneLess = false);
3707 };
3708 
3709 const char* const VmaJsonWriter::INDENT = " ";
3710 
3711 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
3712  m_SB(sb),
3713  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
3714  m_InsideString(false)
3715 {
3716 }
3717 
3718 VmaJsonWriter::~VmaJsonWriter()
3719 {
3720  VMA_ASSERT(!m_InsideString);
3721  VMA_ASSERT(m_Stack.empty());
3722 }
3723 
3724 void VmaJsonWriter::BeginObject(bool singleLine)
3725 {
3726  VMA_ASSERT(!m_InsideString);
3727 
3728  BeginValue(false);
3729  m_SB.Add('{');
3730 
3731  StackItem item;
3732  item.type = COLLECTION_TYPE_OBJECT;
3733  item.valueCount = 0;
3734  item.singleLineMode = singleLine;
3735  m_Stack.push_back(item);
3736 }
3737 
3738 void VmaJsonWriter::EndObject()
3739 {
3740  VMA_ASSERT(!m_InsideString);
3741 
3742  WriteIndent(true);
3743  m_SB.Add('}');
3744 
3745  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
3746  m_Stack.pop_back();
3747 }
3748 
3749 void VmaJsonWriter::BeginArray(bool singleLine)
3750 {
3751  VMA_ASSERT(!m_InsideString);
3752 
3753  BeginValue(false);
3754  m_SB.Add('[');
3755 
3756  StackItem item;
3757  item.type = COLLECTION_TYPE_ARRAY;
3758  item.valueCount = 0;
3759  item.singleLineMode = singleLine;
3760  m_Stack.push_back(item);
3761 }
3762 
3763 void VmaJsonWriter::EndArray()
3764 {
3765  VMA_ASSERT(!m_InsideString);
3766 
3767  WriteIndent(true);
3768  m_SB.Add(']');
3769 
3770  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
3771  m_Stack.pop_back();
3772 }
3773 
3774 void VmaJsonWriter::WriteString(const char* pStr)
3775 {
3776  BeginString(pStr);
3777  EndString();
3778 }
3779 
3780 void VmaJsonWriter::BeginString(const char* pStr)
3781 {
3782  VMA_ASSERT(!m_InsideString);
3783 
3784  BeginValue(true);
3785  m_SB.Add('"');
3786  m_InsideString = true;
3787  if(pStr != VMA_NULL && pStr[0] != '\0')
3788  {
3789  ContinueString(pStr);
3790  }
3791 }
3792 
3793 void VmaJsonWriter::ContinueString(const char* pStr)
3794 {
3795  VMA_ASSERT(m_InsideString);
3796 
3797  const size_t strLen = strlen(pStr);
3798  for(size_t i = 0; i < strLen; ++i)
3799  {
3800  char ch = pStr[i];
3801  if(ch == '\'')
3802  {
3803  m_SB.Add("\\\\");
3804  }
3805  else if(ch == '"')
3806  {
3807  m_SB.Add("\\\"");
3808  }
3809  else if(ch >= 32)
3810  {
3811  m_SB.Add(ch);
3812  }
3813  else switch(ch)
3814  {
3815  case '\n':
3816  m_SB.Add("\\n");
3817  break;
3818  case '\r':
3819  m_SB.Add("\\r");
3820  break;
3821  case '\t':
3822  m_SB.Add("\\t");
3823  break;
3824  default:
3825  VMA_ASSERT(0 && "Character not currently supported.");
3826  break;
3827  }
3828  }
3829 }
3830 
3831 void VmaJsonWriter::ContinueString(uint32_t n)
3832 {
3833  VMA_ASSERT(m_InsideString);
3834  m_SB.AddNumber(n);
3835 }
3836 
3837 void VmaJsonWriter::ContinueString(uint64_t n)
3838 {
3839  VMA_ASSERT(m_InsideString);
3840  m_SB.AddNumber(n);
3841 }
3842 
3843 void VmaJsonWriter::EndString(const char* pStr)
3844 {
3845  VMA_ASSERT(m_InsideString);
3846  if(pStr != VMA_NULL && pStr[0] != '\0')
3847  {
3848  ContinueString(pStr);
3849  }
3850  m_SB.Add('"');
3851  m_InsideString = false;
3852 }
3853 
3854 void VmaJsonWriter::WriteNumber(uint32_t n)
3855 {
3856  VMA_ASSERT(!m_InsideString);
3857  BeginValue(false);
3858  m_SB.AddNumber(n);
3859 }
3860 
3861 void VmaJsonWriter::WriteNumber(uint64_t n)
3862 {
3863  VMA_ASSERT(!m_InsideString);
3864  BeginValue(false);
3865  m_SB.AddNumber(n);
3866 }
3867 
3868 void VmaJsonWriter::WriteBool(bool b)
3869 {
3870  VMA_ASSERT(!m_InsideString);
3871  BeginValue(false);
3872  m_SB.Add(b ? "true" : "false");
3873 }
3874 
3875 void VmaJsonWriter::WriteNull()
3876 {
3877  VMA_ASSERT(!m_InsideString);
3878  BeginValue(false);
3879  m_SB.Add("null");
3880 }
3881 
3882 void VmaJsonWriter::BeginValue(bool isString)
3883 {
3884  if(!m_Stack.empty())
3885  {
3886  StackItem& currItem = m_Stack.back();
3887  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3888  currItem.valueCount % 2 == 0)
3889  {
3890  VMA_ASSERT(isString);
3891  }
3892 
3893  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3894  currItem.valueCount % 2 != 0)
3895  {
3896  m_SB.Add(": ");
3897  }
3898  else if(currItem.valueCount > 0)
3899  {
3900  m_SB.Add(", ");
3901  WriteIndent();
3902  }
3903  else
3904  {
3905  WriteIndent();
3906  }
3907  ++currItem.valueCount;
3908  }
3909 }
3910 
3911 void VmaJsonWriter::WriteIndent(bool oneLess)
3912 {
3913  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
3914  {
3915  m_SB.AddNewLine();
3916 
3917  size_t count = m_Stack.size();
3918  if(count > 0 && oneLess)
3919  {
3920  --count;
3921  }
3922  for(size_t i = 0; i < count; ++i)
3923  {
3924  m_SB.Add(INDENT);
3925  }
3926  }
3927 }
3928 
3929 #endif // #if VMA_STATS_STRING_ENABLED
3930 
3932 
3933 VkDeviceSize VmaAllocation_T::GetOffset() const
3934 {
3935  switch(m_Type)
3936  {
3937  case ALLOCATION_TYPE_BLOCK:
3938  return m_BlockAllocation.m_Offset;
3939  case ALLOCATION_TYPE_OWN:
3940  return 0;
3941  default:
3942  VMA_ASSERT(0);
3943  return 0;
3944  }
3945 }
3946 
3947 VkDeviceMemory VmaAllocation_T::GetMemory() const
3948 {
3949  switch(m_Type)
3950  {
3951  case ALLOCATION_TYPE_BLOCK:
3952  return m_BlockAllocation.m_Block->m_hMemory;
3953  case ALLOCATION_TYPE_OWN:
3954  return m_OwnAllocation.m_hMemory;
3955  default:
3956  VMA_ASSERT(0);
3957  return VK_NULL_HANDLE;
3958  }
3959 }
3960 
3961 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
3962 {
3963  switch(m_Type)
3964  {
3965  case ALLOCATION_TYPE_BLOCK:
3966  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
3967  case ALLOCATION_TYPE_OWN:
3968  return m_OwnAllocation.m_MemoryTypeIndex;
3969  default:
3970  VMA_ASSERT(0);
3971  return UINT32_MAX;
3972  }
3973 }
3974 
3975 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
3976 {
3977  switch(m_Type)
3978  {
3979  case ALLOCATION_TYPE_BLOCK:
3980  return m_BlockAllocation.m_Block->m_BlockVectorType;
3981  case ALLOCATION_TYPE_OWN:
3982  return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
3983  default:
3984  VMA_ASSERT(0);
3985  return VMA_BLOCK_VECTOR_TYPE_COUNT;
3986  }
3987 }
3988 
3989 void* VmaAllocation_T::GetMappedData() const
3990 {
3991  switch(m_Type)
3992  {
3993  case ALLOCATION_TYPE_BLOCK:
3994  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
3995  {
3996  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
3997  }
3998  else
3999  {
4000  return VMA_NULL;
4001  }
4002  break;
4003  case ALLOCATION_TYPE_OWN:
4004  return m_OwnAllocation.m_pMappedData;
4005  default:
4006  VMA_ASSERT(0);
4007  return VMA_NULL;
4008  }
4009 }
4010 
4011 bool VmaAllocation_T::CanBecomeLost() const
4012 {
4013  switch(m_Type)
4014  {
4015  case ALLOCATION_TYPE_BLOCK:
4016  return m_BlockAllocation.m_CanBecomeLost;
4017  case ALLOCATION_TYPE_OWN:
4018  return false;
4019  default:
4020  VMA_ASSERT(0);
4021  return false;
4022  }
4023 }
4024 
4025 VmaPool VmaAllocation_T::GetPool() const
4026 {
4027  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4028  return m_BlockAllocation.m_hPool;
4029 }
4030 
4031 VkResult VmaAllocation_T::OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator)
4032 {
4033  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4034  if(m_OwnAllocation.m_PersistentMap)
4035  {
4036  return (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4037  hAllocator->m_hDevice,
4038  m_OwnAllocation.m_hMemory,
4039  0,
4040  VK_WHOLE_SIZE,
4041  0,
4042  &m_OwnAllocation.m_pMappedData);
4043  }
4044  return VK_SUCCESS;
4045 }
4046 void VmaAllocation_T::OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator)
4047 {
4048  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4049  if(m_OwnAllocation.m_pMappedData)
4050  {
4051  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
4052  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_OwnAllocation.m_hMemory);
4053  m_OwnAllocation.m_pMappedData = VMA_NULL;
4054  }
4055 }
4056 
4057 
4058 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4059 {
4060  VMA_ASSERT(CanBecomeLost());
4061 
4062  /*
4063  Warning: This is a carefully designed algorithm.
4064  Do not modify unless you really know what you're doing :)
4065  */
4066  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4067  for(;;)
4068  {
4069  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4070  {
4071  VMA_ASSERT(0);
4072  return false;
4073  }
4074  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4075  {
4076  return false;
4077  }
4078  else // Last use time earlier than current time.
4079  {
4080  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4081  {
4082  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4083  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4084  return true;
4085  }
4086  }
4087  }
4088 }
4089 
4090 #if VMA_STATS_STRING_ENABLED
4091 
4092 // Correspond to values of enum VmaSuballocationType.
4093 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4094  "FREE",
4095  "UNKNOWN",
4096  "BUFFER",
4097  "IMAGE_UNKNOWN",
4098  "IMAGE_LINEAR",
4099  "IMAGE_OPTIMAL",
4100 };
4101 
4102 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4103 {
4104  json.BeginObject();
4105 
4106  json.WriteString("Blocks");
4107  json.WriteNumber(stat.blockCount);
4108 
4109  json.WriteString("Allocations");
4110  json.WriteNumber(stat.allocationCount);
4111 
4112  json.WriteString("UnusedRanges");
4113  json.WriteNumber(stat.unusedRangeCount);
4114 
4115  json.WriteString("UsedBytes");
4116  json.WriteNumber(stat.usedBytes);
4117 
4118  json.WriteString("UnusedBytes");
4119  json.WriteNumber(stat.unusedBytes);
4120 
4121  if(stat.allocationCount > 1)
4122  {
4123  json.WriteString("AllocationSize");
4124  json.BeginObject(true);
4125  json.WriteString("Min");
4126  json.WriteNumber(stat.allocationSizeMin);
4127  json.WriteString("Avg");
4128  json.WriteNumber(stat.allocationSizeAvg);
4129  json.WriteString("Max");
4130  json.WriteNumber(stat.allocationSizeMax);
4131  json.EndObject();
4132  }
4133 
4134  if(stat.unusedRangeCount > 1)
4135  {
4136  json.WriteString("UnusedRangeSize");
4137  json.BeginObject(true);
4138  json.WriteString("Min");
4139  json.WriteNumber(stat.unusedRangeSizeMin);
4140  json.WriteString("Avg");
4141  json.WriteNumber(stat.unusedRangeSizeAvg);
4142  json.WriteString("Max");
4143  json.WriteNumber(stat.unusedRangeSizeMax);
4144  json.EndObject();
4145  }
4146 
4147  json.EndObject();
4148 }
4149 
4150 #endif // #if VMA_STATS_STRING_ENABLED
4151 
4152 struct VmaSuballocationItemSizeLess
4153 {
4154  bool operator()(
4155  const VmaSuballocationList::iterator lhs,
4156  const VmaSuballocationList::iterator rhs) const
4157  {
4158  return lhs->size < rhs->size;
4159  }
4160  bool operator()(
4161  const VmaSuballocationList::iterator lhs,
4162  VkDeviceSize rhsSize) const
4163  {
4164  return lhs->size < rhsSize;
4165  }
4166 };
4167 
4169 // class VmaBlockMetadata
4170 
4171 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4172  m_Size(0),
4173  m_FreeCount(0),
4174  m_SumFreeSize(0),
4175  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4176  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4177 {
4178 }
4179 
4180 VmaBlockMetadata::~VmaBlockMetadata()
4181 {
4182 }
4183 
4184 void VmaBlockMetadata::Init(VkDeviceSize size)
4185 {
4186  m_Size = size;
4187  m_FreeCount = 1;
4188  m_SumFreeSize = size;
4189 
4190  VmaSuballocation suballoc = {};
4191  suballoc.offset = 0;
4192  suballoc.size = size;
4193  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4194  suballoc.hAllocation = VK_NULL_HANDLE;
4195 
4196  m_Suballocations.push_back(suballoc);
4197  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4198  --suballocItem;
4199  m_FreeSuballocationsBySize.push_back(suballocItem);
4200 }
4201 
4202 bool VmaBlockMetadata::Validate() const
4203 {
4204  if(m_Suballocations.empty())
4205  {
4206  return false;
4207  }
4208 
4209  // Expected offset of new suballocation as calculates from previous ones.
4210  VkDeviceSize calculatedOffset = 0;
4211  // Expected number of free suballocations as calculated from traversing their list.
4212  uint32_t calculatedFreeCount = 0;
4213  // Expected sum size of free suballocations as calculated from traversing their list.
4214  VkDeviceSize calculatedSumFreeSize = 0;
4215  // Expected number of free suballocations that should be registered in
4216  // m_FreeSuballocationsBySize calculated from traversing their list.
4217  size_t freeSuballocationsToRegister = 0;
4218  // True if previous visisted suballocation was free.
4219  bool prevFree = false;
4220 
4221  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4222  suballocItem != m_Suballocations.cend();
4223  ++suballocItem)
4224  {
4225  const VmaSuballocation& subAlloc = *suballocItem;
4226 
4227  // Actual offset of this suballocation doesn't match expected one.
4228  if(subAlloc.offset != calculatedOffset)
4229  {
4230  return false;
4231  }
4232 
4233  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4234  // Two adjacent free suballocations are invalid. They should be merged.
4235  if(prevFree && currFree)
4236  {
4237  return false;
4238  }
4239  prevFree = currFree;
4240 
4241  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4242  {
4243  return false;
4244  }
4245 
4246  if(currFree)
4247  {
4248  calculatedSumFreeSize += subAlloc.size;
4249  ++calculatedFreeCount;
4250  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4251  {
4252  ++freeSuballocationsToRegister;
4253  }
4254  }
4255 
4256  calculatedOffset += subAlloc.size;
4257  }
4258 
4259  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4260  // match expected one.
4261  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4262  {
4263  return false;
4264  }
4265 
4266  VkDeviceSize lastSize = 0;
4267  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4268  {
4269  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4270 
4271  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4272  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4273  {
4274  return false;
4275  }
4276  // They must be sorted by size ascending.
4277  if(suballocItem->size < lastSize)
4278  {
4279  return false;
4280  }
4281 
4282  lastSize = suballocItem->size;
4283  }
4284 
4285  // Check if totals match calculacted values.
4286  return
4287  ValidateFreeSuballocationList() &&
4288  (calculatedOffset == m_Size) &&
4289  (calculatedSumFreeSize == m_SumFreeSize) &&
4290  (calculatedFreeCount == m_FreeCount);
4291 }
4292 
4293 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
4294 {
4295  if(!m_FreeSuballocationsBySize.empty())
4296  {
4297  return m_FreeSuballocationsBySize.back()->size;
4298  }
4299  else
4300  {
4301  return 0;
4302  }
4303 }
4304 
4305 bool VmaBlockMetadata::IsEmpty() const
4306 {
4307  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
4308 }
4309 
4310 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
4311 {
4312  outInfo.blockCount = 1;
4313 
4314  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4315  outInfo.allocationCount = rangeCount - m_FreeCount;
4316  outInfo.unusedRangeCount = m_FreeCount;
4317 
4318  outInfo.unusedBytes = m_SumFreeSize;
4319  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
4320 
4321  outInfo.allocationSizeMin = UINT64_MAX;
4322  outInfo.allocationSizeMax = 0;
4323  outInfo.unusedRangeSizeMin = UINT64_MAX;
4324  outInfo.unusedRangeSizeMax = 0;
4325 
4326  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4327  suballocItem != m_Suballocations.cend();
4328  ++suballocItem)
4329  {
4330  const VmaSuballocation& suballoc = *suballocItem;
4331  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
4332  {
4333  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
4334  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
4335  }
4336  else
4337  {
4338  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
4339  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
4340  }
4341  }
4342 }
4343 
4344 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
4345 {
4346  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4347 
4348  inoutStats.size += m_Size;
4349  inoutStats.unusedSize += m_SumFreeSize;
4350  inoutStats.allocationCount += rangeCount - m_FreeCount;
4351  inoutStats.unusedRangeCount += m_FreeCount;
4352  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
4353 }
4354 
4355 #if VMA_STATS_STRING_ENABLED
4356 
4357 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
4358 {
4359  json.BeginObject();
4360 
4361  json.WriteString("TotalBytes");
4362  json.WriteNumber(m_Size);
4363 
4364  json.WriteString("UnusedBytes");
4365  json.WriteNumber(m_SumFreeSize);
4366 
4367  json.WriteString("Allocations");
4368  json.WriteNumber(m_Suballocations.size() - m_FreeCount);
4369 
4370  json.WriteString("UnusedRanges");
4371  json.WriteNumber(m_FreeCount);
4372 
4373  json.WriteString("Suballocations");
4374  json.BeginArray();
4375  size_t i = 0;
4376  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4377  suballocItem != m_Suballocations.cend();
4378  ++suballocItem, ++i)
4379  {
4380  json.BeginObject(true);
4381 
4382  json.WriteString("Type");
4383  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
4384 
4385  json.WriteString("Size");
4386  json.WriteNumber(suballocItem->size);
4387 
4388  json.WriteString("Offset");
4389  json.WriteNumber(suballocItem->offset);
4390 
4391  json.EndObject();
4392  }
4393  json.EndArray();
4394 
4395  json.EndObject();
4396 }
4397 
4398 #endif // #if VMA_STATS_STRING_ENABLED
4399 
4400 /*
4401 How many suitable free suballocations to analyze before choosing best one.
4402 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
4403  be chosen.
4404 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
4405  suballocations will be analized and best one will be chosen.
4406 - Any other value is also acceptable.
4407 */
4408 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
4409 
4410 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
4411 {
4412  VMA_ASSERT(IsEmpty());
4413  pAllocationRequest->offset = 0;
4414  pAllocationRequest->sumFreeSize = m_SumFreeSize;
4415  pAllocationRequest->sumItemSize = 0;
4416  pAllocationRequest->item = m_Suballocations.begin();
4417  pAllocationRequest->itemsToMakeLostCount = 0;
4418 }
4419 
4420 bool VmaBlockMetadata::CreateAllocationRequest(
4421  uint32_t currentFrameIndex,
4422  uint32_t frameInUseCount,
4423  VkDeviceSize bufferImageGranularity,
4424  VkDeviceSize allocSize,
4425  VkDeviceSize allocAlignment,
4426  VmaSuballocationType allocType,
4427  bool canMakeOtherLost,
4428  VmaAllocationRequest* pAllocationRequest)
4429 {
4430  VMA_ASSERT(allocSize > 0);
4431  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4432  VMA_ASSERT(pAllocationRequest != VMA_NULL);
4433  VMA_HEAVY_ASSERT(Validate());
4434 
4435  // There is not enough total free space in this block to fullfill the request: Early return.
4436  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
4437  {
4438  return false;
4439  }
4440 
4441  // New algorithm, efficiently searching freeSuballocationsBySize.
4442  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
4443  if(freeSuballocCount > 0)
4444  {
4445  if(VMA_BEST_FIT)
4446  {
4447  // Find first free suballocation with size not less than allocSize.
4448  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
4449  m_FreeSuballocationsBySize.data(),
4450  m_FreeSuballocationsBySize.data() + freeSuballocCount,
4451  allocSize,
4452  VmaSuballocationItemSizeLess());
4453  size_t index = it - m_FreeSuballocationsBySize.data();
4454  for(; index < freeSuballocCount; ++index)
4455  {
4456  if(CheckAllocation(
4457  currentFrameIndex,
4458  frameInUseCount,
4459  bufferImageGranularity,
4460  allocSize,
4461  allocAlignment,
4462  allocType,
4463  m_FreeSuballocationsBySize[index],
4464  false, // canMakeOtherLost
4465  &pAllocationRequest->offset,
4466  &pAllocationRequest->itemsToMakeLostCount,
4467  &pAllocationRequest->sumFreeSize,
4468  &pAllocationRequest->sumItemSize))
4469  {
4470  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4471  return true;
4472  }
4473  }
4474  }
4475  else
4476  {
4477  // Search staring from biggest suballocations.
4478  for(size_t index = freeSuballocCount; index--; )
4479  {
4480  if(CheckAllocation(
4481  currentFrameIndex,
4482  frameInUseCount,
4483  bufferImageGranularity,
4484  allocSize,
4485  allocAlignment,
4486  allocType,
4487  m_FreeSuballocationsBySize[index],
4488  false, // canMakeOtherLost
4489  &pAllocationRequest->offset,
4490  &pAllocationRequest->itemsToMakeLostCount,
4491  &pAllocationRequest->sumFreeSize,
4492  &pAllocationRequest->sumItemSize))
4493  {
4494  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4495  return true;
4496  }
4497  }
4498  }
4499  }
4500 
4501  if(canMakeOtherLost)
4502  {
4503  // Brute-force algorithm. TODO: Come up with something better.
4504 
4505  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
4506  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
4507 
4508  VmaAllocationRequest tmpAllocRequest = {};
4509  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
4510  suballocIt != m_Suballocations.end();
4511  ++suballocIt)
4512  {
4513  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
4514  suballocIt->hAllocation->CanBecomeLost())
4515  {
4516  if(CheckAllocation(
4517  currentFrameIndex,
4518  frameInUseCount,
4519  bufferImageGranularity,
4520  allocSize,
4521  allocAlignment,
4522  allocType,
4523  suballocIt,
4524  canMakeOtherLost,
4525  &tmpAllocRequest.offset,
4526  &tmpAllocRequest.itemsToMakeLostCount,
4527  &tmpAllocRequest.sumFreeSize,
4528  &tmpAllocRequest.sumItemSize))
4529  {
4530  tmpAllocRequest.item = suballocIt;
4531 
4532  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
4533  {
4534  *pAllocationRequest = tmpAllocRequest;
4535  }
4536  }
4537  }
4538  }
4539 
4540  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
4541  {
4542  return true;
4543  }
4544  }
4545 
4546  return false;
4547 }
4548 
4549 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
4550  uint32_t currentFrameIndex,
4551  uint32_t frameInUseCount,
4552  VmaAllocationRequest* pAllocationRequest)
4553 {
4554  while(pAllocationRequest->itemsToMakeLostCount > 0)
4555  {
4556  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
4557  {
4558  ++pAllocationRequest->item;
4559  }
4560  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4561  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
4562  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
4563  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4564  {
4565  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
4566  --pAllocationRequest->itemsToMakeLostCount;
4567  }
4568  else
4569  {
4570  return false;
4571  }
4572  }
4573 
4574  VMA_HEAVY_ASSERT(Validate());
4575  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4576  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
4577 
4578  return true;
4579 }
4580 
4581 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4582 {
4583  uint32_t lostAllocationCount = 0;
4584  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
4585  it != m_Suballocations.end();
4586  ++it)
4587  {
4588  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
4589  it->hAllocation->CanBecomeLost() &&
4590  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4591  {
4592  it = FreeSuballocation(it);
4593  ++lostAllocationCount;
4594  }
4595  }
4596  return lostAllocationCount;
4597 }
4598 
4599 void VmaBlockMetadata::Alloc(
4600  const VmaAllocationRequest& request,
4601  VmaSuballocationType type,
4602  VkDeviceSize allocSize,
4603  VmaAllocation hAllocation)
4604 {
4605  VMA_ASSERT(request.item != m_Suballocations.end());
4606  VmaSuballocation& suballoc = *request.item;
4607  // Given suballocation is a free block.
4608  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4609  // Given offset is inside this suballocation.
4610  VMA_ASSERT(request.offset >= suballoc.offset);
4611  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
4612  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
4613  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
4614 
4615  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
4616  // it to become used.
4617  UnregisterFreeSuballocation(request.item);
4618 
4619  suballoc.offset = request.offset;
4620  suballoc.size = allocSize;
4621  suballoc.type = type;
4622  suballoc.hAllocation = hAllocation;
4623 
4624  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
4625  if(paddingEnd)
4626  {
4627  VmaSuballocation paddingSuballoc = {};
4628  paddingSuballoc.offset = request.offset + allocSize;
4629  paddingSuballoc.size = paddingEnd;
4630  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4631  VmaSuballocationList::iterator next = request.item;
4632  ++next;
4633  const VmaSuballocationList::iterator paddingEndItem =
4634  m_Suballocations.insert(next, paddingSuballoc);
4635  RegisterFreeSuballocation(paddingEndItem);
4636  }
4637 
4638  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
4639  if(paddingBegin)
4640  {
4641  VmaSuballocation paddingSuballoc = {};
4642  paddingSuballoc.offset = request.offset - paddingBegin;
4643  paddingSuballoc.size = paddingBegin;
4644  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4645  const VmaSuballocationList::iterator paddingBeginItem =
4646  m_Suballocations.insert(request.item, paddingSuballoc);
4647  RegisterFreeSuballocation(paddingBeginItem);
4648  }
4649 
4650  // Update totals.
4651  m_FreeCount = m_FreeCount - 1;
4652  if(paddingBegin > 0)
4653  {
4654  ++m_FreeCount;
4655  }
4656  if(paddingEnd > 0)
4657  {
4658  ++m_FreeCount;
4659  }
4660  m_SumFreeSize -= allocSize;
4661 }
4662 
4663 void VmaBlockMetadata::Free(const VmaAllocation allocation)
4664 {
4665  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
4666  suballocItem != m_Suballocations.end();
4667  ++suballocItem)
4668  {
4669  VmaSuballocation& suballoc = *suballocItem;
4670  if(suballoc.hAllocation == allocation)
4671  {
4672  FreeSuballocation(suballocItem);
4673  VMA_HEAVY_ASSERT(Validate());
4674  return;
4675  }
4676  }
4677  VMA_ASSERT(0 && "Not found!");
4678 }
4679 
4680 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
4681 {
4682  VkDeviceSize lastSize = 0;
4683  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
4684  {
4685  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
4686 
4687  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
4688  {
4689  VMA_ASSERT(0);
4690  return false;
4691  }
4692  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4693  {
4694  VMA_ASSERT(0);
4695  return false;
4696  }
4697  if(it->size < lastSize)
4698  {
4699  VMA_ASSERT(0);
4700  return false;
4701  }
4702 
4703  lastSize = it->size;
4704  }
4705  return true;
4706 }
4707 
4708 bool VmaBlockMetadata::CheckAllocation(
4709  uint32_t currentFrameIndex,
4710  uint32_t frameInUseCount,
4711  VkDeviceSize bufferImageGranularity,
4712  VkDeviceSize allocSize,
4713  VkDeviceSize allocAlignment,
4714  VmaSuballocationType allocType,
4715  VmaSuballocationList::const_iterator suballocItem,
4716  bool canMakeOtherLost,
4717  VkDeviceSize* pOffset,
4718  size_t* itemsToMakeLostCount,
4719  VkDeviceSize* pSumFreeSize,
4720  VkDeviceSize* pSumItemSize) const
4721 {
4722  VMA_ASSERT(allocSize > 0);
4723  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4724  VMA_ASSERT(suballocItem != m_Suballocations.cend());
4725  VMA_ASSERT(pOffset != VMA_NULL);
4726 
4727  *itemsToMakeLostCount = 0;
4728  *pSumFreeSize = 0;
4729  *pSumItemSize = 0;
4730 
4731  if(canMakeOtherLost)
4732  {
4733  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4734  {
4735  *pSumFreeSize = suballocItem->size;
4736  }
4737  else
4738  {
4739  if(suballocItem->hAllocation->CanBecomeLost() &&
4740  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4741  {
4742  ++*itemsToMakeLostCount;
4743  *pSumItemSize = suballocItem->size;
4744  }
4745  else
4746  {
4747  return false;
4748  }
4749  }
4750 
4751  // Remaining size is too small for this request: Early return.
4752  if(m_Size - suballocItem->offset < allocSize)
4753  {
4754  return false;
4755  }
4756 
4757  // Start from offset equal to beginning of this suballocation.
4758  *pOffset = suballocItem->offset;
4759 
4760  // Apply VMA_DEBUG_MARGIN at the beginning.
4761  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4762  {
4763  *pOffset += VMA_DEBUG_MARGIN;
4764  }
4765 
4766  // Apply alignment.
4767  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4768  *pOffset = VmaAlignUp(*pOffset, alignment);
4769 
4770  // Check previous suballocations for BufferImageGranularity conflicts.
4771  // Make bigger alignment if necessary.
4772  if(bufferImageGranularity > 1)
4773  {
4774  bool bufferImageGranularityConflict = false;
4775  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4776  while(prevSuballocItem != m_Suballocations.cbegin())
4777  {
4778  --prevSuballocItem;
4779  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4780  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4781  {
4782  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4783  {
4784  bufferImageGranularityConflict = true;
4785  break;
4786  }
4787  }
4788  else
4789  // Already on previous page.
4790  break;
4791  }
4792  if(bufferImageGranularityConflict)
4793  {
4794  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4795  }
4796  }
4797 
4798  // Now that we have final *pOffset, check if we are past suballocItem.
4799  // If yes, return false - this function should be called for another suballocItem as starting point.
4800  if(*pOffset >= suballocItem->offset + suballocItem->size)
4801  {
4802  return false;
4803  }
4804 
4805  // Calculate padding at the beginning based on current offset.
4806  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
4807 
4808  // Calculate required margin at the end if this is not last suballocation.
4809  VmaSuballocationList::const_iterator next = suballocItem;
4810  ++next;
4811  const VkDeviceSize requiredEndMargin =
4812  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4813 
4814  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
4815  // Another early return check.
4816  if(suballocItem->offset + totalSize > m_Size)
4817  {
4818  return false;
4819  }
4820 
4821  // Advance lastSuballocItem until desired size is reached.
4822  // Update itemsToMakeLostCount.
4823  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
4824  if(totalSize > suballocItem->size)
4825  {
4826  VkDeviceSize remainingSize = totalSize - suballocItem->size;
4827  while(remainingSize > 0)
4828  {
4829  ++lastSuballocItem;
4830  if(lastSuballocItem == m_Suballocations.cend())
4831  {
4832  return false;
4833  }
4834  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4835  {
4836  *pSumFreeSize += lastSuballocItem->size;
4837  }
4838  else
4839  {
4840  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
4841  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
4842  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4843  {
4844  ++*itemsToMakeLostCount;
4845  *pSumItemSize += lastSuballocItem->size;
4846  }
4847  else
4848  {
4849  return false;
4850  }
4851  }
4852  remainingSize = (lastSuballocItem->size < remainingSize) ?
4853  remainingSize - lastSuballocItem->size : 0;
4854  }
4855  }
4856 
4857  // Check next suballocations for BufferImageGranularity conflicts.
4858  // If conflict exists, we must mark more allocations lost or fail.
4859  if(bufferImageGranularity > 1)
4860  {
4861  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
4862  ++nextSuballocItem;
4863  while(nextSuballocItem != m_Suballocations.cend())
4864  {
4865  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4866  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4867  {
4868  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4869  {
4870  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
4871  if(nextSuballoc.hAllocation->CanBecomeLost() &&
4872  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4873  {
4874  ++*itemsToMakeLostCount;
4875  }
4876  else
4877  {
4878  return false;
4879  }
4880  }
4881  }
4882  else
4883  {
4884  // Already on next page.
4885  break;
4886  }
4887  ++nextSuballocItem;
4888  }
4889  }
4890  }
4891  else
4892  {
4893  const VmaSuballocation& suballoc = *suballocItem;
4894  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4895 
4896  *pSumFreeSize = suballoc.size;
4897 
4898  // Size of this suballocation is too small for this request: Early return.
4899  if(suballoc.size < allocSize)
4900  {
4901  return false;
4902  }
4903 
4904  // Start from offset equal to beginning of this suballocation.
4905  *pOffset = suballoc.offset;
4906 
4907  // Apply VMA_DEBUG_MARGIN at the beginning.
4908  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4909  {
4910  *pOffset += VMA_DEBUG_MARGIN;
4911  }
4912 
4913  // Apply alignment.
4914  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4915  *pOffset = VmaAlignUp(*pOffset, alignment);
4916 
4917  // Check previous suballocations for BufferImageGranularity conflicts.
4918  // Make bigger alignment if necessary.
4919  if(bufferImageGranularity > 1)
4920  {
4921  bool bufferImageGranularityConflict = false;
4922  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4923  while(prevSuballocItem != m_Suballocations.cbegin())
4924  {
4925  --prevSuballocItem;
4926  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4927  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4928  {
4929  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4930  {
4931  bufferImageGranularityConflict = true;
4932  break;
4933  }
4934  }
4935  else
4936  // Already on previous page.
4937  break;
4938  }
4939  if(bufferImageGranularityConflict)
4940  {
4941  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4942  }
4943  }
4944 
4945  // Calculate padding at the beginning based on current offset.
4946  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
4947 
4948  // Calculate required margin at the end if this is not last suballocation.
4949  VmaSuballocationList::const_iterator next = suballocItem;
4950  ++next;
4951  const VkDeviceSize requiredEndMargin =
4952  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4953 
4954  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
4955  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
4956  {
4957  return false;
4958  }
4959 
4960  // Check next suballocations for BufferImageGranularity conflicts.
4961  // If conflict exists, allocation cannot be made here.
4962  if(bufferImageGranularity > 1)
4963  {
4964  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
4965  ++nextSuballocItem;
4966  while(nextSuballocItem != m_Suballocations.cend())
4967  {
4968  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4969  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4970  {
4971  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4972  {
4973  return false;
4974  }
4975  }
4976  else
4977  {
4978  // Already on next page.
4979  break;
4980  }
4981  ++nextSuballocItem;
4982  }
4983  }
4984  }
4985 
4986  // All tests passed: Success. pOffset is already filled.
4987  return true;
4988 }
4989 
4990 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
4991 {
4992  VMA_ASSERT(item != m_Suballocations.end());
4993  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
4994 
4995  VmaSuballocationList::iterator nextItem = item;
4996  ++nextItem;
4997  VMA_ASSERT(nextItem != m_Suballocations.end());
4998  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
4999 
5000  item->size += nextItem->size;
5001  --m_FreeCount;
5002  m_Suballocations.erase(nextItem);
5003 }
5004 
5005 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5006 {
5007  // Change this suballocation to be marked as free.
5008  VmaSuballocation& suballoc = *suballocItem;
5009  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5010  suballoc.hAllocation = VK_NULL_HANDLE;
5011 
5012  // Update totals.
5013  ++m_FreeCount;
5014  m_SumFreeSize += suballoc.size;
5015 
5016  // Merge with previous and/or next suballocation if it's also free.
5017  bool mergeWithNext = false;
5018  bool mergeWithPrev = false;
5019 
5020  VmaSuballocationList::iterator nextItem = suballocItem;
5021  ++nextItem;
5022  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5023  {
5024  mergeWithNext = true;
5025  }
5026 
5027  VmaSuballocationList::iterator prevItem = suballocItem;
5028  if(suballocItem != m_Suballocations.begin())
5029  {
5030  --prevItem;
5031  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5032  {
5033  mergeWithPrev = true;
5034  }
5035  }
5036 
5037  if(mergeWithNext)
5038  {
5039  UnregisterFreeSuballocation(nextItem);
5040  MergeFreeWithNext(suballocItem);
5041  }
5042 
5043  if(mergeWithPrev)
5044  {
5045  UnregisterFreeSuballocation(prevItem);
5046  MergeFreeWithNext(prevItem);
5047  RegisterFreeSuballocation(prevItem);
5048  return prevItem;
5049  }
5050  else
5051  {
5052  RegisterFreeSuballocation(suballocItem);
5053  return suballocItem;
5054  }
5055 }
5056 
5057 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5058 {
5059  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5060  VMA_ASSERT(item->size > 0);
5061 
5062  // You may want to enable this validation at the beginning or at the end of
5063  // this function, depending on what do you want to check.
5064  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5065 
5066  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5067  {
5068  if(m_FreeSuballocationsBySize.empty())
5069  {
5070  m_FreeSuballocationsBySize.push_back(item);
5071  }
5072  else
5073  {
5074  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5075  }
5076  }
5077 
5078  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5079 }
5080 
5081 
5082 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5083 {
5084  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5085  VMA_ASSERT(item->size > 0);
5086 
5087  // You may want to enable this validation at the beginning or at the end of
5088  // this function, depending on what do you want to check.
5089  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5090 
5091  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5092  {
5093  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5094  m_FreeSuballocationsBySize.data(),
5095  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5096  item,
5097  VmaSuballocationItemSizeLess());
5098  for(size_t index = it - m_FreeSuballocationsBySize.data();
5099  index < m_FreeSuballocationsBySize.size();
5100  ++index)
5101  {
5102  if(m_FreeSuballocationsBySize[index] == item)
5103  {
5104  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5105  return;
5106  }
5107  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5108  }
5109  VMA_ASSERT(0 && "Not found.");
5110  }
5111 
5112  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5113 }
5114 
5116 // class VmaDeviceMemoryBlock
5117 
5118 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5119  m_MemoryTypeIndex(UINT32_MAX),
5120  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
5121  m_hMemory(VK_NULL_HANDLE),
5122  m_Size(0),
5123  m_PersistentMap(false),
5124  m_pMappedData(VMA_NULL),
5125  m_Metadata(hAllocator)
5126 {
5127 }
5128 
5129 void VmaDeviceMemoryBlock::Init(
5130  uint32_t newMemoryTypeIndex,
5131  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
5132  VkDeviceMemory newMemory,
5133  VkDeviceSize newSize,
5134  bool persistentMap,
5135  void* pMappedData)
5136 {
5137  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5138 
5139  m_MemoryTypeIndex = newMemoryTypeIndex;
5140  m_BlockVectorType = newBlockVectorType;
5141  m_hMemory = newMemory;
5142  m_Size = newSize;
5143  m_PersistentMap = persistentMap;
5144  m_pMappedData = pMappedData;
5145 
5146  m_Metadata.Init(newSize);
5147 }
5148 
5149 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5150 {
5151  // This is the most important assert in the entire library.
5152  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5153  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5154 
5155  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5156  if(m_pMappedData != VMA_NULL)
5157  {
5158  (allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, m_hMemory);
5159  m_pMappedData = VMA_NULL;
5160  }
5161 
5162  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Size, m_hMemory);
5163  m_hMemory = VK_NULL_HANDLE;
5164 }
5165 
5166 bool VmaDeviceMemoryBlock::Validate() const
5167 {
5168  if((m_hMemory == VK_NULL_HANDLE) ||
5169  (m_Size == 0))
5170  {
5171  return false;
5172  }
5173 
5174  return m_Metadata.Validate();
5175 }
5176 
5177 static void InitStatInfo(VmaStatInfo& outInfo)
5178 {
5179  memset(&outInfo, 0, sizeof(outInfo));
5180  outInfo.allocationSizeMin = UINT64_MAX;
5181  outInfo.unusedRangeSizeMin = UINT64_MAX;
5182 }
5183 
5184 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5185 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5186 {
5187  inoutInfo.blockCount += srcInfo.blockCount;
5188  inoutInfo.allocationCount += srcInfo.allocationCount;
5189  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
5190  inoutInfo.usedBytes += srcInfo.usedBytes;
5191  inoutInfo.unusedBytes += srcInfo.unusedBytes;
5192  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
5193  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
5194  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
5195  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
5196 }
5197 
5198 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
5199 {
5200  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
5201  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
5202  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
5203  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
5204 }
5205 
5206 VmaPool_T::VmaPool_T(
5207  VmaAllocator hAllocator,
5208  const VmaPoolCreateInfo& createInfo) :
5209  m_BlockVector(
5210  hAllocator,
5211  createInfo.memoryTypeIndex,
5212  (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ?
5213  VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
5214  createInfo.blockSize,
5215  createInfo.minBlockCount,
5216  createInfo.maxBlockCount,
5217  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
5218  createInfo.frameInUseCount,
5219  true) // isCustomPool
5220 {
5221 }
5222 
5223 VmaPool_T::~VmaPool_T()
5224 {
5225 }
5226 
5227 #if VMA_STATS_STRING_ENABLED
5228 
5229 #endif // #if VMA_STATS_STRING_ENABLED
5230 
5231 VmaBlockVector::VmaBlockVector(
5232  VmaAllocator hAllocator,
5233  uint32_t memoryTypeIndex,
5234  VMA_BLOCK_VECTOR_TYPE blockVectorType,
5235  VkDeviceSize preferredBlockSize,
5236  size_t minBlockCount,
5237  size_t maxBlockCount,
5238  VkDeviceSize bufferImageGranularity,
5239  uint32_t frameInUseCount,
5240  bool isCustomPool) :
5241  m_hAllocator(hAllocator),
5242  m_MemoryTypeIndex(memoryTypeIndex),
5243  m_BlockVectorType(blockVectorType),
5244  m_PreferredBlockSize(preferredBlockSize),
5245  m_MinBlockCount(minBlockCount),
5246  m_MaxBlockCount(maxBlockCount),
5247  m_BufferImageGranularity(bufferImageGranularity),
5248  m_FrameInUseCount(frameInUseCount),
5249  m_IsCustomPool(isCustomPool),
5250  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
5251  m_HasEmptyBlock(false),
5252  m_pDefragmentator(VMA_NULL)
5253 {
5254 }
5255 
5256 VmaBlockVector::~VmaBlockVector()
5257 {
5258  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
5259 
5260  for(size_t i = m_Blocks.size(); i--; )
5261  {
5262  m_Blocks[i]->Destroy(m_hAllocator);
5263  vma_delete(m_hAllocator, m_Blocks[i]);
5264  }
5265 }
5266 
5267 VkResult VmaBlockVector::CreateMinBlocks()
5268 {
5269  for(size_t i = 0; i < m_MinBlockCount; ++i)
5270  {
5271  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
5272  if(res != VK_SUCCESS)
5273  {
5274  return res;
5275  }
5276  }
5277  return VK_SUCCESS;
5278 }
5279 
5280 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
5281 {
5282  pStats->size = 0;
5283  pStats->unusedSize = 0;
5284  pStats->allocationCount = 0;
5285  pStats->unusedRangeCount = 0;
5286  pStats->unusedRangeSizeMax = 0;
5287 
5288  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5289 
5290  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5291  {
5292  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5293  VMA_ASSERT(pBlock);
5294  VMA_HEAVY_ASSERT(pBlock->Validate());
5295  pBlock->m_Metadata.AddPoolStats(*pStats);
5296  }
5297 }
5298 
5299 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
5300 
5301 VkResult VmaBlockVector::Allocate(
5302  VmaPool hCurrentPool,
5303  uint32_t currentFrameIndex,
5304  const VkMemoryRequirements& vkMemReq,
5305  const VmaAllocationCreateInfo& createInfo,
5306  VmaSuballocationType suballocType,
5307  VmaAllocation* pAllocation)
5308 {
5309  // Validate flags.
5310  if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) !=
5311  (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED))
5312  {
5313  VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT.");
5314  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5315  }
5316 
5317  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5318 
5319  // 1. Search existing allocations. Try to allocate without making other allocations lost.
5320  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5321  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5322  {
5323  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5324  VMA_ASSERT(pCurrBlock);
5325  VmaAllocationRequest currRequest = {};
5326  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5327  currentFrameIndex,
5328  m_FrameInUseCount,
5329  m_BufferImageGranularity,
5330  vkMemReq.size,
5331  vkMemReq.alignment,
5332  suballocType,
5333  false, // canMakeOtherLost
5334  &currRequest))
5335  {
5336  // Allocate from pCurrBlock.
5337  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
5338 
5339  // We no longer have an empty Allocation.
5340  if(pCurrBlock->m_Metadata.IsEmpty())
5341  {
5342  m_HasEmptyBlock = false;
5343  }
5344 
5345  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5346  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
5347  (*pAllocation)->InitBlockAllocation(
5348  hCurrentPool,
5349  pCurrBlock,
5350  currRequest.offset,
5351  vkMemReq.alignment,
5352  vkMemReq.size,
5353  suballocType,
5354  createInfo.pUserData,
5355  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5356  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
5357  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5358  return VK_SUCCESS;
5359  }
5360  }
5361 
5362  const bool canCreateNewBlock =
5363  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
5364  (m_Blocks.size() < m_MaxBlockCount);
5365 
5366  // 2. Try to create new block.
5367  if(canCreateNewBlock)
5368  {
5369  // 2.1. Start with full preferredBlockSize.
5370  VkDeviceSize blockSize = m_PreferredBlockSize;
5371  size_t newBlockIndex = 0;
5372  VkResult res = CreateBlock(blockSize, &newBlockIndex);
5373  // Allocating blocks of other sizes is allowed only in default pools.
5374  // In custom pools block size is fixed.
5375  if(res < 0 && m_IsCustomPool == false)
5376  {
5377  // 2.2. Try half the size.
5378  blockSize /= 2;
5379  if(blockSize >= vkMemReq.size)
5380  {
5381  res = CreateBlock(blockSize, &newBlockIndex);
5382  if(res < 0)
5383  {
5384  // 2.3. Try quarter the size.
5385  blockSize /= 2;
5386  if(blockSize >= vkMemReq.size)
5387  {
5388  res = CreateBlock(blockSize, &newBlockIndex);
5389  }
5390  }
5391  }
5392  }
5393  if(res == VK_SUCCESS)
5394  {
5395  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
5396  VMA_ASSERT(pBlock->m_Size >= vkMemReq.size);
5397 
5398  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
5399  VmaAllocationRequest allocRequest;
5400  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
5401  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5402  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
5403  (*pAllocation)->InitBlockAllocation(
5404  hCurrentPool,
5405  pBlock,
5406  allocRequest.offset,
5407  vkMemReq.alignment,
5408  vkMemReq.size,
5409  suballocType,
5410  createInfo.pUserData,
5411  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5412  VMA_HEAVY_ASSERT(pBlock->Validate());
5413  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
5414 
5415  return VK_SUCCESS;
5416  }
5417  }
5418 
5419  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
5420 
5421  // 3. Try to allocate from existing blocks with making other allocations lost.
5422  if(canMakeOtherLost)
5423  {
5424  uint32_t tryIndex = 0;
5425  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
5426  {
5427  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
5428  VmaAllocationRequest bestRequest = {};
5429  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
5430 
5431  // 1. Search existing allocations.
5432  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5433  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5434  {
5435  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5436  VMA_ASSERT(pCurrBlock);
5437  VmaAllocationRequest currRequest = {};
5438  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5439  currentFrameIndex,
5440  m_FrameInUseCount,
5441  m_BufferImageGranularity,
5442  vkMemReq.size,
5443  vkMemReq.alignment,
5444  suballocType,
5445  canMakeOtherLost,
5446  &currRequest))
5447  {
5448  const VkDeviceSize currRequestCost = currRequest.CalcCost();
5449  if(pBestRequestBlock == VMA_NULL ||
5450  currRequestCost < bestRequestCost)
5451  {
5452  pBestRequestBlock = pCurrBlock;
5453  bestRequest = currRequest;
5454  bestRequestCost = currRequestCost;
5455 
5456  if(bestRequestCost == 0)
5457  {
5458  break;
5459  }
5460  }
5461  }
5462  }
5463 
5464  if(pBestRequestBlock != VMA_NULL)
5465  {
5466  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
5467  currentFrameIndex,
5468  m_FrameInUseCount,
5469  &bestRequest))
5470  {
5471  // We no longer have an empty Allocation.
5472  if(pBestRequestBlock->m_Metadata.IsEmpty())
5473  {
5474  m_HasEmptyBlock = false;
5475  }
5476  // Allocate from this pBlock.
5477  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5478  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
5479  (*pAllocation)->InitBlockAllocation(
5480  hCurrentPool,
5481  pBestRequestBlock,
5482  bestRequest.offset,
5483  vkMemReq.alignment,
5484  vkMemReq.size,
5485  suballocType,
5486  createInfo.pUserData,
5487  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5488  VMA_HEAVY_ASSERT(pBlock->Validate());
5489  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5490  return VK_SUCCESS;
5491  }
5492  // else: Some allocations must have been touched while we are here. Next try.
5493  }
5494  else
5495  {
5496  // Could not find place in any of the blocks - break outer loop.
5497  break;
5498  }
5499  }
5500  /* Maximum number of tries exceeded - a very unlike event when many other
5501  threads are simultaneously touching allocations making it impossible to make
5502  lost at the same time as we try to allocate. */
5503  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
5504  {
5505  return VK_ERROR_TOO_MANY_OBJECTS;
5506  }
5507  }
5508 
5509  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5510 }
5511 
5512 void VmaBlockVector::Free(
5513  VmaAllocation hAllocation)
5514 {
5515  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
5516 
5517  // Scope for lock.
5518  {
5519  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5520 
5521  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
5522 
5523  pBlock->m_Metadata.Free(hAllocation);
5524  VMA_HEAVY_ASSERT(pBlock->Validate());
5525 
5526  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
5527 
5528  // pBlock became empty after this deallocation.
5529  if(pBlock->m_Metadata.IsEmpty())
5530  {
5531  // Already has empty Allocation. We don't want to have two, so delete this one.
5532  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
5533  {
5534  pBlockToDelete = pBlock;
5535  Remove(pBlock);
5536  }
5537  // We now have first empty Allocation.
5538  else
5539  {
5540  m_HasEmptyBlock = true;
5541  }
5542  }
5543  // Must be called after srcBlockIndex is used, because later it may become invalid!
5544  IncrementallySortBlocks();
5545  }
5546 
5547  // Destruction of a free Allocation. Deferred until this point, outside of mutex
5548  // lock, for performance reason.
5549  if(pBlockToDelete != VMA_NULL)
5550  {
5551  VMA_DEBUG_LOG(" Deleted empty allocation");
5552  pBlockToDelete->Destroy(m_hAllocator);
5553  vma_delete(m_hAllocator, pBlockToDelete);
5554  }
5555 }
5556 
5557 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
5558 {
5559  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5560  {
5561  if(m_Blocks[blockIndex] == pBlock)
5562  {
5563  VmaVectorRemove(m_Blocks, blockIndex);
5564  return;
5565  }
5566  }
5567  VMA_ASSERT(0);
5568 }
5569 
5570 void VmaBlockVector::IncrementallySortBlocks()
5571 {
5572  // Bubble sort only until first swap.
5573  for(size_t i = 1; i < m_Blocks.size(); ++i)
5574  {
5575  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
5576  {
5577  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
5578  return;
5579  }
5580  }
5581 }
5582 
5583 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
5584 {
5585  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
5586  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
5587  allocInfo.allocationSize = blockSize;
5588  VkDeviceMemory mem = VK_NULL_HANDLE;
5589  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
5590  if(res < 0)
5591  {
5592  return res;
5593  }
5594 
5595  // New VkDeviceMemory successfully created.
5596 
5597  // Map memory if needed.
5598  void* pMappedData = VMA_NULL;
5599  const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED);
5600  if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0)
5601  {
5602  res = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5603  m_hAllocator->m_hDevice,
5604  mem,
5605  0,
5606  VK_WHOLE_SIZE,
5607  0,
5608  &pMappedData);
5609  if(res < 0)
5610  {
5611  VMA_DEBUG_LOG(" vkMapMemory FAILED");
5612  m_hAllocator->FreeVulkanMemory(m_MemoryTypeIndex, blockSize, mem);
5613  return res;
5614  }
5615  }
5616 
5617  // Create new Allocation for it.
5618  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
5619  pBlock->Init(
5620  m_MemoryTypeIndex,
5621  (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType,
5622  mem,
5623  allocInfo.allocationSize,
5624  persistentMap,
5625  pMappedData);
5626 
5627  m_Blocks.push_back(pBlock);
5628  if(pNewBlockIndex != VMA_NULL)
5629  {
5630  *pNewBlockIndex = m_Blocks.size() - 1;
5631  }
5632 
5633  return VK_SUCCESS;
5634 }
5635 
5636 #if VMA_STATS_STRING_ENABLED
5637 
5638 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
5639 {
5640  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5641 
5642  json.BeginObject();
5643 
5644  if(m_IsCustomPool)
5645  {
5646  json.WriteString("MemoryTypeIndex");
5647  json.WriteNumber(m_MemoryTypeIndex);
5648 
5649  if(m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
5650  {
5651  json.WriteString("Mapped");
5652  json.WriteBool(true);
5653  }
5654 
5655  json.WriteString("BlockSize");
5656  json.WriteNumber(m_PreferredBlockSize);
5657 
5658  json.WriteString("BlockCount");
5659  json.BeginObject(true);
5660  if(m_MinBlockCount > 0)
5661  {
5662  json.WriteString("Min");
5663  json.WriteNumber(m_MinBlockCount);
5664  }
5665  if(m_MaxBlockCount < SIZE_MAX)
5666  {
5667  json.WriteString("Max");
5668  json.WriteNumber(m_MaxBlockCount);
5669  }
5670  json.WriteString("Cur");
5671  json.WriteNumber(m_Blocks.size());
5672  json.EndObject();
5673 
5674  if(m_FrameInUseCount > 0)
5675  {
5676  json.WriteString("FrameInUseCount");
5677  json.WriteNumber(m_FrameInUseCount);
5678  }
5679  }
5680  else
5681  {
5682  json.WriteString("PreferredBlockSize");
5683  json.WriteNumber(m_PreferredBlockSize);
5684  }
5685 
5686  json.WriteString("Blocks");
5687  json.BeginArray();
5688  for(size_t i = 0; i < m_Blocks.size(); ++i)
5689  {
5690  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
5691  }
5692  json.EndArray();
5693 
5694  json.EndObject();
5695 }
5696 
5697 #endif // #if VMA_STATS_STRING_ENABLED
5698 
5699 void VmaBlockVector::UnmapPersistentlyMappedMemory()
5700 {
5701  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5702 
5703  for(size_t i = m_Blocks.size(); i--; )
5704  {
5705  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5706  if(pBlock->m_pMappedData != VMA_NULL)
5707  {
5708  VMA_ASSERT(pBlock->m_PersistentMap != false);
5709  (m_hAllocator->GetVulkanFunctions().vkUnmapMemory)(m_hAllocator->m_hDevice, pBlock->m_hMemory);
5710  pBlock->m_pMappedData = VMA_NULL;
5711  }
5712  }
5713 }
5714 
5715 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
5716 {
5717  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5718 
5719  VkResult finalResult = VK_SUCCESS;
5720  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
5721  {
5722  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5723  if(pBlock->m_PersistentMap)
5724  {
5725  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
5726  VkResult localResult = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5727  m_hAllocator->m_hDevice,
5728  pBlock->m_hMemory,
5729  0,
5730  VK_WHOLE_SIZE,
5731  0,
5732  &pBlock->m_pMappedData);
5733  if(localResult != VK_SUCCESS)
5734  {
5735  finalResult = localResult;
5736  }
5737  }
5738  }
5739  return finalResult;
5740 }
5741 
5742 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
5743  VmaAllocator hAllocator,
5744  uint32_t currentFrameIndex)
5745 {
5746  if(m_pDefragmentator == VMA_NULL)
5747  {
5748  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
5749  hAllocator,
5750  this,
5751  currentFrameIndex);
5752  }
5753 
5754  return m_pDefragmentator;
5755 }
5756 
5757 VkResult VmaBlockVector::Defragment(
5758  VmaDefragmentationStats* pDefragmentationStats,
5759  VkDeviceSize& maxBytesToMove,
5760  uint32_t& maxAllocationsToMove)
5761 {
5762  if(m_pDefragmentator == VMA_NULL)
5763  {
5764  return VK_SUCCESS;
5765  }
5766 
5767  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5768 
5769  // Defragment.
5770  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
5771 
5772  // Accumulate statistics.
5773  if(pDefragmentationStats != VMA_NULL)
5774  {
5775  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
5776  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
5777  pDefragmentationStats->bytesMoved += bytesMoved;
5778  pDefragmentationStats->allocationsMoved += allocationsMoved;
5779  VMA_ASSERT(bytesMoved <= maxBytesToMove);
5780  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
5781  maxBytesToMove -= bytesMoved;
5782  maxAllocationsToMove -= allocationsMoved;
5783  }
5784 
5785  // Free empty blocks.
5786  m_HasEmptyBlock = false;
5787  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
5788  {
5789  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
5790  if(pBlock->m_Metadata.IsEmpty())
5791  {
5792  if(m_Blocks.size() > m_MinBlockCount)
5793  {
5794  if(pDefragmentationStats != VMA_NULL)
5795  {
5796  ++pDefragmentationStats->deviceMemoryBlocksFreed;
5797  pDefragmentationStats->bytesFreed += pBlock->m_Size;
5798  }
5799 
5800  VmaVectorRemove(m_Blocks, blockIndex);
5801  pBlock->Destroy(m_hAllocator);
5802  vma_delete(m_hAllocator, pBlock);
5803  }
5804  else
5805  {
5806  m_HasEmptyBlock = true;
5807  }
5808  }
5809  }
5810 
5811  return result;
5812 }
5813 
5814 void VmaBlockVector::DestroyDefragmentator()
5815 {
5816  if(m_pDefragmentator != VMA_NULL)
5817  {
5818  vma_delete(m_hAllocator, m_pDefragmentator);
5819  m_pDefragmentator = VMA_NULL;
5820  }
5821 }
5822 
5823 void VmaBlockVector::MakePoolAllocationsLost(
5824  uint32_t currentFrameIndex,
5825  size_t* pLostAllocationCount)
5826 {
5827  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5828 
5829  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5830  {
5831  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5832  VMA_ASSERT(pBlock);
5833  pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
5834  }
5835 }
5836 
5837 void VmaBlockVector::AddStats(VmaStats* pStats)
5838 {
5839  const uint32_t memTypeIndex = m_MemoryTypeIndex;
5840  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
5841 
5842  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5843 
5844  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5845  {
5846  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5847  VMA_ASSERT(pBlock);
5848  VMA_HEAVY_ASSERT(pBlock->Validate());
5849  VmaStatInfo allocationStatInfo;
5850  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
5851  VmaAddStatInfo(pStats->total, allocationStatInfo);
5852  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
5853  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
5854  }
5855 }
5856 
5858 // VmaDefragmentator members definition
5859 
5860 VmaDefragmentator::VmaDefragmentator(
5861  VmaAllocator hAllocator,
5862  VmaBlockVector* pBlockVector,
5863  uint32_t currentFrameIndex) :
5864  m_hAllocator(hAllocator),
5865  m_pBlockVector(pBlockVector),
5866  m_CurrentFrameIndex(currentFrameIndex),
5867  m_BytesMoved(0),
5868  m_AllocationsMoved(0),
5869  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
5870  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
5871 {
5872 }
5873 
5874 VmaDefragmentator::~VmaDefragmentator()
5875 {
5876  for(size_t i = m_Blocks.size(); i--; )
5877  {
5878  vma_delete(m_hAllocator, m_Blocks[i]);
5879  }
5880 }
5881 
5882 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
5883 {
5884  AllocationInfo allocInfo;
5885  allocInfo.m_hAllocation = hAlloc;
5886  allocInfo.m_pChanged = pChanged;
5887  m_Allocations.push_back(allocInfo);
5888 }
5889 
5890 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
5891 {
5892  // It has already been mapped for defragmentation.
5893  if(m_pMappedDataForDefragmentation)
5894  {
5895  *ppMappedData = m_pMappedDataForDefragmentation;
5896  return VK_SUCCESS;
5897  }
5898 
5899  // It is persistently mapped.
5900  if(m_pBlock->m_PersistentMap)
5901  {
5902  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
5903  *ppMappedData = m_pBlock->m_pMappedData;
5904  return VK_SUCCESS;
5905  }
5906 
5907  // Map on first usage.
5908  VkResult res = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5909  hAllocator->m_hDevice,
5910  m_pBlock->m_hMemory,
5911  0,
5912  VK_WHOLE_SIZE,
5913  0,
5914  &m_pMappedDataForDefragmentation);
5915  *ppMappedData = m_pMappedDataForDefragmentation;
5916  return res;
5917 }
5918 
5919 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
5920 {
5921  if(m_pMappedDataForDefragmentation != VMA_NULL)
5922  {
5923  (hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_pBlock->m_hMemory);
5924  }
5925 }
5926 
5927 VkResult VmaDefragmentator::DefragmentRound(
5928  VkDeviceSize maxBytesToMove,
5929  uint32_t maxAllocationsToMove)
5930 {
5931  if(m_Blocks.empty())
5932  {
5933  return VK_SUCCESS;
5934  }
5935 
5936  size_t srcBlockIndex = m_Blocks.size() - 1;
5937  size_t srcAllocIndex = SIZE_MAX;
5938  for(;;)
5939  {
5940  // 1. Find next allocation to move.
5941  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
5942  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
5943  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
5944  {
5945  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
5946  {
5947  // Finished: no more allocations to process.
5948  if(srcBlockIndex == 0)
5949  {
5950  return VK_SUCCESS;
5951  }
5952  else
5953  {
5954  --srcBlockIndex;
5955  srcAllocIndex = SIZE_MAX;
5956  }
5957  }
5958  else
5959  {
5960  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
5961  }
5962  }
5963 
5964  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
5965  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
5966 
5967  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
5968  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
5969  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
5970  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
5971 
5972  // 2. Try to find new place for this allocation in preceding or current block.
5973  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
5974  {
5975  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
5976  VmaAllocationRequest dstAllocRequest;
5977  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
5978  m_CurrentFrameIndex,
5979  m_pBlockVector->GetFrameInUseCount(),
5980  m_pBlockVector->GetBufferImageGranularity(),
5981  size,
5982  alignment,
5983  suballocType,
5984  false, // canMakeOtherLost
5985  &dstAllocRequest) &&
5986  MoveMakesSense(
5987  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
5988  {
5989  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
5990 
5991  // Reached limit on number of allocations or bytes to move.
5992  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
5993  (m_BytesMoved + size > maxBytesToMove))
5994  {
5995  return VK_INCOMPLETE;
5996  }
5997 
5998  void* pDstMappedData = VMA_NULL;
5999  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6000  if(res != VK_SUCCESS)
6001  {
6002  return res;
6003  }
6004 
6005  void* pSrcMappedData = VMA_NULL;
6006  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6007  if(res != VK_SUCCESS)
6008  {
6009  return res;
6010  }
6011 
6012  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6013  memcpy(
6014  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6015  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6016  static_cast<size_t>(size));
6017 
6018  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6019  pSrcBlockInfo->m_pBlock->m_Metadata.Free(allocInfo.m_hAllocation);
6020 
6021  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6022 
6023  if(allocInfo.m_pChanged != VMA_NULL)
6024  {
6025  *allocInfo.m_pChanged = VK_TRUE;
6026  }
6027 
6028  ++m_AllocationsMoved;
6029  m_BytesMoved += size;
6030 
6031  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6032 
6033  break;
6034  }
6035  }
6036 
6037  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6038 
6039  if(srcAllocIndex > 0)
6040  {
6041  --srcAllocIndex;
6042  }
6043  else
6044  {
6045  if(srcBlockIndex > 0)
6046  {
6047  --srcBlockIndex;
6048  srcAllocIndex = SIZE_MAX;
6049  }
6050  else
6051  {
6052  return VK_SUCCESS;
6053  }
6054  }
6055  }
6056 }
6057 
6058 VkResult VmaDefragmentator::Defragment(
6059  VkDeviceSize maxBytesToMove,
6060  uint32_t maxAllocationsToMove)
6061 {
6062  if(m_Allocations.empty())
6063  {
6064  return VK_SUCCESS;
6065  }
6066 
6067  // Create block info for each block.
6068  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6069  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6070  {
6071  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6072  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6073  m_Blocks.push_back(pBlockInfo);
6074  }
6075 
6076  // Sort them by m_pBlock pointer value.
6077  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6078 
6079  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6080  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6081  {
6082  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6083  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6084  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6085  {
6086  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6087  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6088  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6089  {
6090  (*it)->m_Allocations.push_back(allocInfo);
6091  }
6092  else
6093  {
6094  VMA_ASSERT(0);
6095  }
6096  }
6097  }
6098  m_Allocations.clear();
6099 
6100  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6101  {
6102  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6103  pBlockInfo->CalcHasNonMovableAllocations();
6104  pBlockInfo->SortAllocationsBySizeDescecnding();
6105  }
6106 
6107  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6108  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6109 
6110  // Execute defragmentation rounds (the main part).
6111  VkResult result = VK_SUCCESS;
6112  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6113  {
6114  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6115  }
6116 
6117  // Unmap blocks that were mapped for defragmentation.
6118  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6119  {
6120  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6121  }
6122 
6123  return result;
6124 }
6125 
6126 bool VmaDefragmentator::MoveMakesSense(
6127  size_t dstBlockIndex, VkDeviceSize dstOffset,
6128  size_t srcBlockIndex, VkDeviceSize srcOffset)
6129 {
6130  if(dstBlockIndex < srcBlockIndex)
6131  {
6132  return true;
6133  }
6134  if(dstBlockIndex > srcBlockIndex)
6135  {
6136  return false;
6137  }
6138  if(dstOffset < srcOffset)
6139  {
6140  return true;
6141  }
6142  return false;
6143 }
6144 
6146 // VmaAllocator_T
6147 
6148 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6149  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6150  m_PhysicalDevice(pCreateInfo->physicalDevice),
6151  m_hDevice(pCreateInfo->device),
6152  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6153  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6154  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6155  m_UnmapPersistentlyMappedMemoryCounter(0),
6156  m_PreferredLargeHeapBlockSize(0),
6157  m_PreferredSmallHeapBlockSize(0),
6158  m_CurrentFrameIndex(0),
6159  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6160 {
6161  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6162 
6163  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6164  memset(&m_MemProps, 0, sizeof(m_MemProps));
6165  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6166 
6167  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6168  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
6169 
6170  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6171  {
6172  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6173  }
6174 
6175  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6176  {
6177  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6178  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6179  }
6180 
6181  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6182 
6183  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6184  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6185 
6186  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6187  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6188  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
6189  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
6190 
6191  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
6192  {
6193  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
6194  {
6195  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
6196  if(limit != VK_WHOLE_SIZE)
6197  {
6198  m_HeapSizeLimit[heapIndex] = limit;
6199  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
6200  {
6201  m_MemProps.memoryHeaps[heapIndex].size = limit;
6202  }
6203  }
6204  }
6205  }
6206 
6207  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6208  {
6209  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
6210 
6211  for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex)
6212  {
6213  m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)(
6214  this,
6215  memTypeIndex,
6216  static_cast<VMA_BLOCK_VECTOR_TYPE>(blockVectorTypeIndex),
6217  preferredBlockSize,
6218  0,
6219  SIZE_MAX,
6220  GetBufferImageGranularity(),
6221  pCreateInfo->frameInUseCount,
6222  false); // isCustomPool
6223  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
6224  // becase minBlockCount is 0.
6225  m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
6226  }
6227  }
6228 }
6229 
6230 VmaAllocator_T::~VmaAllocator_T()
6231 {
6232  VMA_ASSERT(m_Pools.empty());
6233 
6234  for(size_t i = GetMemoryTypeCount(); i--; )
6235  {
6236  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
6237  {
6238  vma_delete(this, m_pOwnAllocations[i][j]);
6239  vma_delete(this, m_pBlockVectors[i][j]);
6240  }
6241  }
6242 }
6243 
6244 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
6245 {
6246 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6247  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
6248  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
6249  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
6250  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
6251  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
6252  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
6253  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
6254  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
6255  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
6256  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
6257  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
6258  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
6259  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
6260  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
6261 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6262 
6263  if(pVulkanFunctions != VMA_NULL)
6264  {
6265  m_VulkanFunctions = *pVulkanFunctions;
6266  }
6267 
6268  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
6269  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
6270  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
6271  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
6272  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
6273  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
6274  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
6275  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
6276  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
6277  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
6278  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
6279  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
6280  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
6281  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
6282  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
6283  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
6284 }
6285 
6286 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
6287 {
6288  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6289  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
6290  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
6291  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
6292 }
6293 
6294 VkResult VmaAllocator_T::AllocateMemoryOfType(
6295  const VkMemoryRequirements& vkMemReq,
6296  const VmaAllocationCreateInfo& createInfo,
6297  uint32_t memTypeIndex,
6298  VmaSuballocationType suballocType,
6299  VmaAllocation* pAllocation)
6300 {
6301  VMA_ASSERT(pAllocation != VMA_NULL);
6302  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
6303 
6304  uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags);
6305  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6306  VMA_ASSERT(blockVector);
6307 
6308  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
6309  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
6310  const bool ownMemory =
6311  (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 ||
6312  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
6313  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
6314  vkMemReq.size > preferredBlockSize / 2);
6315 
6316  if(ownMemory)
6317  {
6318  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6319  {
6320  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6321  }
6322  else
6323  {
6324  return AllocateOwnMemory(
6325  vkMemReq.size,
6326  suballocType,
6327  memTypeIndex,
6328  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6329  createInfo.pUserData,
6330  pAllocation);
6331  }
6332  }
6333  else
6334  {
6335  VkResult res = blockVector->Allocate(
6336  VK_NULL_HANDLE, // hCurrentPool
6337  m_CurrentFrameIndex.load(),
6338  vkMemReq,
6339  createInfo,
6340  suballocType,
6341  pAllocation);
6342  if(res == VK_SUCCESS)
6343  {
6344  return res;
6345  }
6346 
6347  // 5. Try own memory.
6348  res = AllocateOwnMemory(
6349  vkMemReq.size,
6350  suballocType,
6351  memTypeIndex,
6352  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6353  createInfo.pUserData,
6354  pAllocation);
6355  if(res == VK_SUCCESS)
6356  {
6357  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
6358  VMA_DEBUG_LOG(" Allocated as OwnMemory");
6359  return VK_SUCCESS;
6360  }
6361  else
6362  {
6363  // Everything failed: Return error code.
6364  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6365  return res;
6366  }
6367  }
6368 }
6369 
6370 VkResult VmaAllocator_T::AllocateOwnMemory(
6371  VkDeviceSize size,
6372  VmaSuballocationType suballocType,
6373  uint32_t memTypeIndex,
6374  bool map,
6375  void* pUserData,
6376  VmaAllocation* pAllocation)
6377 {
6378  VMA_ASSERT(pAllocation);
6379 
6380  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6381  allocInfo.memoryTypeIndex = memTypeIndex;
6382  allocInfo.allocationSize = size;
6383 
6384  // Allocate VkDeviceMemory.
6385  VkDeviceMemory hMemory = VK_NULL_HANDLE;
6386  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
6387  if(res < 0)
6388  {
6389  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6390  return res;
6391  }
6392 
6393  void* pMappedData = nullptr;
6394  if(map)
6395  {
6396  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
6397  {
6398  res = vkMapMemory(m_hDevice, hMemory, 0, VK_WHOLE_SIZE, 0, &pMappedData);
6399  if(res < 0)
6400  {
6401  VMA_DEBUG_LOG(" vkMapMemory FAILED");
6402  FreeVulkanMemory(memTypeIndex, size, hMemory);
6403  return res;
6404  }
6405  }
6406  }
6407 
6408  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load());
6409  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
6410 
6411  // Register it in m_pOwnAllocations.
6412  {
6413  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6414  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
6415  VMA_ASSERT(pOwnAllocations);
6416  VmaVectorInsertSorted<VmaPointerLess>(*pOwnAllocations, *pAllocation);
6417  }
6418 
6419  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
6420 
6421  return VK_SUCCESS;
6422 }
6423 
6424 VkResult VmaAllocator_T::AllocateMemory(
6425  const VkMemoryRequirements& vkMemReq,
6426  const VmaAllocationCreateInfo& createInfo,
6427  VmaSuballocationType suballocType,
6428  VmaAllocation* pAllocation)
6429 {
6430  if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 &&
6431  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6432  {
6433  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
6434  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6435  }
6436  if((createInfo.pool != VK_NULL_HANDLE) &&
6437  ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0))
6438  {
6439  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid.");
6440  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6441  }
6442 
6443  if(createInfo.pool != VK_NULL_HANDLE)
6444  {
6445  return createInfo.pool->m_BlockVector.Allocate(
6446  createInfo.pool,
6447  m_CurrentFrameIndex.load(),
6448  vkMemReq,
6449  createInfo,
6450  suballocType,
6451  pAllocation);
6452  }
6453  else
6454  {
6455  // Bit mask of memory Vulkan types acceptable for this allocation.
6456  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
6457  uint32_t memTypeIndex = UINT32_MAX;
6458  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6459  if(res == VK_SUCCESS)
6460  {
6461  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6462  // Succeeded on first try.
6463  if(res == VK_SUCCESS)
6464  {
6465  return res;
6466  }
6467  // Allocation from this memory type failed. Try other compatible memory types.
6468  else
6469  {
6470  for(;;)
6471  {
6472  // Remove old memTypeIndex from list of possibilities.
6473  memoryTypeBits &= ~(1u << memTypeIndex);
6474  // Find alternative memTypeIndex.
6475  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6476  if(res == VK_SUCCESS)
6477  {
6478  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6479  // Allocation from this alternative memory type succeeded.
6480  if(res == VK_SUCCESS)
6481  {
6482  return res;
6483  }
6484  // else: Allocation from this memory type failed. Try next one - next loop iteration.
6485  }
6486  // No other matching memory type index could be found.
6487  else
6488  {
6489  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
6490  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6491  }
6492  }
6493  }
6494  }
6495  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
6496  else
6497  return res;
6498  }
6499 }
6500 
6501 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
6502 {
6503  VMA_ASSERT(allocation);
6504 
6505  if(allocation->CanBecomeLost() == false ||
6506  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6507  {
6508  switch(allocation->GetType())
6509  {
6510  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
6511  {
6512  VmaBlockVector* pBlockVector = VMA_NULL;
6513  VmaPool hPool = allocation->GetPool();
6514  if(hPool != VK_NULL_HANDLE)
6515  {
6516  pBlockVector = &hPool->m_BlockVector;
6517  }
6518  else
6519  {
6520  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
6521  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
6522  pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6523  }
6524  pBlockVector->Free(allocation);
6525  }
6526  break;
6527  case VmaAllocation_T::ALLOCATION_TYPE_OWN:
6528  FreeOwnMemory(allocation);
6529  break;
6530  default:
6531  VMA_ASSERT(0);
6532  }
6533  }
6534 
6535  vma_delete(this, allocation);
6536 }
6537 
6538 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
6539 {
6540  // Initialize.
6541  InitStatInfo(pStats->total);
6542  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
6543  InitStatInfo(pStats->memoryType[i]);
6544  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6545  InitStatInfo(pStats->memoryHeap[i]);
6546 
6547  // Process default pools.
6548  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6549  {
6550  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6551  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6552  {
6553  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6554  VMA_ASSERT(pBlockVector);
6555  pBlockVector->AddStats(pStats);
6556  }
6557  }
6558 
6559  // Process custom pools.
6560  {
6561  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6562  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6563  {
6564  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
6565  }
6566  }
6567 
6568  // Process own allocations.
6569  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6570  {
6571  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6572  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6573  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6574  {
6575  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
6576  VMA_ASSERT(pOwnAllocVector);
6577  for(size_t allocIndex = 0, allocCount = pOwnAllocVector->size(); allocIndex < allocCount; ++allocIndex)
6578  {
6579  VmaStatInfo allocationStatInfo;
6580  (*pOwnAllocVector)[allocIndex]->OwnAllocCalcStatsInfo(allocationStatInfo);
6581  VmaAddStatInfo(pStats->total, allocationStatInfo);
6582  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6583  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6584  }
6585  }
6586  }
6587 
6588  // Postprocess.
6589  VmaPostprocessCalcStatInfo(pStats->total);
6590  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
6591  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
6592  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
6593  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
6594 }
6595 
6596 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
6597 
6598 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
6599 {
6600  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
6601  {
6602  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6603  {
6604  for(uint32_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
6605  {
6606  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6607  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6608  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6609  {
6610  // Process OwnAllocations.
6611  {
6612  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6613  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6614  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
6615  {
6616  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
6617  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(this);
6618  }
6619  }
6620 
6621  // Process normal Allocations.
6622  {
6623  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6624  pBlockVector->UnmapPersistentlyMappedMemory();
6625  }
6626  }
6627  }
6628 
6629  // Process custom pools.
6630  {
6631  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6632  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6633  {
6634  m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory();
6635  }
6636  }
6637  }
6638  }
6639 }
6640 
6641 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
6642 {
6643  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
6644  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
6645  {
6646  VkResult finalResult = VK_SUCCESS;
6647  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6648  {
6649  // Process custom pools.
6650  {
6651  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6652  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6653  {
6654  m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory();
6655  }
6656  }
6657 
6658  for(uint32_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
6659  {
6660  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6661  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6662  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6663  {
6664  // Process OwnAllocations.
6665  {
6666  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6667  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6668  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
6669  {
6670  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
6671  hAlloc->OwnAllocMapPersistentlyMappedMemory(this);
6672  }
6673  }
6674 
6675  // Process normal Allocations.
6676  {
6677  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6678  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
6679  if(localResult != VK_SUCCESS)
6680  {
6681  finalResult = localResult;
6682  }
6683  }
6684  }
6685  }
6686  }
6687  return finalResult;
6688  }
6689  else
6690  return VK_SUCCESS;
6691 }
6692 
6693 VkResult VmaAllocator_T::Defragment(
6694  VmaAllocation* pAllocations,
6695  size_t allocationCount,
6696  VkBool32* pAllocationsChanged,
6697  const VmaDefragmentationInfo* pDefragmentationInfo,
6698  VmaDefragmentationStats* pDefragmentationStats)
6699 {
6700  if(pAllocationsChanged != VMA_NULL)
6701  {
6702  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
6703  }
6704  if(pDefragmentationStats != VMA_NULL)
6705  {
6706  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
6707  }
6708 
6709  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
6710  {
6711  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
6712  return VK_ERROR_MEMORY_MAP_FAILED;
6713  }
6714 
6715  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
6716 
6717  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
6718 
6719  const size_t poolCount = m_Pools.size();
6720 
6721  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
6722  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
6723  {
6724  VmaAllocation hAlloc = pAllocations[allocIndex];
6725  VMA_ASSERT(hAlloc);
6726  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
6727  // OwnAlloc cannot be defragmented.
6728  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
6729  // Only HOST_VISIBLE memory types can be defragmented.
6730  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
6731  // Lost allocation cannot be defragmented.
6732  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
6733  {
6734  VmaBlockVector* pAllocBlockVector = nullptr;
6735 
6736  const VmaPool hAllocPool = hAlloc->GetPool();
6737  // This allocation belongs to custom pool.
6738  if(hAllocPool != VK_NULL_HANDLE)
6739  {
6740  pAllocBlockVector = &hAllocPool->GetBlockVector();
6741  }
6742  // This allocation belongs to general pool.
6743  else
6744  {
6745  pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()];
6746  }
6747 
6748  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
6749 
6750  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
6751  &pAllocationsChanged[allocIndex] : VMA_NULL;
6752  pDefragmentator->AddAllocation(hAlloc, pChanged);
6753  }
6754  }
6755 
6756  VkResult result = VK_SUCCESS;
6757 
6758  // ======== Main processing.
6759 
6760  VkDeviceSize maxBytesToMove = SIZE_MAX;
6761  uint32_t maxAllocationsToMove = UINT32_MAX;
6762  if(pDefragmentationInfo != VMA_NULL)
6763  {
6764  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
6765  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
6766  }
6767 
6768  // Process standard memory.
6769  for(uint32_t memTypeIndex = 0;
6770  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
6771  ++memTypeIndex)
6772  {
6773  // Only HOST_VISIBLE memory types can be defragmented.
6774  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6775  {
6776  for(uint32_t blockVectorType = 0;
6777  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
6778  ++blockVectorType)
6779  {
6780  result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment(
6781  pDefragmentationStats,
6782  maxBytesToMove,
6783  maxAllocationsToMove);
6784  }
6785  }
6786  }
6787 
6788  // Process custom pools.
6789  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
6790  {
6791  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
6792  pDefragmentationStats,
6793  maxBytesToMove,
6794  maxAllocationsToMove);
6795  }
6796 
6797  // ======== Destroy defragmentators.
6798 
6799  // Process custom pools.
6800  for(size_t poolIndex = poolCount; poolIndex--; )
6801  {
6802  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
6803  }
6804 
6805  // Process standard memory.
6806  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
6807  {
6808  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6809  {
6810  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
6811  {
6812  m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator();
6813  }
6814  }
6815  }
6816 
6817  return result;
6818 }
6819 
6820 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
6821 {
6822  if(hAllocation->CanBecomeLost())
6823  {
6824  /*
6825  Warning: This is a carefully designed algorithm.
6826  Do not modify unless you really know what you're doing :)
6827  */
6828  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
6829  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
6830  for(;;)
6831  {
6832  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6833  {
6834  pAllocationInfo->memoryType = UINT32_MAX;
6835  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
6836  pAllocationInfo->offset = 0;
6837  pAllocationInfo->size = hAllocation->GetSize();
6838  pAllocationInfo->pMappedData = VMA_NULL;
6839  pAllocationInfo->pUserData = hAllocation->GetUserData();
6840  return;
6841  }
6842  else if(localLastUseFrameIndex == localCurrFrameIndex)
6843  {
6844  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6845  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6846  pAllocationInfo->offset = hAllocation->GetOffset();
6847  pAllocationInfo->size = hAllocation->GetSize();
6848  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6849  pAllocationInfo->pUserData = hAllocation->GetUserData();
6850  return;
6851  }
6852  else // Last use time earlier than current time.
6853  {
6854  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
6855  {
6856  localLastUseFrameIndex = localCurrFrameIndex;
6857  }
6858  }
6859  }
6860  }
6861  // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic.
6862  else
6863  {
6864  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6865  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6866  pAllocationInfo->offset = hAllocation->GetOffset();
6867  pAllocationInfo->size = hAllocation->GetSize();
6868  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6869  pAllocationInfo->pUserData = hAllocation->GetUserData();
6870  }
6871 }
6872 
6873 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
6874 {
6875  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
6876 
6877  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
6878 
6879  if(newCreateInfo.maxBlockCount == 0)
6880  {
6881  newCreateInfo.maxBlockCount = SIZE_MAX;
6882  }
6883  if(newCreateInfo.blockSize == 0)
6884  {
6885  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
6886  }
6887 
6888  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
6889 
6890  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
6891  if(res != VK_SUCCESS)
6892  {
6893  vma_delete(this, *pPool);
6894  *pPool = VMA_NULL;
6895  return res;
6896  }
6897 
6898  // Add to m_Pools.
6899  {
6900  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6901  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
6902  }
6903 
6904  return VK_SUCCESS;
6905 }
6906 
6907 void VmaAllocator_T::DestroyPool(VmaPool pool)
6908 {
6909  // Remove from m_Pools.
6910  {
6911  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6912  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
6913  VMA_ASSERT(success && "Pool not found in Allocator.");
6914  }
6915 
6916  vma_delete(this, pool);
6917 }
6918 
6919 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
6920 {
6921  pool->m_BlockVector.GetPoolStats(pPoolStats);
6922 }
6923 
6924 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
6925 {
6926  m_CurrentFrameIndex.store(frameIndex);
6927 }
6928 
6929 void VmaAllocator_T::MakePoolAllocationsLost(
6930  VmaPool hPool,
6931  size_t* pLostAllocationCount)
6932 {
6933  hPool->m_BlockVector.MakePoolAllocationsLost(
6934  m_CurrentFrameIndex.load(),
6935  pLostAllocationCount);
6936 }
6937 
6938 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
6939 {
6940  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST);
6941  (*pAllocation)->InitLost();
6942 }
6943 
6944 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
6945 {
6946  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
6947 
6948  VkResult res;
6949  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
6950  {
6951  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
6952  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
6953  {
6954  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6955  if(res == VK_SUCCESS)
6956  {
6957  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
6958  }
6959  }
6960  else
6961  {
6962  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
6963  }
6964  }
6965  else
6966  {
6967  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6968  }
6969 
6970  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
6971  {
6972  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
6973  }
6974 
6975  return res;
6976 }
6977 
6978 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
6979 {
6980  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
6981  {
6982  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
6983  }
6984 
6985  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
6986 
6987  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
6988  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
6989  {
6990  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
6991  m_HeapSizeLimit[heapIndex] += size;
6992  }
6993 }
6994 
6995 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
6996 {
6997  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
6998 
6999  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7000  {
7001  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7002  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
7003  VMA_ASSERT(pOwnAllocations);
7004  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pOwnAllocations, allocation);
7005  VMA_ASSERT(success);
7006  }
7007 
7008  VkDeviceMemory hMemory = allocation->GetMemory();
7009 
7010  if(allocation->GetMappedData() != VMA_NULL)
7011  {
7012  vkUnmapMemory(m_hDevice, hMemory);
7013  }
7014 
7015  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7016 
7017  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
7018 }
7019 
7020 #if VMA_STATS_STRING_ENABLED
7021 
7022 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7023 {
7024  bool ownAllocationsStarted = false;
7025  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7026  {
7027  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7028  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7029  {
7030  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
7031  VMA_ASSERT(pOwnAllocVector);
7032  if(pOwnAllocVector->empty() == false)
7033  {
7034  if(ownAllocationsStarted == false)
7035  {
7036  ownAllocationsStarted = true;
7037  json.WriteString("OwnAllocations");
7038  json.BeginObject();
7039  }
7040 
7041  json.BeginString("Type ");
7042  json.ContinueString(memTypeIndex);
7043  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7044  {
7045  json.ContinueString(" Mapped");
7046  }
7047  json.EndString();
7048 
7049  json.BeginArray();
7050 
7051  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
7052  {
7053  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
7054  json.BeginObject(true);
7055 
7056  json.WriteString("Size");
7057  json.WriteNumber(hAlloc->GetSize());
7058 
7059  json.WriteString("Type");
7060  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7061 
7062  json.EndObject();
7063  }
7064 
7065  json.EndArray();
7066  }
7067  }
7068  }
7069  if(ownAllocationsStarted)
7070  {
7071  json.EndObject();
7072  }
7073 
7074  {
7075  bool allocationsStarted = false;
7076  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7077  {
7078  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7079  {
7080  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
7081  {
7082  if(allocationsStarted == false)
7083  {
7084  allocationsStarted = true;
7085  json.WriteString("DefaultPools");
7086  json.BeginObject();
7087  }
7088 
7089  json.BeginString("Type ");
7090  json.ContinueString(memTypeIndex);
7091  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7092  {
7093  json.ContinueString(" Mapped");
7094  }
7095  json.EndString();
7096 
7097  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json);
7098  }
7099  }
7100  }
7101  if(allocationsStarted)
7102  {
7103  json.EndObject();
7104  }
7105  }
7106 
7107  {
7108  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7109  const size_t poolCount = m_Pools.size();
7110  if(poolCount > 0)
7111  {
7112  json.WriteString("Pools");
7113  json.BeginArray();
7114  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
7115  {
7116  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
7117  }
7118  json.EndArray();
7119  }
7120  }
7121 }
7122 
7123 #endif // #if VMA_STATS_STRING_ENABLED
7124 
7125 static VkResult AllocateMemoryForImage(
7126  VmaAllocator allocator,
7127  VkImage image,
7128  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7129  VmaSuballocationType suballocType,
7130  VmaAllocation* pAllocation)
7131 {
7132  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
7133 
7134  VkMemoryRequirements vkMemReq = {};
7135  (*allocator->GetVulkanFunctions().vkGetImageMemoryRequirements)(allocator->m_hDevice, image, &vkMemReq);
7136 
7137  return allocator->AllocateMemory(
7138  vkMemReq,
7139  *pAllocationCreateInfo,
7140  suballocType,
7141  pAllocation);
7142 }
7143 
7145 // Public interface
7146 
7147 VkResult vmaCreateAllocator(
7148  const VmaAllocatorCreateInfo* pCreateInfo,
7149  VmaAllocator* pAllocator)
7150 {
7151  VMA_ASSERT(pCreateInfo && pAllocator);
7152  VMA_DEBUG_LOG("vmaCreateAllocator");
7153  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
7154  return VK_SUCCESS;
7155 }
7156 
7157 void vmaDestroyAllocator(
7158  VmaAllocator allocator)
7159 {
7160  if(allocator != VK_NULL_HANDLE)
7161  {
7162  VMA_DEBUG_LOG("vmaDestroyAllocator");
7163  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
7164  vma_delete(&allocationCallbacks, allocator);
7165  }
7166 }
7167 
7168 void vmaGetPhysicalDeviceProperties(
7169  VmaAllocator allocator,
7170  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
7171 {
7172  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
7173  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
7174 }
7175 
7176 void vmaGetMemoryProperties(
7177  VmaAllocator allocator,
7178  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
7179 {
7180  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
7181  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
7182 }
7183 
7184 void vmaGetMemoryTypeProperties(
7185  VmaAllocator allocator,
7186  uint32_t memoryTypeIndex,
7187  VkMemoryPropertyFlags* pFlags)
7188 {
7189  VMA_ASSERT(allocator && pFlags);
7190  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
7191  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
7192 }
7193 
7194 void vmaSetCurrentFrameIndex(
7195  VmaAllocator allocator,
7196  uint32_t frameIndex)
7197 {
7198  VMA_ASSERT(allocator);
7199  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
7200 
7201  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7202 
7203  allocator->SetCurrentFrameIndex(frameIndex);
7204 }
7205 
7206 void vmaCalculateStats(
7207  VmaAllocator allocator,
7208  VmaStats* pStats)
7209 {
7210  VMA_ASSERT(allocator && pStats);
7211  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7212  allocator->CalculateStats(pStats);
7213 }
7214 
7215 #if VMA_STATS_STRING_ENABLED
7216 
7217 void vmaBuildStatsString(
7218  VmaAllocator allocator,
7219  char** ppStatsString,
7220  VkBool32 detailedMap)
7221 {
7222  VMA_ASSERT(allocator && ppStatsString);
7223  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7224 
7225  VmaStringBuilder sb(allocator);
7226  {
7227  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
7228  json.BeginObject();
7229 
7230  VmaStats stats;
7231  allocator->CalculateStats(&stats);
7232 
7233  json.WriteString("Total");
7234  VmaPrintStatInfo(json, stats.total);
7235 
7236  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
7237  {
7238  json.BeginString("Heap ");
7239  json.ContinueString(heapIndex);
7240  json.EndString();
7241  json.BeginObject();
7242 
7243  json.WriteString("Size");
7244  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
7245 
7246  json.WriteString("Flags");
7247  json.BeginArray(true);
7248  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
7249  {
7250  json.WriteString("DEVICE_LOCAL");
7251  }
7252  json.EndArray();
7253 
7254  if(stats.memoryHeap[heapIndex].blockCount > 0)
7255  {
7256  json.WriteString("Stats");
7257  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
7258  }
7259 
7260  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
7261  {
7262  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
7263  {
7264  json.BeginString("Type ");
7265  json.ContinueString(typeIndex);
7266  json.EndString();
7267 
7268  json.BeginObject();
7269 
7270  json.WriteString("Flags");
7271  json.BeginArray(true);
7272  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
7273  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
7274  {
7275  json.WriteString("DEVICE_LOCAL");
7276  }
7277  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7278  {
7279  json.WriteString("HOST_VISIBLE");
7280  }
7281  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
7282  {
7283  json.WriteString("HOST_COHERENT");
7284  }
7285  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
7286  {
7287  json.WriteString("HOST_CACHED");
7288  }
7289  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
7290  {
7291  json.WriteString("LAZILY_ALLOCATED");
7292  }
7293  json.EndArray();
7294 
7295  if(stats.memoryType[typeIndex].blockCount > 0)
7296  {
7297  json.WriteString("Stats");
7298  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
7299  }
7300 
7301  json.EndObject();
7302  }
7303  }
7304 
7305  json.EndObject();
7306  }
7307  if(detailedMap == VK_TRUE)
7308  {
7309  allocator->PrintDetailedMap(json);
7310  }
7311 
7312  json.EndObject();
7313  }
7314 
7315  const size_t len = sb.GetLength();
7316  char* const pChars = vma_new_array(allocator, char, len + 1);
7317  if(len > 0)
7318  {
7319  memcpy(pChars, sb.GetData(), len);
7320  }
7321  pChars[len] = '\0';
7322  *ppStatsString = pChars;
7323 }
7324 
7325 void vmaFreeStatsString(
7326  VmaAllocator allocator,
7327  char* pStatsString)
7328 {
7329  if(pStatsString != VMA_NULL)
7330  {
7331  VMA_ASSERT(allocator);
7332  size_t len = strlen(pStatsString);
7333  vma_delete_array(allocator, pStatsString, len + 1);
7334  }
7335 }
7336 
7337 #endif // #if VMA_STATS_STRING_ENABLED
7338 
7341 VkResult vmaFindMemoryTypeIndex(
7342  VmaAllocator allocator,
7343  uint32_t memoryTypeBits,
7344  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7345  uint32_t* pMemoryTypeIndex)
7346 {
7347  VMA_ASSERT(allocator != VK_NULL_HANDLE);
7348  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
7349  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
7350 
7351  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
7352  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
7353  if(preferredFlags == 0)
7354  {
7355  preferredFlags = requiredFlags;
7356  }
7357  // preferredFlags, if not 0, must be a superset of requiredFlags.
7358  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
7359 
7360  // Convert usage to requiredFlags and preferredFlags.
7361  switch(pAllocationCreateInfo->usage)
7362  {
7363  case VMA_MEMORY_USAGE_UNKNOWN:
7364  break;
7365  case VMA_MEMORY_USAGE_GPU_ONLY:
7366  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7367  break;
7368  case VMA_MEMORY_USAGE_CPU_ONLY:
7369  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7370  break;
7371  case VMA_MEMORY_USAGE_CPU_TO_GPU:
7372  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7373  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7374  break;
7375  case VMA_MEMORY_USAGE_GPU_TO_CPU:
7376  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7377  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
7378  break;
7379  default:
7380  break;
7381  }
7382 
7383  if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0)
7384  {
7385  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7386  }
7387 
7388  *pMemoryTypeIndex = UINT32_MAX;
7389  uint32_t minCost = UINT32_MAX;
7390  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
7391  memTypeIndex < allocator->GetMemoryTypeCount();
7392  ++memTypeIndex, memTypeBit <<= 1)
7393  {
7394  // This memory type is acceptable according to memoryTypeBits bitmask.
7395  if((memTypeBit & memoryTypeBits) != 0)
7396  {
7397  const VkMemoryPropertyFlags currFlags =
7398  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
7399  // This memory type contains requiredFlags.
7400  if((requiredFlags & ~currFlags) == 0)
7401  {
7402  // Calculate cost as number of bits from preferredFlags not present in this memory type.
7403  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
7404  // Remember memory type with lowest cost.
7405  if(currCost < minCost)
7406  {
7407  *pMemoryTypeIndex = memTypeIndex;
7408  if(currCost == 0)
7409  {
7410  return VK_SUCCESS;
7411  }
7412  minCost = currCost;
7413  }
7414  }
7415  }
7416  }
7417  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
7418 }
7419 
7420 VkResult vmaCreatePool(
7421  VmaAllocator allocator,
7422  const VmaPoolCreateInfo* pCreateInfo,
7423  VmaPool* pPool)
7424 {
7425  VMA_ASSERT(allocator && pCreateInfo && pPool);
7426 
7427  VMA_DEBUG_LOG("vmaCreatePool");
7428 
7429  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7430 
7431  return allocator->CreatePool(pCreateInfo, pPool);
7432 }
7433 
7434 void vmaDestroyPool(
7435  VmaAllocator allocator,
7436  VmaPool pool)
7437 {
7438  VMA_ASSERT(allocator && pool);
7439 
7440  VMA_DEBUG_LOG("vmaDestroyPool");
7441 
7442  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7443 
7444  allocator->DestroyPool(pool);
7445 }
7446 
7447 void vmaGetPoolStats(
7448  VmaAllocator allocator,
7449  VmaPool pool,
7450  VmaPoolStats* pPoolStats)
7451 {
7452  VMA_ASSERT(allocator && pool && pPoolStats);
7453 
7454  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7455 
7456  allocator->GetPoolStats(pool, pPoolStats);
7457 }
7458 
7459 void vmaMakePoolAllocationsLost(
7460  VmaAllocator allocator,
7461  VmaPool pool,
7462  size_t* pLostAllocationCount)
7463 {
7464  VMA_ASSERT(allocator && pool);
7465 
7466  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7467 
7468  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
7469 }
7470 
7471 VkResult vmaAllocateMemory(
7472  VmaAllocator allocator,
7473  const VkMemoryRequirements* pVkMemoryRequirements,
7474  const VmaAllocationCreateInfo* pCreateInfo,
7475  VmaAllocation* pAllocation,
7476  VmaAllocationInfo* pAllocationInfo)
7477 {
7478  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
7479 
7480  VMA_DEBUG_LOG("vmaAllocateMemory");
7481 
7482  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7483 
7484  VkResult result = allocator->AllocateMemory(
7485  *pVkMemoryRequirements,
7486  *pCreateInfo,
7487  VMA_SUBALLOCATION_TYPE_UNKNOWN,
7488  pAllocation);
7489 
7490  if(pAllocationInfo && result == VK_SUCCESS)
7491  {
7492  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7493  }
7494 
7495  return result;
7496 }
7497 
7498 VkResult vmaAllocateMemoryForBuffer(
7499  VmaAllocator allocator,
7500  VkBuffer buffer,
7501  const VmaAllocationCreateInfo* pCreateInfo,
7502  VmaAllocation* pAllocation,
7503  VmaAllocationInfo* pAllocationInfo)
7504 {
7505  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7506 
7507  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
7508 
7509  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7510 
7511  VkMemoryRequirements vkMemReq = {};
7512  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, buffer, &vkMemReq);
7513 
7514  VkResult result = allocator->AllocateMemory(
7515  vkMemReq,
7516  *pCreateInfo,
7517  VMA_SUBALLOCATION_TYPE_BUFFER,
7518  pAllocation);
7519 
7520  if(pAllocationInfo && result == VK_SUCCESS)
7521  {
7522  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7523  }
7524 
7525  return result;
7526 }
7527 
7528 VkResult vmaAllocateMemoryForImage(
7529  VmaAllocator allocator,
7530  VkImage image,
7531  const VmaAllocationCreateInfo* pCreateInfo,
7532  VmaAllocation* pAllocation,
7533  VmaAllocationInfo* pAllocationInfo)
7534 {
7535  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7536 
7537  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
7538 
7539  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7540 
7541  VkResult result = AllocateMemoryForImage(
7542  allocator,
7543  image,
7544  pCreateInfo,
7545  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
7546  pAllocation);
7547 
7548  if(pAllocationInfo && result == VK_SUCCESS)
7549  {
7550  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7551  }
7552 
7553  return result;
7554 }
7555 
7556 void vmaFreeMemory(
7557  VmaAllocator allocator,
7558  VmaAllocation allocation)
7559 {
7560  VMA_ASSERT(allocator && allocation);
7561 
7562  VMA_DEBUG_LOG("vmaFreeMemory");
7563 
7564  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7565 
7566  allocator->FreeMemory(allocation);
7567 }
7568 
7569 void vmaGetAllocationInfo(
7570  VmaAllocator allocator,
7571  VmaAllocation allocation,
7572  VmaAllocationInfo* pAllocationInfo)
7573 {
7574  VMA_ASSERT(allocator && allocation && pAllocationInfo);
7575 
7576  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7577 
7578  allocator->GetAllocationInfo(allocation, pAllocationInfo);
7579 }
7580 
7581 void vmaSetAllocationUserData(
7582  VmaAllocator allocator,
7583  VmaAllocation allocation,
7584  void* pUserData)
7585 {
7586  VMA_ASSERT(allocator && allocation);
7587 
7588  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7589 
7590  allocation->SetUserData(pUserData);
7591 }
7592 
7593 void vmaCreateLostAllocation(
7594  VmaAllocator allocator,
7595  VmaAllocation* pAllocation)
7596 {
7597  VMA_ASSERT(allocator && pAllocation);
7598 
7599  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
7600 
7601  allocator->CreateLostAllocation(pAllocation);
7602 }
7603 
7604 VkResult vmaMapMemory(
7605  VmaAllocator allocator,
7606  VmaAllocation allocation,
7607  void** ppData)
7608 {
7609  VMA_ASSERT(allocator && allocation && ppData);
7610 
7611  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7612 
7613  return vkMapMemory(allocator->m_hDevice, allocation->GetMemory(),
7614  allocation->GetOffset(), allocation->GetSize(), 0, ppData);
7615 }
7616 
7617 void vmaUnmapMemory(
7618  VmaAllocator allocator,
7619  VmaAllocation allocation)
7620 {
7621  VMA_ASSERT(allocator && allocation);
7622 
7623  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7624 
7625  vkUnmapMemory(allocator->m_hDevice, allocation->GetMemory());
7626 }
7627 
7628 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
7629 {
7630  VMA_ASSERT(allocator);
7631 
7632  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7633 
7634  allocator->UnmapPersistentlyMappedMemory();
7635 }
7636 
7637 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
7638 {
7639  VMA_ASSERT(allocator);
7640 
7641  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7642 
7643  return allocator->MapPersistentlyMappedMemory();
7644 }
7645 
7646 VkResult vmaDefragment(
7647  VmaAllocator allocator,
7648  VmaAllocation* pAllocations,
7649  size_t allocationCount,
7650  VkBool32* pAllocationsChanged,
7651  const VmaDefragmentationInfo *pDefragmentationInfo,
7652  VmaDefragmentationStats* pDefragmentationStats)
7653 {
7654  VMA_ASSERT(allocator && pAllocations);
7655 
7656  VMA_DEBUG_LOG("vmaDefragment");
7657 
7658  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7659 
7660  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
7661 }
7662 
7663 VkResult vmaCreateBuffer(
7664  VmaAllocator allocator,
7665  const VkBufferCreateInfo* pBufferCreateInfo,
7666  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7667  VkBuffer* pBuffer,
7668  VmaAllocation* pAllocation,
7669  VmaAllocationInfo* pAllocationInfo)
7670 {
7671  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
7672 
7673  VMA_DEBUG_LOG("vmaCreateBuffer");
7674 
7675  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7676 
7677  *pBuffer = VK_NULL_HANDLE;
7678  *pAllocation = VK_NULL_HANDLE;
7679 
7680  // 1. Create VkBuffer.
7681  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
7682  allocator->m_hDevice,
7683  pBufferCreateInfo,
7684  allocator->GetAllocationCallbacks(),
7685  pBuffer);
7686  if(res >= 0)
7687  {
7688  // 2. vkGetBufferMemoryRequirements.
7689  VkMemoryRequirements vkMemReq = {};
7690  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, *pBuffer, &vkMemReq);
7691 
7692  // 3. Allocate memory using allocator.
7693  res = allocator->AllocateMemory(
7694  vkMemReq,
7695  *pAllocationCreateInfo,
7696  VMA_SUBALLOCATION_TYPE_BUFFER,
7697  pAllocation);
7698  if(res >= 0)
7699  {
7700  // 3. Bind buffer with memory.
7701  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
7702  allocator->m_hDevice,
7703  *pBuffer,
7704  (*pAllocation)->GetMemory(),
7705  (*pAllocation)->GetOffset());
7706  if(res >= 0)
7707  {
7708  // All steps succeeded.
7709  if(pAllocationInfo != VMA_NULL)
7710  {
7711  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7712  }
7713  return VK_SUCCESS;
7714  }
7715  allocator->FreeMemory(*pAllocation);
7716  *pAllocation = VK_NULL_HANDLE;
7717  return res;
7718  }
7719  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
7720  *pBuffer = VK_NULL_HANDLE;
7721  return res;
7722  }
7723  return res;
7724 }
7725 
7726 void vmaDestroyBuffer(
7727  VmaAllocator allocator,
7728  VkBuffer buffer,
7729  VmaAllocation allocation)
7730 {
7731  if(buffer != VK_NULL_HANDLE)
7732  {
7733  VMA_ASSERT(allocator);
7734 
7735  VMA_DEBUG_LOG("vmaDestroyBuffer");
7736 
7737  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7738 
7739  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
7740 
7741  allocator->FreeMemory(allocation);
7742  }
7743 }
7744 
7745 VkResult vmaCreateImage(
7746  VmaAllocator allocator,
7747  const VkImageCreateInfo* pImageCreateInfo,
7748  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7749  VkImage* pImage,
7750  VmaAllocation* pAllocation,
7751  VmaAllocationInfo* pAllocationInfo)
7752 {
7753  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
7754 
7755  VMA_DEBUG_LOG("vmaCreateImage");
7756 
7757  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7758 
7759  *pImage = VK_NULL_HANDLE;
7760  *pAllocation = VK_NULL_HANDLE;
7761 
7762  // 1. Create VkImage.
7763  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
7764  allocator->m_hDevice,
7765  pImageCreateInfo,
7766  allocator->GetAllocationCallbacks(),
7767  pImage);
7768  if(res >= 0)
7769  {
7770  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
7771  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
7772  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
7773 
7774  // 2. Allocate memory using allocator.
7775  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
7776  if(res >= 0)
7777  {
7778  // 3. Bind image with memory.
7779  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
7780  allocator->m_hDevice,
7781  *pImage,
7782  (*pAllocation)->GetMemory(),
7783  (*pAllocation)->GetOffset());
7784  if(res >= 0)
7785  {
7786  // All steps succeeded.
7787  if(pAllocationInfo != VMA_NULL)
7788  {
7789  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7790  }
7791  return VK_SUCCESS;
7792  }
7793  allocator->FreeMemory(*pAllocation);
7794  *pAllocation = VK_NULL_HANDLE;
7795  return res;
7796  }
7797  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
7798  *pImage = VK_NULL_HANDLE;
7799  return res;
7800  }
7801  return res;
7802 }
7803 
7804 void vmaDestroyImage(
7805  VmaAllocator allocator,
7806  VkImage image,
7807  VmaAllocation allocation)
7808 {
7809  if(image != VK_NULL_HANDLE)
7810  {
7811  VMA_ASSERT(allocator);
7812 
7813  VMA_DEBUG_LOG("vmaDestroyImage");
7814 
7815  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7816 
7817  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
7818 
7819  allocator->FreeMemory(allocation);
7820  }
7821 }
7822 
7823 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:440
-
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:463
-
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:794
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
393 #include <vulkan/vulkan.h>
394 
396 
400 VK_DEFINE_HANDLE(VmaAllocator)
401 
402 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
404  VmaAllocator allocator,
405  uint32_t memoryType,
406  VkDeviceMemory memory,
407  VkDeviceSize size);
409 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
410  VmaAllocator allocator,
411  uint32_t memoryType,
412  VkDeviceMemory memory,
413  VkDeviceSize size);
414 
422 typedef struct VmaDeviceMemoryCallbacks {
424  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
426  PFN_vmaFreeDeviceMemoryFunction pfnFree;
427 } VmaDeviceMemoryCallbacks;
428 
430 typedef enum VmaAllocatorFlagBits {
435  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
436 
437  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
438 } VmaAllocatorFlagBits;
439 typedef VkFlags VmaAllocatorFlags;
440 
445 typedef struct VmaVulkanFunctions {
446  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
447  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
448  PFN_vkAllocateMemory vkAllocateMemory;
449  PFN_vkFreeMemory vkFreeMemory;
450  PFN_vkMapMemory vkMapMemory;
451  PFN_vkUnmapMemory vkUnmapMemory;
452  PFN_vkBindBufferMemory vkBindBufferMemory;
453  PFN_vkBindImageMemory vkBindImageMemory;
454  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
455  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
456  PFN_vkCreateBuffer vkCreateBuffer;
457  PFN_vkDestroyBuffer vkDestroyBuffer;
458  PFN_vkCreateImage vkCreateImage;
459  PFN_vkDestroyImage vkDestroyImage;
460 } VmaVulkanFunctions;
461 
463 typedef struct VmaAllocatorCreateInfo
464 {
466  VmaAllocatorFlags flags;
468 
469  VkPhysicalDevice physicalDevice;
471 
472  VkDevice device;
474 
475  VkDeviceSize preferredLargeHeapBlockSize;
477 
478  VkDeviceSize preferredSmallHeapBlockSize;
480 
481  const VkAllocationCallbacks* pAllocationCallbacks;
483 
484  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
498  uint32_t frameInUseCount;
516  const VkDeviceSize* pHeapSizeLimit;
528  const VmaVulkanFunctions* pVulkanFunctions;
529 } VmaAllocatorCreateInfo;
530 
532 VkResult vmaCreateAllocator(
533  const VmaAllocatorCreateInfo* pCreateInfo,
534  VmaAllocator* pAllocator);
535 
537 void vmaDestroyAllocator(
538  VmaAllocator allocator);
539 
544 void vmaGetPhysicalDeviceProperties(
545  VmaAllocator allocator,
546  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
547 
552 void vmaGetMemoryProperties(
553  VmaAllocator allocator,
554  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
555 
562 void vmaGetMemoryTypeProperties(
563  VmaAllocator allocator,
564  uint32_t memoryTypeIndex,
565  VkMemoryPropertyFlags* pFlags);
566 
575 void vmaSetCurrentFrameIndex(
576  VmaAllocator allocator,
577  uint32_t frameIndex);
578 
581 typedef struct VmaStatInfo
582 {
584  uint32_t blockCount;
586  uint32_t allocationCount;
588  uint32_t unusedRangeCount;
590  VkDeviceSize usedBytes;
592  VkDeviceSize unusedBytes;
593  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
594  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
595 } VmaStatInfo;
596 
598 typedef struct VmaStats
599 {
600  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
601  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
602  VmaStatInfo total;
603 } VmaStats;
604 
606 void vmaCalculateStats(
607  VmaAllocator allocator,
608  VmaStats* pStats);
609 
610 #define VMA_STATS_STRING_ENABLED 1
611 
612 #if VMA_STATS_STRING_ENABLED
613 
615 
617 void vmaBuildStatsString(
618  VmaAllocator allocator,
619  char** ppStatsString,
620  VkBool32 detailedMap);
621 
622 void vmaFreeStatsString(
623  VmaAllocator allocator,
624  char* pStatsString);
625 
626 #endif // #if VMA_STATS_STRING_ENABLED
627 
630 
635 VK_DEFINE_HANDLE(VmaPool)
636 
637 typedef enum VmaMemoryUsage
638 {
640  VMA_MEMORY_USAGE_UNKNOWN = 0,
642  VMA_MEMORY_USAGE_GPU_ONLY = 1,
644 
645  VMA_MEMORY_USAGE_CPU_ONLY = 2,
647 
648  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
650 
651  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
652  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
653 } VmaMemoryUsage;
654 
656 typedef enum VmaAllocationCreateFlagBits {
668  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001,
669 
679  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
689  VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004,
700  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
707  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
708 
709  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
710 } VmaAllocationCreateFlagBits;
711 typedef VkFlags VmaAllocationCreateFlags;
712 
713 typedef struct VmaAllocationCreateInfo
714 {
716  VmaAllocationCreateFlags flags;
722  VmaMemoryUsage usage;
727  VkMemoryPropertyFlags requiredFlags;
733  VkMemoryPropertyFlags preferredFlags;
735  void* pUserData;
740  VmaPool pool;
741 } VmaAllocationCreateInfo;
742 
757 VkResult vmaFindMemoryTypeIndex(
758  VmaAllocator allocator,
759  uint32_t memoryTypeBits,
760  const VmaAllocationCreateInfo* pAllocationCreateInfo,
761  uint32_t* pMemoryTypeIndex);
762 
765 
770 typedef enum VmaPoolCreateFlagBits {
780  VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001,
798  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
799 
800  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
801 } VmaPoolCreateFlagBits;
802 typedef VkFlags VmaPoolCreateFlags;
803 
806 typedef struct VmaPoolCreateInfo {
809  uint32_t memoryTypeIndex;
812  VmaPoolCreateFlags flags;
817  VkDeviceSize blockSize;
822  size_t minBlockCount;
830  size_t maxBlockCount;
844  uint32_t frameInUseCount;
845 } VmaPoolCreateInfo;
846 
849 typedef struct VmaPoolStats {
852  VkDeviceSize size;
855  VkDeviceSize unusedSize;
858  size_t allocationCount;
861  size_t unusedRangeCount;
868  VkDeviceSize unusedRangeSizeMax;
869 } VmaPoolStats;
870 
877 VkResult vmaCreatePool(
878  VmaAllocator allocator,
879  const VmaPoolCreateInfo* pCreateInfo,
880  VmaPool* pPool);
881 
884 void vmaDestroyPool(
885  VmaAllocator allocator,
886  VmaPool pool);
887 
894 void vmaGetPoolStats(
895  VmaAllocator allocator,
896  VmaPool pool,
897  VmaPoolStats* pPoolStats);
898 
905 void vmaMakePoolAllocationsLost(
906  VmaAllocator allocator,
907  VmaPool pool,
908  size_t* pLostAllocationCount);
909 
910 VK_DEFINE_HANDLE(VmaAllocation)
911 
912 
914 typedef struct VmaAllocationInfo {
919  uint32_t memoryType;
928  VkDeviceMemory deviceMemory;
933  VkDeviceSize offset;
938  VkDeviceSize size;
944  void* pMappedData;
949  void* pUserData;
950 } VmaAllocationInfo;
951 
962 VkResult vmaAllocateMemory(
963  VmaAllocator allocator,
964  const VkMemoryRequirements* pVkMemoryRequirements,
965  const VmaAllocationCreateInfo* pCreateInfo,
966  VmaAllocation* pAllocation,
967  VmaAllocationInfo* pAllocationInfo);
968 
975 VkResult vmaAllocateMemoryForBuffer(
976  VmaAllocator allocator,
977  VkBuffer buffer,
978  const VmaAllocationCreateInfo* pCreateInfo,
979  VmaAllocation* pAllocation,
980  VmaAllocationInfo* pAllocationInfo);
981 
983 VkResult vmaAllocateMemoryForImage(
984  VmaAllocator allocator,
985  VkImage image,
986  const VmaAllocationCreateInfo* pCreateInfo,
987  VmaAllocation* pAllocation,
988  VmaAllocationInfo* pAllocationInfo);
989 
991 void vmaFreeMemory(
992  VmaAllocator allocator,
993  VmaAllocation allocation);
994 
996 void vmaGetAllocationInfo(
997  VmaAllocator allocator,
998  VmaAllocation allocation,
999  VmaAllocationInfo* pAllocationInfo);
1000 
1002 void vmaSetAllocationUserData(
1003  VmaAllocator allocator,
1004  VmaAllocation allocation,
1005  void* pUserData);
1006 
1017 void vmaCreateLostAllocation(
1018  VmaAllocator allocator,
1019  VmaAllocation* pAllocation);
1020 
1029 VkResult vmaMapMemory(
1030  VmaAllocator allocator,
1031  VmaAllocation allocation,
1032  void** ppData);
1033 
1034 void vmaUnmapMemory(
1035  VmaAllocator allocator,
1036  VmaAllocation allocation);
1037 
1059 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
1060 
1068 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
1069 
1071 typedef struct VmaDefragmentationInfo {
1076  VkDeviceSize maxBytesToMove;
1081  uint32_t maxAllocationsToMove;
1082 } VmaDefragmentationInfo;
1083 
1085 typedef struct VmaDefragmentationStats {
1087  VkDeviceSize bytesMoved;
1089  VkDeviceSize bytesFreed;
1091  uint32_t allocationsMoved;
1093  uint32_t deviceMemoryBlocksFreed;
1094 } VmaDefragmentationStats;
1095 
1166 VkResult vmaDefragment(
1167  VmaAllocator allocator,
1168  VmaAllocation* pAllocations,
1169  size_t allocationCount,
1170  VkBool32* pAllocationsChanged,
1171  const VmaDefragmentationInfo *pDefragmentationInfo,
1172  VmaDefragmentationStats* pDefragmentationStats);
1173 
1176 
1199 VkResult vmaCreateBuffer(
1200  VmaAllocator allocator,
1201  const VkBufferCreateInfo* pBufferCreateInfo,
1202  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1203  VkBuffer* pBuffer,
1204  VmaAllocation* pAllocation,
1205  VmaAllocationInfo* pAllocationInfo);
1206 
1215 void vmaDestroyBuffer(
1216  VmaAllocator allocator,
1217  VkBuffer buffer,
1218  VmaAllocation allocation);
1219 
1221 VkResult vmaCreateImage(
1222  VmaAllocator allocator,
1223  const VkImageCreateInfo* pImageCreateInfo,
1224  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1225  VkImage* pImage,
1226  VmaAllocation* pAllocation,
1227  VmaAllocationInfo* pAllocationInfo);
1228 
1237 void vmaDestroyImage(
1238  VmaAllocator allocator,
1239  VkImage image,
1240  VmaAllocation allocation);
1241 
1244 #ifdef __cplusplus
1245 }
1246 #endif
1247 
1248 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
1249 
1250 // For Visual Studio IntelliSense.
1251 #ifdef __INTELLISENSE__
1252 #define VMA_IMPLEMENTATION
1253 #endif
1254 
1255 #ifdef VMA_IMPLEMENTATION
1256 #undef VMA_IMPLEMENTATION
1257 
1258 #include <cstdint>
1259 #include <cstdlib>
1260 #include <cstring>
1261 
1262 /*******************************************************************************
1263 CONFIGURATION SECTION
1264 
1265 Define some of these macros before each #include of this header or change them
1266 here if you need other then default behavior depending on your environment.
1267 */
1268 
1269 /*
1270 Define this macro to 1 to make the library fetch pointers to Vulkan functions
1271 internally, like:
1272 
1273  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1274 
1275 Define to 0 if you are going to provide you own pointers to Vulkan functions via
1276 VmaAllocatorCreateInfo::pVulkanFunctions.
1277 */
1278 #ifndef VMA_STATIC_VULKAN_FUNCTIONS
1279 #define VMA_STATIC_VULKAN_FUNCTIONS 1
1280 #endif
1281 
1282 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
1283 //#define VMA_USE_STL_CONTAINERS 1
1284 
1285 /* Set this macro to 1 to make the library including and using STL containers:
1286 std::pair, std::vector, std::list, std::unordered_map.
1287 
1288 Set it to 0 or undefined to make the library using its own implementation of
1289 the containers.
1290 */
1291 #if VMA_USE_STL_CONTAINERS
1292  #define VMA_USE_STL_VECTOR 1
1293  #define VMA_USE_STL_UNORDERED_MAP 1
1294  #define VMA_USE_STL_LIST 1
1295 #endif
1296 
1297 #if VMA_USE_STL_VECTOR
1298  #include <vector>
1299 #endif
1300 
1301 #if VMA_USE_STL_UNORDERED_MAP
1302  #include <unordered_map>
1303 #endif
1304 
1305 #if VMA_USE_STL_LIST
1306  #include <list>
1307 #endif
1308 
1309 /*
1310 Following headers are used in this CONFIGURATION section only, so feel free to
1311 remove them if not needed.
1312 */
1313 #include <cassert> // for assert
1314 #include <algorithm> // for min, max
1315 #include <mutex> // for std::mutex
1316 #include <atomic> // for std::atomic
1317 
1318 #if !defined(_WIN32)
1319  #include <malloc.h> // for aligned_alloc()
1320 #endif
1321 
1322 // Normal assert to check for programmer's errors, especially in Debug configuration.
1323 #ifndef VMA_ASSERT
1324  #ifdef _DEBUG
1325  #define VMA_ASSERT(expr) assert(expr)
1326  #else
1327  #define VMA_ASSERT(expr)
1328  #endif
1329 #endif
1330 
1331 // Assert that will be called very often, like inside data structures e.g. operator[].
1332 // Making it non-empty can make program slow.
1333 #ifndef VMA_HEAVY_ASSERT
1334  #ifdef _DEBUG
1335  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
1336  #else
1337  #define VMA_HEAVY_ASSERT(expr)
1338  #endif
1339 #endif
1340 
1341 #ifndef VMA_NULL
1342  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
1343  #define VMA_NULL nullptr
1344 #endif
1345 
1346 #ifndef VMA_ALIGN_OF
1347  #define VMA_ALIGN_OF(type) (__alignof(type))
1348 #endif
1349 
1350 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
1351  #if defined(_WIN32)
1352  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
1353  #else
1354  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
1355  #endif
1356 #endif
1357 
1358 #ifndef VMA_SYSTEM_FREE
1359  #if defined(_WIN32)
1360  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
1361  #else
1362  #define VMA_SYSTEM_FREE(ptr) free(ptr)
1363  #endif
1364 #endif
1365 
1366 #ifndef VMA_MIN
1367  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
1368 #endif
1369 
1370 #ifndef VMA_MAX
1371  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
1372 #endif
1373 
1374 #ifndef VMA_SWAP
1375  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
1376 #endif
1377 
1378 #ifndef VMA_SORT
1379  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
1380 #endif
1381 
1382 #ifndef VMA_DEBUG_LOG
1383  #define VMA_DEBUG_LOG(format, ...)
1384  /*
1385  #define VMA_DEBUG_LOG(format, ...) do { \
1386  printf(format, __VA_ARGS__); \
1387  printf("\n"); \
1388  } while(false)
1389  */
1390 #endif
1391 
1392 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
1393 #if VMA_STATS_STRING_ENABLED
1394  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
1395  {
1396  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
1397  }
1398  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
1399  {
1400  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
1401  }
1402  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
1403  {
1404  snprintf(outStr, strLen, "%p", ptr);
1405  }
1406 #endif
1407 
1408 #ifndef VMA_MUTEX
1409  class VmaMutex
1410  {
1411  public:
1412  VmaMutex() { }
1413  ~VmaMutex() { }
1414  void Lock() { m_Mutex.lock(); }
1415  void Unlock() { m_Mutex.unlock(); }
1416  private:
1417  std::mutex m_Mutex;
1418  };
1419  #define VMA_MUTEX VmaMutex
1420 #endif
1421 
1422 /*
1423 If providing your own implementation, you need to implement a subset of std::atomic:
1424 
1425 - Constructor(uint32_t desired)
1426 - uint32_t load() const
1427 - void store(uint32_t desired)
1428 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
1429 */
1430 #ifndef VMA_ATOMIC_UINT32
1431  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
1432 #endif
1433 
1434 #ifndef VMA_BEST_FIT
1435 
1447  #define VMA_BEST_FIT (1)
1448 #endif
1449 
1450 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
1451 
1455  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
1456 #endif
1457 
1458 #ifndef VMA_DEBUG_ALIGNMENT
1459 
1463  #define VMA_DEBUG_ALIGNMENT (1)
1464 #endif
1465 
1466 #ifndef VMA_DEBUG_MARGIN
1467 
1471  #define VMA_DEBUG_MARGIN (0)
1472 #endif
1473 
1474 #ifndef VMA_DEBUG_GLOBAL_MUTEX
1475 
1479  #define VMA_DEBUG_GLOBAL_MUTEX (0)
1480 #endif
1481 
1482 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
1483 
1487  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
1488 #endif
1489 
1490 #ifndef VMA_SMALL_HEAP_MAX_SIZE
1491  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
1493 #endif
1494 
1495 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
1496  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
1498 #endif
1499 
1500 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
1501  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
1503 #endif
1504 
1505 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
1506 
1507 /*******************************************************************************
1508 END OF CONFIGURATION
1509 */
1510 
1511 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
1512  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
1513 
1514 // Returns number of bits set to 1 in (v).
1515 static inline uint32_t CountBitsSet(uint32_t v)
1516 {
1517  uint32_t c = v - ((v >> 1) & 0x55555555);
1518  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
1519  c = ((c >> 4) + c) & 0x0F0F0F0F;
1520  c = ((c >> 8) + c) & 0x00FF00FF;
1521  c = ((c >> 16) + c) & 0x0000FFFF;
1522  return c;
1523 }
1524 
1525 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
1526 // Use types like uint32_t, uint64_t as T.
1527 template <typename T>
1528 static inline T VmaAlignUp(T val, T align)
1529 {
1530  return (val + align - 1) / align * align;
1531 }
1532 
1533 // Division with mathematical rounding to nearest number.
1534 template <typename T>
1535 inline T VmaRoundDiv(T x, T y)
1536 {
1537  return (x + (y / (T)2)) / y;
1538 }
1539 
1540 #ifndef VMA_SORT
1541 
1542 template<typename Iterator, typename Compare>
1543 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
1544 {
1545  Iterator centerValue = end; --centerValue;
1546  Iterator insertIndex = beg;
1547  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
1548  {
1549  if(cmp(*memTypeIndex, *centerValue))
1550  {
1551  if(insertIndex != memTypeIndex)
1552  {
1553  VMA_SWAP(*memTypeIndex, *insertIndex);
1554  }
1555  ++insertIndex;
1556  }
1557  }
1558  if(insertIndex != centerValue)
1559  {
1560  VMA_SWAP(*insertIndex, *centerValue);
1561  }
1562  return insertIndex;
1563 }
1564 
1565 template<typename Iterator, typename Compare>
1566 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
1567 {
1568  if(beg < end)
1569  {
1570  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
1571  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
1572  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
1573  }
1574 }
1575 
1576 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
1577 
1578 #endif // #ifndef VMA_SORT
1579 
1580 /*
1581 Returns true if two memory blocks occupy overlapping pages.
1582 ResourceA must be in less memory offset than ResourceB.
1583 
1584 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1585 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1586 */
1587 static inline bool VmaBlocksOnSamePage(
1588  VkDeviceSize resourceAOffset,
1589  VkDeviceSize resourceASize,
1590  VkDeviceSize resourceBOffset,
1591  VkDeviceSize pageSize)
1592 {
1593  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1594  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1595  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1596  VkDeviceSize resourceBStart = resourceBOffset;
1597  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1598  return resourceAEndPage == resourceBStartPage;
1599 }
1600 
1601 enum VmaSuballocationType
1602 {
1603  VMA_SUBALLOCATION_TYPE_FREE = 0,
1604  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1605  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1606  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1607  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1608  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1609  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1610 };
1611 
1612 /*
1613 Returns true if given suballocation types could conflict and must respect
1614 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1615 or linear image and another one is optimal image. If type is unknown, behave
1616 conservatively.
1617 */
1618 static inline bool VmaIsBufferImageGranularityConflict(
1619  VmaSuballocationType suballocType1,
1620  VmaSuballocationType suballocType2)
1621 {
1622  if(suballocType1 > suballocType2)
1623  {
1624  VMA_SWAP(suballocType1, suballocType2);
1625  }
1626 
1627  switch(suballocType1)
1628  {
1629  case VMA_SUBALLOCATION_TYPE_FREE:
1630  return false;
1631  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1632  return true;
1633  case VMA_SUBALLOCATION_TYPE_BUFFER:
1634  return
1635  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1636  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1637  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1638  return
1639  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1640  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1641  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1642  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1643  return
1644  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1645  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1646  return false;
1647  default:
1648  VMA_ASSERT(0);
1649  return true;
1650  }
1651 }
1652 
1653 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1654 struct VmaMutexLock
1655 {
1656 public:
1657  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1658  m_pMutex(useMutex ? &mutex : VMA_NULL)
1659  {
1660  if(m_pMutex)
1661  {
1662  m_pMutex->Lock();
1663  }
1664  }
1665 
1666  ~VmaMutexLock()
1667  {
1668  if(m_pMutex)
1669  {
1670  m_pMutex->Unlock();
1671  }
1672  }
1673 
1674 private:
1675  VMA_MUTEX* m_pMutex;
1676 };
1677 
1678 #if VMA_DEBUG_GLOBAL_MUTEX
1679  static VMA_MUTEX gDebugGlobalMutex;
1680  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
1681 #else
1682  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1683 #endif
1684 
1685 // Minimum size of a free suballocation to register it in the free suballocation collection.
1686 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1687 
1688 /*
1689 Performs binary search and returns iterator to first element that is greater or
1690 equal to (key), according to comparison (cmp).
1691 
1692 Cmp should return true if first argument is less than second argument.
1693 
1694 Returned value is the found element, if present in the collection or place where
1695 new element with value (key) should be inserted.
1696 */
1697 template <typename IterT, typename KeyT, typename CmpT>
1698 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1699 {
1700  size_t down = 0, up = (end - beg);
1701  while(down < up)
1702  {
1703  const size_t mid = (down + up) / 2;
1704  if(cmp(*(beg+mid), key))
1705  {
1706  down = mid + 1;
1707  }
1708  else
1709  {
1710  up = mid;
1711  }
1712  }
1713  return beg + down;
1714 }
1715 
1717 // Memory allocation
1718 
1719 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1720 {
1721  if((pAllocationCallbacks != VMA_NULL) &&
1722  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1723  {
1724  return (*pAllocationCallbacks->pfnAllocation)(
1725  pAllocationCallbacks->pUserData,
1726  size,
1727  alignment,
1728  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1729  }
1730  else
1731  {
1732  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1733  }
1734 }
1735 
1736 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1737 {
1738  if((pAllocationCallbacks != VMA_NULL) &&
1739  (pAllocationCallbacks->pfnFree != VMA_NULL))
1740  {
1741  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1742  }
1743  else
1744  {
1745  VMA_SYSTEM_FREE(ptr);
1746  }
1747 }
1748 
1749 template<typename T>
1750 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1751 {
1752  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1753 }
1754 
1755 template<typename T>
1756 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1757 {
1758  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1759 }
1760 
1761 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1762 
1763 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1764 
1765 template<typename T>
1766 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1767 {
1768  ptr->~T();
1769  VmaFree(pAllocationCallbacks, ptr);
1770 }
1771 
1772 template<typename T>
1773 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1774 {
1775  if(ptr != VMA_NULL)
1776  {
1777  for(size_t i = count; i--; )
1778  {
1779  ptr[i].~T();
1780  }
1781  VmaFree(pAllocationCallbacks, ptr);
1782  }
1783 }
1784 
1785 // STL-compatible allocator.
1786 template<typename T>
1787 class VmaStlAllocator
1788 {
1789 public:
1790  const VkAllocationCallbacks* const m_pCallbacks;
1791  typedef T value_type;
1792 
1793  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1794  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1795 
1796  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1797  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1798 
1799  template<typename U>
1800  bool operator==(const VmaStlAllocator<U>& rhs) const
1801  {
1802  return m_pCallbacks == rhs.m_pCallbacks;
1803  }
1804  template<typename U>
1805  bool operator!=(const VmaStlAllocator<U>& rhs) const
1806  {
1807  return m_pCallbacks != rhs.m_pCallbacks;
1808  }
1809 
1810  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1811 };
1812 
1813 #if VMA_USE_STL_VECTOR
1814 
1815 #define VmaVector std::vector
1816 
1817 template<typename T, typename allocatorT>
1818 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1819 {
1820  vec.insert(vec.begin() + index, item);
1821 }
1822 
1823 template<typename T, typename allocatorT>
1824 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1825 {
1826  vec.erase(vec.begin() + index);
1827 }
1828 
1829 #else // #if VMA_USE_STL_VECTOR
1830 
1831 /* Class with interface compatible with subset of std::vector.
1832 T must be POD because constructors and destructors are not called and memcpy is
1833 used for these objects. */
1834 template<typename T, typename AllocatorT>
1835 class VmaVector
1836 {
1837 public:
1838  typedef T value_type;
1839 
1840  VmaVector(const AllocatorT& allocator) :
1841  m_Allocator(allocator),
1842  m_pArray(VMA_NULL),
1843  m_Count(0),
1844  m_Capacity(0)
1845  {
1846  }
1847 
1848  VmaVector(size_t count, const AllocatorT& allocator) :
1849  m_Allocator(allocator),
1850  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
1851  m_Count(count),
1852  m_Capacity(count)
1853  {
1854  }
1855 
1856  VmaVector(const VmaVector<T, AllocatorT>& src) :
1857  m_Allocator(src.m_Allocator),
1858  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
1859  m_Count(src.m_Count),
1860  m_Capacity(src.m_Count)
1861  {
1862  if(m_Count != 0)
1863  {
1864  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1865  }
1866  }
1867 
1868  ~VmaVector()
1869  {
1870  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1871  }
1872 
1873  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1874  {
1875  if(&rhs != this)
1876  {
1877  resize(rhs.m_Count);
1878  if(m_Count != 0)
1879  {
1880  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1881  }
1882  }
1883  return *this;
1884  }
1885 
1886  bool empty() const { return m_Count == 0; }
1887  size_t size() const { return m_Count; }
1888  T* data() { return m_pArray; }
1889  const T* data() const { return m_pArray; }
1890 
1891  T& operator[](size_t index)
1892  {
1893  VMA_HEAVY_ASSERT(index < m_Count);
1894  return m_pArray[index];
1895  }
1896  const T& operator[](size_t index) const
1897  {
1898  VMA_HEAVY_ASSERT(index < m_Count);
1899  return m_pArray[index];
1900  }
1901 
1902  T& front()
1903  {
1904  VMA_HEAVY_ASSERT(m_Count > 0);
1905  return m_pArray[0];
1906  }
1907  const T& front() const
1908  {
1909  VMA_HEAVY_ASSERT(m_Count > 0);
1910  return m_pArray[0];
1911  }
1912  T& back()
1913  {
1914  VMA_HEAVY_ASSERT(m_Count > 0);
1915  return m_pArray[m_Count - 1];
1916  }
1917  const T& back() const
1918  {
1919  VMA_HEAVY_ASSERT(m_Count > 0);
1920  return m_pArray[m_Count - 1];
1921  }
1922 
1923  void reserve(size_t newCapacity, bool freeMemory = false)
1924  {
1925  newCapacity = VMA_MAX(newCapacity, m_Count);
1926 
1927  if((newCapacity < m_Capacity) && !freeMemory)
1928  {
1929  newCapacity = m_Capacity;
1930  }
1931 
1932  if(newCapacity != m_Capacity)
1933  {
1934  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1935  if(m_Count != 0)
1936  {
1937  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1938  }
1939  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1940  m_Capacity = newCapacity;
1941  m_pArray = newArray;
1942  }
1943  }
1944 
1945  void resize(size_t newCount, bool freeMemory = false)
1946  {
1947  size_t newCapacity = m_Capacity;
1948  if(newCount > m_Capacity)
1949  {
1950  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1951  }
1952  else if(freeMemory)
1953  {
1954  newCapacity = newCount;
1955  }
1956 
1957  if(newCapacity != m_Capacity)
1958  {
1959  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1960  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1961  if(elementsToCopy != 0)
1962  {
1963  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1964  }
1965  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1966  m_Capacity = newCapacity;
1967  m_pArray = newArray;
1968  }
1969 
1970  m_Count = newCount;
1971  }
1972 
1973  void clear(bool freeMemory = false)
1974  {
1975  resize(0, freeMemory);
1976  }
1977 
1978  void insert(size_t index, const T& src)
1979  {
1980  VMA_HEAVY_ASSERT(index <= m_Count);
1981  const size_t oldCount = size();
1982  resize(oldCount + 1);
1983  if(index < oldCount)
1984  {
1985  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1986  }
1987  m_pArray[index] = src;
1988  }
1989 
1990  void remove(size_t index)
1991  {
1992  VMA_HEAVY_ASSERT(index < m_Count);
1993  const size_t oldCount = size();
1994  if(index < oldCount - 1)
1995  {
1996  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1997  }
1998  resize(oldCount - 1);
1999  }
2000 
2001  void push_back(const T& src)
2002  {
2003  const size_t newIndex = size();
2004  resize(newIndex + 1);
2005  m_pArray[newIndex] = src;
2006  }
2007 
2008  void pop_back()
2009  {
2010  VMA_HEAVY_ASSERT(m_Count > 0);
2011  resize(size() - 1);
2012  }
2013 
2014  void push_front(const T& src)
2015  {
2016  insert(0, src);
2017  }
2018 
2019  void pop_front()
2020  {
2021  VMA_HEAVY_ASSERT(m_Count > 0);
2022  remove(0);
2023  }
2024 
2025  typedef T* iterator;
2026 
2027  iterator begin() { return m_pArray; }
2028  iterator end() { return m_pArray + m_Count; }
2029 
2030 private:
2031  AllocatorT m_Allocator;
2032  T* m_pArray;
2033  size_t m_Count;
2034  size_t m_Capacity;
2035 };
2036 
2037 template<typename T, typename allocatorT>
2038 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2039 {
2040  vec.insert(index, item);
2041 }
2042 
2043 template<typename T, typename allocatorT>
2044 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2045 {
2046  vec.remove(index);
2047 }
2048 
2049 #endif // #if VMA_USE_STL_VECTOR
2050 
2051 template<typename CmpLess, typename VectorT>
2052 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2053 {
2054  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2055  vector.data(),
2056  vector.data() + vector.size(),
2057  value,
2058  CmpLess()) - vector.data();
2059  VmaVectorInsert(vector, indexToInsert, value);
2060  return indexToInsert;
2061 }
2062 
2063 template<typename CmpLess, typename VectorT>
2064 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2065 {
2066  CmpLess comparator;
2067  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2068  vector.begin(),
2069  vector.end(),
2070  value,
2071  comparator);
2072  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
2073  {
2074  size_t indexToRemove = it - vector.begin();
2075  VmaVectorRemove(vector, indexToRemove);
2076  return true;
2077  }
2078  return false;
2079 }
2080 
2081 template<typename CmpLess, typename VectorT>
2082 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
2083 {
2084  CmpLess comparator;
2085  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2086  vector.data(),
2087  vector.data() + vector.size(),
2088  value,
2089  comparator);
2090  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
2091  {
2092  return it - vector.begin();
2093  }
2094  else
2095  {
2096  return vector.size();
2097  }
2098 }
2099 
2101 // class VmaPoolAllocator
2102 
2103 /*
2104 Allocator for objects of type T using a list of arrays (pools) to speed up
2105 allocation. Number of elements that can be allocated is not bounded because
2106 allocator can create multiple blocks.
2107 */
2108 template<typename T>
2109 class VmaPoolAllocator
2110 {
2111 public:
2112  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
2113  ~VmaPoolAllocator();
2114  void Clear();
2115  T* Alloc();
2116  void Free(T* ptr);
2117 
2118 private:
2119  union Item
2120  {
2121  uint32_t NextFreeIndex;
2122  T Value;
2123  };
2124 
2125  struct ItemBlock
2126  {
2127  Item* pItems;
2128  uint32_t FirstFreeIndex;
2129  };
2130 
2131  const VkAllocationCallbacks* m_pAllocationCallbacks;
2132  size_t m_ItemsPerBlock;
2133  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
2134 
2135  ItemBlock& CreateNewBlock();
2136 };
2137 
2138 template<typename T>
2139 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
2140  m_pAllocationCallbacks(pAllocationCallbacks),
2141  m_ItemsPerBlock(itemsPerBlock),
2142  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
2143 {
2144  VMA_ASSERT(itemsPerBlock > 0);
2145 }
2146 
2147 template<typename T>
2148 VmaPoolAllocator<T>::~VmaPoolAllocator()
2149 {
2150  Clear();
2151 }
2152 
2153 template<typename T>
2154 void VmaPoolAllocator<T>::Clear()
2155 {
2156  for(size_t i = m_ItemBlocks.size(); i--; )
2157  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
2158  m_ItemBlocks.clear();
2159 }
2160 
2161 template<typename T>
2162 T* VmaPoolAllocator<T>::Alloc()
2163 {
2164  for(size_t i = m_ItemBlocks.size(); i--; )
2165  {
2166  ItemBlock& block = m_ItemBlocks[i];
2167  // This block has some free items: Use first one.
2168  if(block.FirstFreeIndex != UINT32_MAX)
2169  {
2170  Item* const pItem = &block.pItems[block.FirstFreeIndex];
2171  block.FirstFreeIndex = pItem->NextFreeIndex;
2172  return &pItem->Value;
2173  }
2174  }
2175 
2176  // No block has free item: Create new one and use it.
2177  ItemBlock& newBlock = CreateNewBlock();
2178  Item* const pItem = &newBlock.pItems[0];
2179  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
2180  return &pItem->Value;
2181 }
2182 
2183 template<typename T>
2184 void VmaPoolAllocator<T>::Free(T* ptr)
2185 {
2186  // Search all memory blocks to find ptr.
2187  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
2188  {
2189  ItemBlock& block = m_ItemBlocks[i];
2190 
2191  // Casting to union.
2192  Item* pItemPtr;
2193  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
2194 
2195  // Check if pItemPtr is in address range of this block.
2196  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
2197  {
2198  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
2199  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
2200  block.FirstFreeIndex = index;
2201  return;
2202  }
2203  }
2204  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
2205 }
2206 
2207 template<typename T>
2208 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
2209 {
2210  ItemBlock newBlock = {
2211  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
2212 
2213  m_ItemBlocks.push_back(newBlock);
2214 
2215  // Setup singly-linked list of all free items in this block.
2216  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
2217  newBlock.pItems[i].NextFreeIndex = i + 1;
2218  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
2219  return m_ItemBlocks.back();
2220 }
2221 
2223 // class VmaRawList, VmaList
2224 
2225 #if VMA_USE_STL_LIST
2226 
2227 #define VmaList std::list
2228 
2229 #else // #if VMA_USE_STL_LIST
2230 
2231 template<typename T>
2232 struct VmaListItem
2233 {
2234  VmaListItem* pPrev;
2235  VmaListItem* pNext;
2236  T Value;
2237 };
2238 
2239 // Doubly linked list.
2240 template<typename T>
2241 class VmaRawList
2242 {
2243 public:
2244  typedef VmaListItem<T> ItemType;
2245 
2246  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
2247  ~VmaRawList();
2248  void Clear();
2249 
2250  size_t GetCount() const { return m_Count; }
2251  bool IsEmpty() const { return m_Count == 0; }
2252 
2253  ItemType* Front() { return m_pFront; }
2254  const ItemType* Front() const { return m_pFront; }
2255  ItemType* Back() { return m_pBack; }
2256  const ItemType* Back() const { return m_pBack; }
2257 
2258  ItemType* PushBack();
2259  ItemType* PushFront();
2260  ItemType* PushBack(const T& value);
2261  ItemType* PushFront(const T& value);
2262  void PopBack();
2263  void PopFront();
2264 
2265  // Item can be null - it means PushBack.
2266  ItemType* InsertBefore(ItemType* pItem);
2267  // Item can be null - it means PushFront.
2268  ItemType* InsertAfter(ItemType* pItem);
2269 
2270  ItemType* InsertBefore(ItemType* pItem, const T& value);
2271  ItemType* InsertAfter(ItemType* pItem, const T& value);
2272 
2273  void Remove(ItemType* pItem);
2274 
2275 private:
2276  const VkAllocationCallbacks* const m_pAllocationCallbacks;
2277  VmaPoolAllocator<ItemType> m_ItemAllocator;
2278  ItemType* m_pFront;
2279  ItemType* m_pBack;
2280  size_t m_Count;
2281 
2282  // Declared not defined, to block copy constructor and assignment operator.
2283  VmaRawList(const VmaRawList<T>& src);
2284  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
2285 };
2286 
2287 template<typename T>
2288 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
2289  m_pAllocationCallbacks(pAllocationCallbacks),
2290  m_ItemAllocator(pAllocationCallbacks, 128),
2291  m_pFront(VMA_NULL),
2292  m_pBack(VMA_NULL),
2293  m_Count(0)
2294 {
2295 }
2296 
2297 template<typename T>
2298 VmaRawList<T>::~VmaRawList()
2299 {
2300  // Intentionally not calling Clear, because that would be unnecessary
2301  // computations to return all items to m_ItemAllocator as free.
2302 }
2303 
2304 template<typename T>
2305 void VmaRawList<T>::Clear()
2306 {
2307  if(IsEmpty() == false)
2308  {
2309  ItemType* pItem = m_pBack;
2310  while(pItem != VMA_NULL)
2311  {
2312  ItemType* const pPrevItem = pItem->pPrev;
2313  m_ItemAllocator.Free(pItem);
2314  pItem = pPrevItem;
2315  }
2316  m_pFront = VMA_NULL;
2317  m_pBack = VMA_NULL;
2318  m_Count = 0;
2319  }
2320 }
2321 
2322 template<typename T>
2323 VmaListItem<T>* VmaRawList<T>::PushBack()
2324 {
2325  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2326  pNewItem->pNext = VMA_NULL;
2327  if(IsEmpty())
2328  {
2329  pNewItem->pPrev = VMA_NULL;
2330  m_pFront = pNewItem;
2331  m_pBack = pNewItem;
2332  m_Count = 1;
2333  }
2334  else
2335  {
2336  pNewItem->pPrev = m_pBack;
2337  m_pBack->pNext = pNewItem;
2338  m_pBack = pNewItem;
2339  ++m_Count;
2340  }
2341  return pNewItem;
2342 }
2343 
2344 template<typename T>
2345 VmaListItem<T>* VmaRawList<T>::PushFront()
2346 {
2347  ItemType* const pNewItem = m_ItemAllocator.Alloc();
2348  pNewItem->pPrev = VMA_NULL;
2349  if(IsEmpty())
2350  {
2351  pNewItem->pNext = VMA_NULL;
2352  m_pFront = pNewItem;
2353  m_pBack = pNewItem;
2354  m_Count = 1;
2355  }
2356  else
2357  {
2358  pNewItem->pNext = m_pFront;
2359  m_pFront->pPrev = pNewItem;
2360  m_pFront = pNewItem;
2361  ++m_Count;
2362  }
2363  return pNewItem;
2364 }
2365 
2366 template<typename T>
2367 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
2368 {
2369  ItemType* const pNewItem = PushBack();
2370  pNewItem->Value = value;
2371  return pNewItem;
2372 }
2373 
2374 template<typename T>
2375 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
2376 {
2377  ItemType* const pNewItem = PushFront();
2378  pNewItem->Value = value;
2379  return pNewItem;
2380 }
2381 
2382 template<typename T>
2383 void VmaRawList<T>::PopBack()
2384 {
2385  VMA_HEAVY_ASSERT(m_Count > 0);
2386  ItemType* const pBackItem = m_pBack;
2387  ItemType* const pPrevItem = pBackItem->pPrev;
2388  if(pPrevItem != VMA_NULL)
2389  {
2390  pPrevItem->pNext = VMA_NULL;
2391  }
2392  m_pBack = pPrevItem;
2393  m_ItemAllocator.Free(pBackItem);
2394  --m_Count;
2395 }
2396 
2397 template<typename T>
2398 void VmaRawList<T>::PopFront()
2399 {
2400  VMA_HEAVY_ASSERT(m_Count > 0);
2401  ItemType* const pFrontItem = m_pFront;
2402  ItemType* const pNextItem = pFrontItem->pNext;
2403  if(pNextItem != VMA_NULL)
2404  {
2405  pNextItem->pPrev = VMA_NULL;
2406  }
2407  m_pFront = pNextItem;
2408  m_ItemAllocator.Free(pFrontItem);
2409  --m_Count;
2410 }
2411 
2412 template<typename T>
2413 void VmaRawList<T>::Remove(ItemType* pItem)
2414 {
2415  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
2416  VMA_HEAVY_ASSERT(m_Count > 0);
2417 
2418  if(pItem->pPrev != VMA_NULL)
2419  {
2420  pItem->pPrev->pNext = pItem->pNext;
2421  }
2422  else
2423  {
2424  VMA_HEAVY_ASSERT(m_pFront == pItem);
2425  m_pFront = pItem->pNext;
2426  }
2427 
2428  if(pItem->pNext != VMA_NULL)
2429  {
2430  pItem->pNext->pPrev = pItem->pPrev;
2431  }
2432  else
2433  {
2434  VMA_HEAVY_ASSERT(m_pBack == pItem);
2435  m_pBack = pItem->pPrev;
2436  }
2437 
2438  m_ItemAllocator.Free(pItem);
2439  --m_Count;
2440 }
2441 
2442 template<typename T>
2443 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
2444 {
2445  if(pItem != VMA_NULL)
2446  {
2447  ItemType* const prevItem = pItem->pPrev;
2448  ItemType* const newItem = m_ItemAllocator.Alloc();
2449  newItem->pPrev = prevItem;
2450  newItem->pNext = pItem;
2451  pItem->pPrev = newItem;
2452  if(prevItem != VMA_NULL)
2453  {
2454  prevItem->pNext = newItem;
2455  }
2456  else
2457  {
2458  VMA_HEAVY_ASSERT(m_pFront == pItem);
2459  m_pFront = newItem;
2460  }
2461  ++m_Count;
2462  return newItem;
2463  }
2464  else
2465  return PushBack();
2466 }
2467 
2468 template<typename T>
2469 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
2470 {
2471  if(pItem != VMA_NULL)
2472  {
2473  ItemType* const nextItem = pItem->pNext;
2474  ItemType* const newItem = m_ItemAllocator.Alloc();
2475  newItem->pNext = nextItem;
2476  newItem->pPrev = pItem;
2477  pItem->pNext = newItem;
2478  if(nextItem != VMA_NULL)
2479  {
2480  nextItem->pPrev = newItem;
2481  }
2482  else
2483  {
2484  VMA_HEAVY_ASSERT(m_pBack == pItem);
2485  m_pBack = newItem;
2486  }
2487  ++m_Count;
2488  return newItem;
2489  }
2490  else
2491  return PushFront();
2492 }
2493 
2494 template<typename T>
2495 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
2496 {
2497  ItemType* const newItem = InsertBefore(pItem);
2498  newItem->Value = value;
2499  return newItem;
2500 }
2501 
2502 template<typename T>
2503 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
2504 {
2505  ItemType* const newItem = InsertAfter(pItem);
2506  newItem->Value = value;
2507  return newItem;
2508 }
2509 
2510 template<typename T, typename AllocatorT>
2511 class VmaList
2512 {
2513 public:
2514  class iterator
2515  {
2516  public:
2517  iterator() :
2518  m_pList(VMA_NULL),
2519  m_pItem(VMA_NULL)
2520  {
2521  }
2522 
2523  T& operator*() const
2524  {
2525  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2526  return m_pItem->Value;
2527  }
2528  T* operator->() const
2529  {
2530  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2531  return &m_pItem->Value;
2532  }
2533 
2534  iterator& operator++()
2535  {
2536  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2537  m_pItem = m_pItem->pNext;
2538  return *this;
2539  }
2540  iterator& operator--()
2541  {
2542  if(m_pItem != VMA_NULL)
2543  {
2544  m_pItem = m_pItem->pPrev;
2545  }
2546  else
2547  {
2548  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
2549  m_pItem = m_pList->Back();
2550  }
2551  return *this;
2552  }
2553 
2554  iterator operator++(int)
2555  {
2556  iterator result = *this;
2557  ++*this;
2558  return result;
2559  }
2560  iterator operator--(int)
2561  {
2562  iterator result = *this;
2563  --*this;
2564  return result;
2565  }
2566 
2567  bool operator==(const iterator& rhs) const
2568  {
2569  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2570  return m_pItem == rhs.m_pItem;
2571  }
2572  bool operator!=(const iterator& rhs) const
2573  {
2574  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2575  return m_pItem != rhs.m_pItem;
2576  }
2577 
2578  private:
2579  VmaRawList<T>* m_pList;
2580  VmaListItem<T>* m_pItem;
2581 
2582  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
2583  m_pList(pList),
2584  m_pItem(pItem)
2585  {
2586  }
2587 
2588  friend class VmaList<T, AllocatorT>;
2589  };
2590 
2591  class const_iterator
2592  {
2593  public:
2594  const_iterator() :
2595  m_pList(VMA_NULL),
2596  m_pItem(VMA_NULL)
2597  {
2598  }
2599 
2600  const_iterator(const iterator& src) :
2601  m_pList(src.m_pList),
2602  m_pItem(src.m_pItem)
2603  {
2604  }
2605 
2606  const T& operator*() const
2607  {
2608  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2609  return m_pItem->Value;
2610  }
2611  const T* operator->() const
2612  {
2613  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2614  return &m_pItem->Value;
2615  }
2616 
2617  const_iterator& operator++()
2618  {
2619  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
2620  m_pItem = m_pItem->pNext;
2621  return *this;
2622  }
2623  const_iterator& operator--()
2624  {
2625  if(m_pItem != VMA_NULL)
2626  {
2627  m_pItem = m_pItem->pPrev;
2628  }
2629  else
2630  {
2631  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2632  m_pItem = m_pList->Back();
2633  }
2634  return *this;
2635  }
2636 
2637  const_iterator operator++(int)
2638  {
2639  const_iterator result = *this;
2640  ++*this;
2641  return result;
2642  }
2643  const_iterator operator--(int)
2644  {
2645  const_iterator result = *this;
2646  --*this;
2647  return result;
2648  }
2649 
2650  bool operator==(const const_iterator& rhs) const
2651  {
2652  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2653  return m_pItem == rhs.m_pItem;
2654  }
2655  bool operator!=(const const_iterator& rhs) const
2656  {
2657  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2658  return m_pItem != rhs.m_pItem;
2659  }
2660 
2661  private:
2662  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2663  m_pList(pList),
2664  m_pItem(pItem)
2665  {
2666  }
2667 
2668  const VmaRawList<T>* m_pList;
2669  const VmaListItem<T>* m_pItem;
2670 
2671  friend class VmaList<T, AllocatorT>;
2672  };
2673 
2674  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2675 
2676  bool empty() const { return m_RawList.IsEmpty(); }
2677  size_t size() const { return m_RawList.GetCount(); }
2678 
2679  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2680  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2681 
2682  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2683  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2684 
2685  void clear() { m_RawList.Clear(); }
2686  void push_back(const T& value) { m_RawList.PushBack(value); }
2687  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2688  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2689 
2690 private:
2691  VmaRawList<T> m_RawList;
2692 };
2693 
2694 #endif // #if VMA_USE_STL_LIST
2695 
2697 // class VmaMap
2698 
2699 // Unused in this version.
2700 #if 0
2701 
2702 #if VMA_USE_STL_UNORDERED_MAP
2703 
2704 #define VmaPair std::pair
2705 
2706 #define VMA_MAP_TYPE(KeyT, ValueT) \
2707  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2708 
2709 #else // #if VMA_USE_STL_UNORDERED_MAP
2710 
2711 template<typename T1, typename T2>
2712 struct VmaPair
2713 {
2714  T1 first;
2715  T2 second;
2716 
2717  VmaPair() : first(), second() { }
2718  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2719 };
2720 
2721 /* Class compatible with subset of interface of std::unordered_map.
2722 KeyT, ValueT must be POD because they will be stored in VmaVector.
2723 */
2724 template<typename KeyT, typename ValueT>
2725 class VmaMap
2726 {
2727 public:
2728  typedef VmaPair<KeyT, ValueT> PairType;
2729  typedef PairType* iterator;
2730 
2731  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2732 
2733  iterator begin() { return m_Vector.begin(); }
2734  iterator end() { return m_Vector.end(); }
2735 
2736  void insert(const PairType& pair);
2737  iterator find(const KeyT& key);
2738  void erase(iterator it);
2739 
2740 private:
2741  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2742 };
2743 
2744 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2745 
2746 template<typename FirstT, typename SecondT>
2747 struct VmaPairFirstLess
2748 {
2749  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2750  {
2751  return lhs.first < rhs.first;
2752  }
2753  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2754  {
2755  return lhs.first < rhsFirst;
2756  }
2757 };
2758 
2759 template<typename KeyT, typename ValueT>
2760 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2761 {
2762  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2763  m_Vector.data(),
2764  m_Vector.data() + m_Vector.size(),
2765  pair,
2766  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2767  VmaVectorInsert(m_Vector, indexToInsert, pair);
2768 }
2769 
2770 template<typename KeyT, typename ValueT>
2771 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2772 {
2773  PairType* it = VmaBinaryFindFirstNotLess(
2774  m_Vector.data(),
2775  m_Vector.data() + m_Vector.size(),
2776  key,
2777  VmaPairFirstLess<KeyT, ValueT>());
2778  if((it != m_Vector.end()) && (it->first == key))
2779  {
2780  return it;
2781  }
2782  else
2783  {
2784  return m_Vector.end();
2785  }
2786 }
2787 
2788 template<typename KeyT, typename ValueT>
2789 void VmaMap<KeyT, ValueT>::erase(iterator it)
2790 {
2791  VmaVectorRemove(m_Vector, it - m_Vector.begin());
2792 }
2793 
2794 #endif // #if VMA_USE_STL_UNORDERED_MAP
2795 
2796 #endif // #if 0
2797 
2799 
2800 class VmaDeviceMemoryBlock;
2801 
2802 enum VMA_BLOCK_VECTOR_TYPE
2803 {
2804  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2805  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2806  VMA_BLOCK_VECTOR_TYPE_COUNT
2807 };
2808 
2809 static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags)
2810 {
2811  return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ?
2812  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2813  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2814 }
2815 
2816 struct VmaAllocation_T
2817 {
2818 public:
2819  enum ALLOCATION_TYPE
2820  {
2821  ALLOCATION_TYPE_NONE,
2822  ALLOCATION_TYPE_BLOCK,
2823  ALLOCATION_TYPE_OWN,
2824  };
2825 
2826  VmaAllocation_T(uint32_t currentFrameIndex) :
2827  m_Alignment(1),
2828  m_Size(0),
2829  m_pUserData(VMA_NULL),
2830  m_Type(ALLOCATION_TYPE_NONE),
2831  m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN),
2832  m_LastUseFrameIndex(currentFrameIndex)
2833  {
2834  }
2835 
2836  void InitBlockAllocation(
2837  VmaPool hPool,
2838  VmaDeviceMemoryBlock* block,
2839  VkDeviceSize offset,
2840  VkDeviceSize alignment,
2841  VkDeviceSize size,
2842  VmaSuballocationType suballocationType,
2843  void* pUserData,
2844  bool canBecomeLost)
2845  {
2846  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2847  VMA_ASSERT(block != VMA_NULL);
2848  m_Type = ALLOCATION_TYPE_BLOCK;
2849  m_Alignment = alignment;
2850  m_Size = size;
2851  m_pUserData = pUserData;
2852  m_SuballocationType = suballocationType;
2853  m_BlockAllocation.m_hPool = hPool;
2854  m_BlockAllocation.m_Block = block;
2855  m_BlockAllocation.m_Offset = offset;
2856  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
2857  }
2858 
2859  void InitLost()
2860  {
2861  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2862  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
2863  m_Type = ALLOCATION_TYPE_BLOCK;
2864  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
2865  m_BlockAllocation.m_Block = VMA_NULL;
2866  m_BlockAllocation.m_Offset = 0;
2867  m_BlockAllocation.m_CanBecomeLost = true;
2868  }
2869 
2870  void ChangeBlockAllocation(
2871  VmaDeviceMemoryBlock* block,
2872  VkDeviceSize offset)
2873  {
2874  VMA_ASSERT(block != VMA_NULL);
2875  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2876  m_BlockAllocation.m_Block = block;
2877  m_BlockAllocation.m_Offset = offset;
2878  }
2879 
2880  void InitOwnAllocation(
2881  uint32_t memoryTypeIndex,
2882  VkDeviceMemory hMemory,
2883  VmaSuballocationType suballocationType,
2884  bool persistentMap,
2885  void* pMappedData,
2886  VkDeviceSize size,
2887  void* pUserData)
2888  {
2889  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2890  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2891  m_Type = ALLOCATION_TYPE_OWN;
2892  m_Alignment = 0;
2893  m_Size = size;
2894  m_pUserData = pUserData;
2895  m_SuballocationType = suballocationType;
2896  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2897  m_OwnAllocation.m_hMemory = hMemory;
2898  m_OwnAllocation.m_PersistentMap = persistentMap;
2899  m_OwnAllocation.m_pMappedData = pMappedData;
2900  }
2901 
2902  ALLOCATION_TYPE GetType() const { return m_Type; }
2903  VkDeviceSize GetAlignment() const { return m_Alignment; }
2904  VkDeviceSize GetSize() const { return m_Size; }
2905  void* GetUserData() const { return m_pUserData; }
2906  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2907  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2908 
2909  VmaDeviceMemoryBlock* GetBlock() const
2910  {
2911  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2912  return m_BlockAllocation.m_Block;
2913  }
2914  VkDeviceSize GetOffset() const;
2915  VkDeviceMemory GetMemory() const;
2916  uint32_t GetMemoryTypeIndex() const;
2917  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2918  void* GetMappedData() const;
2919  bool CanBecomeLost() const;
2920  VmaPool GetPool() const;
2921 
2922  VkResult OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator);
2923  void OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator);
2924 
2925  uint32_t GetLastUseFrameIndex() const
2926  {
2927  return m_LastUseFrameIndex.load();
2928  }
2929  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
2930  {
2931  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
2932  }
2933  /*
2934  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
2935  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
2936  - Else, returns false.
2937 
2938  If hAllocation is already lost, assert - you should not call it then.
2939  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
2940  */
2941  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
2942 
2943  void OwnAllocCalcStatsInfo(VmaStatInfo& outInfo)
2944  {
2945  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2946  outInfo.blockCount = 1;
2947  outInfo.allocationCount = 1;
2948  outInfo.unusedRangeCount = 0;
2949  outInfo.usedBytes = m_Size;
2950  outInfo.unusedBytes = 0;
2951  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
2952  outInfo.unusedRangeSizeMin = UINT64_MAX;
2953  outInfo.unusedRangeSizeMax = 0;
2954  }
2955 
2956 private:
2957  VkDeviceSize m_Alignment;
2958  VkDeviceSize m_Size;
2959  void* m_pUserData;
2960  ALLOCATION_TYPE m_Type;
2961  VmaSuballocationType m_SuballocationType;
2962  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
2963 
2964  // Allocation out of VmaDeviceMemoryBlock.
2965  struct BlockAllocation
2966  {
2967  VmaPool m_hPool; // Null if belongs to general memory.
2968  VmaDeviceMemoryBlock* m_Block;
2969  VkDeviceSize m_Offset;
2970  bool m_CanBecomeLost;
2971  };
2972 
2973  // Allocation for an object that has its own private VkDeviceMemory.
2974  struct OwnAllocation
2975  {
2976  uint32_t m_MemoryTypeIndex;
2977  VkDeviceMemory m_hMemory;
2978  bool m_PersistentMap;
2979  void* m_pMappedData;
2980  };
2981 
2982  union
2983  {
2984  // Allocation out of VmaDeviceMemoryBlock.
2985  BlockAllocation m_BlockAllocation;
2986  // Allocation for an object that has its own private VkDeviceMemory.
2987  OwnAllocation m_OwnAllocation;
2988  };
2989 };
2990 
2991 /*
2992 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
2993 allocated memory block or free.
2994 */
2995 struct VmaSuballocation
2996 {
2997  VkDeviceSize offset;
2998  VkDeviceSize size;
2999  VmaAllocation hAllocation;
3000  VmaSuballocationType type;
3001 };
3002 
3003 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3004 
3005 // Cost of one additional allocation lost, as equivalent in bytes.
3006 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3007 
3008 /*
3009 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3010 
3011 If canMakeOtherLost was false:
3012 - item points to a FREE suballocation.
3013 - itemsToMakeLostCount is 0.
3014 
3015 If canMakeOtherLost was true:
3016 - item points to first of sequence of suballocations, which are either FREE,
3017  or point to VmaAllocations that can become lost.
3018 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3019  the requested allocation to succeed.
3020 */
3021 struct VmaAllocationRequest
3022 {
3023  VkDeviceSize offset;
3024  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3025  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3026  VmaSuballocationList::iterator item;
3027  size_t itemsToMakeLostCount;
3028 
3029  VkDeviceSize CalcCost() const
3030  {
3031  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3032  }
3033 };
3034 
3035 /*
3036 Data structure used for bookkeeping of allocations and unused ranges of memory
3037 in a single VkDeviceMemory block.
3038 */
3039 class VmaBlockMetadata
3040 {
3041 public:
3042  VmaBlockMetadata(VmaAllocator hAllocator);
3043  ~VmaBlockMetadata();
3044  void Init(VkDeviceSize size);
3045 
3046  // Validates all data structures inside this object. If not valid, returns false.
3047  bool Validate() const;
3048  VkDeviceSize GetSize() const { return m_Size; }
3049  size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
3050  VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3051  VkDeviceSize GetUnusedRangeSizeMax() const;
3052  // Returns true if this block is empty - contains only single free suballocation.
3053  bool IsEmpty() const;
3054 
3055  void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3056  void AddPoolStats(VmaPoolStats& inoutStats) const;
3057 
3058 #if VMA_STATS_STRING_ENABLED
3059  void PrintDetailedMap(class VmaJsonWriter& json) const;
3060 #endif
3061 
3062  // Creates trivial request for case when block is empty.
3063  void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3064 
3065  // Tries to find a place for suballocation with given parameters inside this block.
3066  // If succeeded, fills pAllocationRequest and returns true.
3067  // If failed, returns false.
3068  bool CreateAllocationRequest(
3069  uint32_t currentFrameIndex,
3070  uint32_t frameInUseCount,
3071  VkDeviceSize bufferImageGranularity,
3072  VkDeviceSize allocSize,
3073  VkDeviceSize allocAlignment,
3074  VmaSuballocationType allocType,
3075  bool canMakeOtherLost,
3076  VmaAllocationRequest* pAllocationRequest);
3077 
3078  bool MakeRequestedAllocationsLost(
3079  uint32_t currentFrameIndex,
3080  uint32_t frameInUseCount,
3081  VmaAllocationRequest* pAllocationRequest);
3082 
3083  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3084 
3085  // Makes actual allocation based on request. Request must already be checked and valid.
3086  void Alloc(
3087  const VmaAllocationRequest& request,
3088  VmaSuballocationType type,
3089  VkDeviceSize allocSize,
3090  VmaAllocation hAllocation);
3091 
3092  // Frees suballocation assigned to given memory region.
3093  void Free(const VmaAllocation allocation);
3094 
3095 private:
3096  VkDeviceSize m_Size;
3097  uint32_t m_FreeCount;
3098  VkDeviceSize m_SumFreeSize;
3099  VmaSuballocationList m_Suballocations;
3100  // Suballocations that are free and have size greater than certain threshold.
3101  // Sorted by size, ascending.
3102  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
3103 
3104  bool ValidateFreeSuballocationList() const;
3105 
3106  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
3107  // If yes, fills pOffset and returns true. If no, returns false.
3108  bool CheckAllocation(
3109  uint32_t currentFrameIndex,
3110  uint32_t frameInUseCount,
3111  VkDeviceSize bufferImageGranularity,
3112  VkDeviceSize allocSize,
3113  VkDeviceSize allocAlignment,
3114  VmaSuballocationType allocType,
3115  VmaSuballocationList::const_iterator suballocItem,
3116  bool canMakeOtherLost,
3117  VkDeviceSize* pOffset,
3118  size_t* itemsToMakeLostCount,
3119  VkDeviceSize* pSumFreeSize,
3120  VkDeviceSize* pSumItemSize) const;
3121  // Given free suballocation, it merges it with following one, which must also be free.
3122  void MergeFreeWithNext(VmaSuballocationList::iterator item);
3123  // Releases given suballocation, making it free.
3124  // Merges it with adjacent free suballocations if applicable.
3125  // Returns iterator to new free suballocation at this place.
3126  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
3127  // Given free suballocation, it inserts it into sorted list of
3128  // m_FreeSuballocationsBySize if it's suitable.
3129  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
3130  // Given free suballocation, it removes it from sorted list of
3131  // m_FreeSuballocationsBySize if it's suitable.
3132  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
3133 };
3134 
3135 /*
3136 Represents a single block of device memory (`VkDeviceMemory`) with all the
3137 data about its regions (aka suballocations, `VmaAllocation`), assigned and free.
3138 
3139 Thread-safety: This class must be externally synchronized.
3140 */
3141 class VmaDeviceMemoryBlock
3142 {
3143 public:
3144  uint32_t m_MemoryTypeIndex;
3145  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3146  VkDeviceMemory m_hMemory;
3147  bool m_PersistentMap;
3148  void* m_pMappedData;
3149  VmaBlockMetadata m_Metadata;
3150 
3151  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
3152 
3153  ~VmaDeviceMemoryBlock()
3154  {
3155  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
3156  }
3157 
3158  // Always call after construction.
3159  void Init(
3160  uint32_t newMemoryTypeIndex,
3161  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
3162  VkDeviceMemory newMemory,
3163  VkDeviceSize newSize,
3164  bool persistentMap,
3165  void* pMappedData);
3166  // Always call before destruction.
3167  void Destroy(VmaAllocator allocator);
3168 
3169  // Validates all data structures inside this object. If not valid, returns false.
3170  bool Validate() const;
3171 };
3172 
3173 struct VmaPointerLess
3174 {
3175  bool operator()(const void* lhs, const void* rhs) const
3176  {
3177  return lhs < rhs;
3178  }
3179 };
3180 
3181 class VmaDefragmentator;
3182 
3183 /*
3184 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
3185 Vulkan memory type.
3186 
3187 Synchronized internally with a mutex.
3188 */
3189 struct VmaBlockVector
3190 {
3191  VmaBlockVector(
3192  VmaAllocator hAllocator,
3193  uint32_t memoryTypeIndex,
3194  VMA_BLOCK_VECTOR_TYPE blockVectorType,
3195  VkDeviceSize preferredBlockSize,
3196  size_t minBlockCount,
3197  size_t maxBlockCount,
3198  VkDeviceSize bufferImageGranularity,
3199  uint32_t frameInUseCount,
3200  bool isCustomPool);
3201  ~VmaBlockVector();
3202 
3203  VkResult CreateMinBlocks();
3204 
3205  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
3206  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
3207  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
3208  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
3209  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; }
3210 
3211  void GetPoolStats(VmaPoolStats* pStats);
3212 
3213  bool IsEmpty() const { return m_Blocks.empty(); }
3214 
3215  VkResult Allocate(
3216  VmaPool hCurrentPool,
3217  uint32_t currentFrameIndex,
3218  const VkMemoryRequirements& vkMemReq,
3219  const VmaAllocationCreateInfo& createInfo,
3220  VmaSuballocationType suballocType,
3221  VmaAllocation* pAllocation);
3222 
3223  void Free(
3224  VmaAllocation hAllocation);
3225 
3226  // Adds statistics of this BlockVector to pStats.
3227  void AddStats(VmaStats* pStats);
3228 
3229 #if VMA_STATS_STRING_ENABLED
3230  void PrintDetailedMap(class VmaJsonWriter& json);
3231 #endif
3232 
3233  void UnmapPersistentlyMappedMemory();
3234  VkResult MapPersistentlyMappedMemory();
3235 
3236  void MakePoolAllocationsLost(
3237  uint32_t currentFrameIndex,
3238  size_t* pLostAllocationCount);
3239 
3240  VmaDefragmentator* EnsureDefragmentator(
3241  VmaAllocator hAllocator,
3242  uint32_t currentFrameIndex);
3243 
3244  VkResult Defragment(
3245  VmaDefragmentationStats* pDefragmentationStats,
3246  VkDeviceSize& maxBytesToMove,
3247  uint32_t& maxAllocationsToMove);
3248 
3249  void DestroyDefragmentator();
3250 
3251 private:
3252  friend class VmaDefragmentator;
3253 
3254  const VmaAllocator m_hAllocator;
3255  const uint32_t m_MemoryTypeIndex;
3256  const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3257  const VkDeviceSize m_PreferredBlockSize;
3258  const size_t m_MinBlockCount;
3259  const size_t m_MaxBlockCount;
3260  const VkDeviceSize m_BufferImageGranularity;
3261  const uint32_t m_FrameInUseCount;
3262  const bool m_IsCustomPool;
3263  VMA_MUTEX m_Mutex;
3264  // Incrementally sorted by sumFreeSize, ascending.
3265  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
3266  /* There can be at most one allocation that is completely empty - a
3267  hysteresis to avoid pessimistic case of alternating creation and destruction
3268  of a VkDeviceMemory. */
3269  bool m_HasEmptyBlock;
3270  VmaDefragmentator* m_pDefragmentator;
3271 
3272  // Finds and removes given block from vector.
3273  void Remove(VmaDeviceMemoryBlock* pBlock);
3274 
3275  // Performs single step in sorting m_Blocks. They may not be fully sorted
3276  // after this call.
3277  void IncrementallySortBlocks();
3278 
3279  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
3280 };
3281 
3282 struct VmaPool_T
3283 {
3284 public:
3285  VmaBlockVector m_BlockVector;
3286 
3287  // Takes ownership.
3288  VmaPool_T(
3289  VmaAllocator hAllocator,
3290  const VmaPoolCreateInfo& createInfo);
3291  ~VmaPool_T();
3292 
3293  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
3294 
3295 #if VMA_STATS_STRING_ENABLED
3296  //void PrintDetailedMap(class VmaStringBuilder& sb);
3297 #endif
3298 };
3299 
3300 class VmaDefragmentator
3301 {
3302  const VmaAllocator m_hAllocator;
3303  VmaBlockVector* const m_pBlockVector;
3304  uint32_t m_CurrentFrameIndex;
3305  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3306  VkDeviceSize m_BytesMoved;
3307  uint32_t m_AllocationsMoved;
3308 
3309  struct AllocationInfo
3310  {
3311  VmaAllocation m_hAllocation;
3312  VkBool32* m_pChanged;
3313 
3314  AllocationInfo() :
3315  m_hAllocation(VK_NULL_HANDLE),
3316  m_pChanged(VMA_NULL)
3317  {
3318  }
3319  };
3320 
3321  struct AllocationInfoSizeGreater
3322  {
3323  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3324  {
3325  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3326  }
3327  };
3328 
3329  // Used between AddAllocation and Defragment.
3330  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3331 
3332  struct BlockInfo
3333  {
3334  VmaDeviceMemoryBlock* m_pBlock;
3335  bool m_HasNonMovableAllocations;
3336  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3337 
3338  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3339  m_pBlock(VMA_NULL),
3340  m_HasNonMovableAllocations(true),
3341  m_Allocations(pAllocationCallbacks),
3342  m_pMappedDataForDefragmentation(VMA_NULL)
3343  {
3344  }
3345 
3346  void CalcHasNonMovableAllocations()
3347  {
3348  const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
3349  const size_t defragmentAllocCount = m_Allocations.size();
3350  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3351  }
3352 
3353  void SortAllocationsBySizeDescecnding()
3354  {
3355  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3356  }
3357 
3358  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
3359  void Unmap(VmaAllocator hAllocator);
3360 
3361  private:
3362  // Not null if mapped for defragmentation only, not persistently mapped.
3363  void* m_pMappedDataForDefragmentation;
3364  };
3365 
3366  struct BlockPointerLess
3367  {
3368  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
3369  {
3370  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3371  }
3372  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3373  {
3374  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3375  }
3376  };
3377 
3378  // 1. Blocks with some non-movable allocations go first.
3379  // 2. Blocks with smaller sumFreeSize go first.
3380  struct BlockInfoCompareMoveDestination
3381  {
3382  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3383  {
3384  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3385  {
3386  return true;
3387  }
3388  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3389  {
3390  return false;
3391  }
3392  if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
3393  {
3394  return true;
3395  }
3396  return false;
3397  }
3398  };
3399 
3400  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3401  BlockInfoVector m_Blocks;
3402 
3403  VkResult DefragmentRound(
3404  VkDeviceSize maxBytesToMove,
3405  uint32_t maxAllocationsToMove);
3406 
3407  static bool MoveMakesSense(
3408  size_t dstBlockIndex, VkDeviceSize dstOffset,
3409  size_t srcBlockIndex, VkDeviceSize srcOffset);
3410 
3411 public:
3412  VmaDefragmentator(
3413  VmaAllocator hAllocator,
3414  VmaBlockVector* pBlockVector,
3415  uint32_t currentFrameIndex);
3416 
3417  ~VmaDefragmentator();
3418 
3419  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3420  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3421 
3422  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3423 
3424  VkResult Defragment(
3425  VkDeviceSize maxBytesToMove,
3426  uint32_t maxAllocationsToMove);
3427 };
3428 
3429 // Main allocator object.
3430 struct VmaAllocator_T
3431 {
3432  bool m_UseMutex;
3433  VkDevice m_hDevice;
3434  bool m_AllocationCallbacksSpecified;
3435  VkAllocationCallbacks m_AllocationCallbacks;
3436  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
3437  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
3438  // Counter to allow nested calls to these functions.
3439  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
3440 
3441  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
3442  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
3443  VMA_MUTEX m_HeapSizeLimitMutex;
3444 
3445  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
3446  VkPhysicalDeviceMemoryProperties m_MemProps;
3447 
3448  // Default pools.
3449  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3450 
3451  // Each vector is sorted by memory (handle value).
3452  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
3453  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
3454  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
3455 
3456  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
3457  ~VmaAllocator_T();
3458 
3459  const VkAllocationCallbacks* GetAllocationCallbacks() const
3460  {
3461  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
3462  }
3463  const VmaVulkanFunctions& GetVulkanFunctions() const
3464  {
3465  return m_VulkanFunctions;
3466  }
3467 
3468  VkDeviceSize GetBufferImageGranularity() const
3469  {
3470  return VMA_MAX(
3471  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
3472  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
3473  }
3474 
3475  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
3476  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
3477 
3478  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
3479  {
3480  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
3481  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3482  }
3483 
3484  // Main allocation function.
3485  VkResult AllocateMemory(
3486  const VkMemoryRequirements& vkMemReq,
3487  const VmaAllocationCreateInfo& createInfo,
3488  VmaSuballocationType suballocType,
3489  VmaAllocation* pAllocation);
3490 
3491  // Main deallocation function.
3492  void FreeMemory(const VmaAllocation allocation);
3493 
3494  void CalculateStats(VmaStats* pStats);
3495 
3496 #if VMA_STATS_STRING_ENABLED
3497  void PrintDetailedMap(class VmaJsonWriter& json);
3498 #endif
3499 
3500  void UnmapPersistentlyMappedMemory();
3501  VkResult MapPersistentlyMappedMemory();
3502 
3503  VkResult Defragment(
3504  VmaAllocation* pAllocations,
3505  size_t allocationCount,
3506  VkBool32* pAllocationsChanged,
3507  const VmaDefragmentationInfo* pDefragmentationInfo,
3508  VmaDefragmentationStats* pDefragmentationStats);
3509 
3510  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
3511 
3512  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
3513  void DestroyPool(VmaPool pool);
3514  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
3515 
3516  void SetCurrentFrameIndex(uint32_t frameIndex);
3517 
3518  void MakePoolAllocationsLost(
3519  VmaPool hPool,
3520  size_t* pLostAllocationCount);
3521 
3522  void CreateLostAllocation(VmaAllocation* pAllocation);
3523 
3524  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
3525  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
3526 
3527 private:
3528  VkDeviceSize m_PreferredLargeHeapBlockSize;
3529  VkDeviceSize m_PreferredSmallHeapBlockSize;
3530 
3531  VkPhysicalDevice m_PhysicalDevice;
3532  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
3533 
3534  VMA_MUTEX m_PoolsMutex;
3535  // Protected by m_PoolsMutex. Sorted by pointer value.
3536  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
3537 
3538  VmaVulkanFunctions m_VulkanFunctions;
3539 
3540  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
3541 
3542  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
3543 
3544  VkResult AllocateMemoryOfType(
3545  const VkMemoryRequirements& vkMemReq,
3546  const VmaAllocationCreateInfo& createInfo,
3547  uint32_t memTypeIndex,
3548  VmaSuballocationType suballocType,
3549  VmaAllocation* pAllocation);
3550 
3551  // Allocates and registers new VkDeviceMemory specifically for single allocation.
3552  VkResult AllocateOwnMemory(
3553  VkDeviceSize size,
3554  VmaSuballocationType suballocType,
3555  uint32_t memTypeIndex,
3556  bool map,
3557  void* pUserData,
3558  VmaAllocation* pAllocation);
3559 
3560  // Tries to free pMemory as Own Memory. Returns true if found and freed.
3561  void FreeOwnMemory(VmaAllocation allocation);
3562 };
3563 
3565 // Memory allocation #2 after VmaAllocator_T definition
3566 
3567 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
3568 {
3569  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
3570 }
3571 
3572 static void VmaFree(VmaAllocator hAllocator, void* ptr)
3573 {
3574  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
3575 }
3576 
3577 template<typename T>
3578 static T* VmaAllocate(VmaAllocator hAllocator)
3579 {
3580  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
3581 }
3582 
3583 template<typename T>
3584 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
3585 {
3586  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
3587 }
3588 
3589 template<typename T>
3590 static void vma_delete(VmaAllocator hAllocator, T* ptr)
3591 {
3592  if(ptr != VMA_NULL)
3593  {
3594  ptr->~T();
3595  VmaFree(hAllocator, ptr);
3596  }
3597 }
3598 
3599 template<typename T>
3600 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
3601 {
3602  if(ptr != VMA_NULL)
3603  {
3604  for(size_t i = count; i--; )
3605  ptr[i].~T();
3606  VmaFree(hAllocator, ptr);
3607  }
3608 }
3609 
3611 // VmaStringBuilder
3612 
3613 #if VMA_STATS_STRING_ENABLED
3614 
3615 class VmaStringBuilder
3616 {
3617 public:
3618  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
3619  size_t GetLength() const { return m_Data.size(); }
3620  const char* GetData() const { return m_Data.data(); }
3621 
3622  void Add(char ch) { m_Data.push_back(ch); }
3623  void Add(const char* pStr);
3624  void AddNewLine() { Add('\n'); }
3625  void AddNumber(uint32_t num);
3626  void AddNumber(uint64_t num);
3627  void AddPointer(const void* ptr);
3628 
3629 private:
3630  VmaVector< char, VmaStlAllocator<char> > m_Data;
3631 };
3632 
3633 void VmaStringBuilder::Add(const char* pStr)
3634 {
3635  const size_t strLen = strlen(pStr);
3636  if(strLen > 0)
3637  {
3638  const size_t oldCount = m_Data.size();
3639  m_Data.resize(oldCount + strLen);
3640  memcpy(m_Data.data() + oldCount, pStr, strLen);
3641  }
3642 }
3643 
3644 void VmaStringBuilder::AddNumber(uint32_t num)
3645 {
3646  char buf[11];
3647  VmaUint32ToStr(buf, sizeof(buf), num);
3648  Add(buf);
3649 }
3650 
3651 void VmaStringBuilder::AddNumber(uint64_t num)
3652 {
3653  char buf[21];
3654  VmaUint64ToStr(buf, sizeof(buf), num);
3655  Add(buf);
3656 }
3657 
3658 void VmaStringBuilder::AddPointer(const void* ptr)
3659 {
3660  char buf[21];
3661  VmaPtrToStr(buf, sizeof(buf), ptr);
3662  Add(buf);
3663 }
3664 
3665 #endif // #if VMA_STATS_STRING_ENABLED
3666 
3668 // VmaJsonWriter
3669 
3670 #if VMA_STATS_STRING_ENABLED
3671 
3672 class VmaJsonWriter
3673 {
3674 public:
3675  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
3676  ~VmaJsonWriter();
3677 
3678  void BeginObject(bool singleLine = false);
3679  void EndObject();
3680 
3681  void BeginArray(bool singleLine = false);
3682  void EndArray();
3683 
3684  void WriteString(const char* pStr);
3685  void BeginString(const char* pStr = VMA_NULL);
3686  void ContinueString(const char* pStr);
3687  void ContinueString(uint32_t n);
3688  void ContinueString(uint64_t n);
3689  void EndString(const char* pStr = VMA_NULL);
3690 
3691  void WriteNumber(uint32_t n);
3692  void WriteNumber(uint64_t n);
3693  void WriteBool(bool b);
3694  void WriteNull();
3695 
3696 private:
3697  static const char* const INDENT;
3698 
3699  enum COLLECTION_TYPE
3700  {
3701  COLLECTION_TYPE_OBJECT,
3702  COLLECTION_TYPE_ARRAY,
3703  };
3704  struct StackItem
3705  {
3706  COLLECTION_TYPE type;
3707  uint32_t valueCount;
3708  bool singleLineMode;
3709  };
3710 
3711  VmaStringBuilder& m_SB;
3712  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
3713  bool m_InsideString;
3714 
3715  void BeginValue(bool isString);
3716  void WriteIndent(bool oneLess = false);
3717 };
3718 
3719 const char* const VmaJsonWriter::INDENT = " ";
3720 
3721 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
3722  m_SB(sb),
3723  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
3724  m_InsideString(false)
3725 {
3726 }
3727 
3728 VmaJsonWriter::~VmaJsonWriter()
3729 {
3730  VMA_ASSERT(!m_InsideString);
3731  VMA_ASSERT(m_Stack.empty());
3732 }
3733 
3734 void VmaJsonWriter::BeginObject(bool singleLine)
3735 {
3736  VMA_ASSERT(!m_InsideString);
3737 
3738  BeginValue(false);
3739  m_SB.Add('{');
3740 
3741  StackItem item;
3742  item.type = COLLECTION_TYPE_OBJECT;
3743  item.valueCount = 0;
3744  item.singleLineMode = singleLine;
3745  m_Stack.push_back(item);
3746 }
3747 
3748 void VmaJsonWriter::EndObject()
3749 {
3750  VMA_ASSERT(!m_InsideString);
3751 
3752  WriteIndent(true);
3753  m_SB.Add('}');
3754 
3755  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
3756  m_Stack.pop_back();
3757 }
3758 
3759 void VmaJsonWriter::BeginArray(bool singleLine)
3760 {
3761  VMA_ASSERT(!m_InsideString);
3762 
3763  BeginValue(false);
3764  m_SB.Add('[');
3765 
3766  StackItem item;
3767  item.type = COLLECTION_TYPE_ARRAY;
3768  item.valueCount = 0;
3769  item.singleLineMode = singleLine;
3770  m_Stack.push_back(item);
3771 }
3772 
3773 void VmaJsonWriter::EndArray()
3774 {
3775  VMA_ASSERT(!m_InsideString);
3776 
3777  WriteIndent(true);
3778  m_SB.Add(']');
3779 
3780  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
3781  m_Stack.pop_back();
3782 }
3783 
3784 void VmaJsonWriter::WriteString(const char* pStr)
3785 {
3786  BeginString(pStr);
3787  EndString();
3788 }
3789 
3790 void VmaJsonWriter::BeginString(const char* pStr)
3791 {
3792  VMA_ASSERT(!m_InsideString);
3793 
3794  BeginValue(true);
3795  m_SB.Add('"');
3796  m_InsideString = true;
3797  if(pStr != VMA_NULL && pStr[0] != '\0')
3798  {
3799  ContinueString(pStr);
3800  }
3801 }
3802 
3803 void VmaJsonWriter::ContinueString(const char* pStr)
3804 {
3805  VMA_ASSERT(m_InsideString);
3806 
3807  const size_t strLen = strlen(pStr);
3808  for(size_t i = 0; i < strLen; ++i)
3809  {
3810  char ch = pStr[i];
3811  if(ch == '\'')
3812  {
3813  m_SB.Add("\\\\");
3814  }
3815  else if(ch == '"')
3816  {
3817  m_SB.Add("\\\"");
3818  }
3819  else if(ch >= 32)
3820  {
3821  m_SB.Add(ch);
3822  }
3823  else switch(ch)
3824  {
3825  case '\n':
3826  m_SB.Add("\\n");
3827  break;
3828  case '\r':
3829  m_SB.Add("\\r");
3830  break;
3831  case '\t':
3832  m_SB.Add("\\t");
3833  break;
3834  default:
3835  VMA_ASSERT(0 && "Character not currently supported.");
3836  break;
3837  }
3838  }
3839 }
3840 
3841 void VmaJsonWriter::ContinueString(uint32_t n)
3842 {
3843  VMA_ASSERT(m_InsideString);
3844  m_SB.AddNumber(n);
3845 }
3846 
3847 void VmaJsonWriter::ContinueString(uint64_t n)
3848 {
3849  VMA_ASSERT(m_InsideString);
3850  m_SB.AddNumber(n);
3851 }
3852 
3853 void VmaJsonWriter::EndString(const char* pStr)
3854 {
3855  VMA_ASSERT(m_InsideString);
3856  if(pStr != VMA_NULL && pStr[0] != '\0')
3857  {
3858  ContinueString(pStr);
3859  }
3860  m_SB.Add('"');
3861  m_InsideString = false;
3862 }
3863 
3864 void VmaJsonWriter::WriteNumber(uint32_t n)
3865 {
3866  VMA_ASSERT(!m_InsideString);
3867  BeginValue(false);
3868  m_SB.AddNumber(n);
3869 }
3870 
3871 void VmaJsonWriter::WriteNumber(uint64_t n)
3872 {
3873  VMA_ASSERT(!m_InsideString);
3874  BeginValue(false);
3875  m_SB.AddNumber(n);
3876 }
3877 
3878 void VmaJsonWriter::WriteBool(bool b)
3879 {
3880  VMA_ASSERT(!m_InsideString);
3881  BeginValue(false);
3882  m_SB.Add(b ? "true" : "false");
3883 }
3884 
3885 void VmaJsonWriter::WriteNull()
3886 {
3887  VMA_ASSERT(!m_InsideString);
3888  BeginValue(false);
3889  m_SB.Add("null");
3890 }
3891 
3892 void VmaJsonWriter::BeginValue(bool isString)
3893 {
3894  if(!m_Stack.empty())
3895  {
3896  StackItem& currItem = m_Stack.back();
3897  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3898  currItem.valueCount % 2 == 0)
3899  {
3900  VMA_ASSERT(isString);
3901  }
3902 
3903  if(currItem.type == COLLECTION_TYPE_OBJECT &&
3904  currItem.valueCount % 2 != 0)
3905  {
3906  m_SB.Add(": ");
3907  }
3908  else if(currItem.valueCount > 0)
3909  {
3910  m_SB.Add(", ");
3911  WriteIndent();
3912  }
3913  else
3914  {
3915  WriteIndent();
3916  }
3917  ++currItem.valueCount;
3918  }
3919 }
3920 
3921 void VmaJsonWriter::WriteIndent(bool oneLess)
3922 {
3923  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
3924  {
3925  m_SB.AddNewLine();
3926 
3927  size_t count = m_Stack.size();
3928  if(count > 0 && oneLess)
3929  {
3930  --count;
3931  }
3932  for(size_t i = 0; i < count; ++i)
3933  {
3934  m_SB.Add(INDENT);
3935  }
3936  }
3937 }
3938 
3939 #endif // #if VMA_STATS_STRING_ENABLED
3940 
3942 
3943 VkDeviceSize VmaAllocation_T::GetOffset() const
3944 {
3945  switch(m_Type)
3946  {
3947  case ALLOCATION_TYPE_BLOCK:
3948  return m_BlockAllocation.m_Offset;
3949  case ALLOCATION_TYPE_OWN:
3950  return 0;
3951  default:
3952  VMA_ASSERT(0);
3953  return 0;
3954  }
3955 }
3956 
3957 VkDeviceMemory VmaAllocation_T::GetMemory() const
3958 {
3959  switch(m_Type)
3960  {
3961  case ALLOCATION_TYPE_BLOCK:
3962  return m_BlockAllocation.m_Block->m_hMemory;
3963  case ALLOCATION_TYPE_OWN:
3964  return m_OwnAllocation.m_hMemory;
3965  default:
3966  VMA_ASSERT(0);
3967  return VK_NULL_HANDLE;
3968  }
3969 }
3970 
3971 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
3972 {
3973  switch(m_Type)
3974  {
3975  case ALLOCATION_TYPE_BLOCK:
3976  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
3977  case ALLOCATION_TYPE_OWN:
3978  return m_OwnAllocation.m_MemoryTypeIndex;
3979  default:
3980  VMA_ASSERT(0);
3981  return UINT32_MAX;
3982  }
3983 }
3984 
3985 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
3986 {
3987  switch(m_Type)
3988  {
3989  case ALLOCATION_TYPE_BLOCK:
3990  return m_BlockAllocation.m_Block->m_BlockVectorType;
3991  case ALLOCATION_TYPE_OWN:
3992  return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
3993  default:
3994  VMA_ASSERT(0);
3995  return VMA_BLOCK_VECTOR_TYPE_COUNT;
3996  }
3997 }
3998 
3999 void* VmaAllocation_T::GetMappedData() const
4000 {
4001  switch(m_Type)
4002  {
4003  case ALLOCATION_TYPE_BLOCK:
4004  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
4005  {
4006  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
4007  }
4008  else
4009  {
4010  return VMA_NULL;
4011  }
4012  break;
4013  case ALLOCATION_TYPE_OWN:
4014  return m_OwnAllocation.m_pMappedData;
4015  default:
4016  VMA_ASSERT(0);
4017  return VMA_NULL;
4018  }
4019 }
4020 
4021 bool VmaAllocation_T::CanBecomeLost() const
4022 {
4023  switch(m_Type)
4024  {
4025  case ALLOCATION_TYPE_BLOCK:
4026  return m_BlockAllocation.m_CanBecomeLost;
4027  case ALLOCATION_TYPE_OWN:
4028  return false;
4029  default:
4030  VMA_ASSERT(0);
4031  return false;
4032  }
4033 }
4034 
4035 VmaPool VmaAllocation_T::GetPool() const
4036 {
4037  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4038  return m_BlockAllocation.m_hPool;
4039 }
4040 
4041 VkResult VmaAllocation_T::OwnAllocMapPersistentlyMappedMemory(VmaAllocator hAllocator)
4042 {
4043  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4044  if(m_OwnAllocation.m_PersistentMap)
4045  {
4046  return (*hAllocator->GetVulkanFunctions().vkMapMemory)(
4047  hAllocator->m_hDevice,
4048  m_OwnAllocation.m_hMemory,
4049  0,
4050  VK_WHOLE_SIZE,
4051  0,
4052  &m_OwnAllocation.m_pMappedData);
4053  }
4054  return VK_SUCCESS;
4055 }
4056 void VmaAllocation_T::OwnAllocUnmapPersistentlyMappedMemory(VmaAllocator hAllocator)
4057 {
4058  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
4059  if(m_OwnAllocation.m_pMappedData)
4060  {
4061  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
4062  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_OwnAllocation.m_hMemory);
4063  m_OwnAllocation.m_pMappedData = VMA_NULL;
4064  }
4065 }
4066 
4067 
4068 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4069 {
4070  VMA_ASSERT(CanBecomeLost());
4071 
4072  /*
4073  Warning: This is a carefully designed algorithm.
4074  Do not modify unless you really know what you're doing :)
4075  */
4076  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
4077  for(;;)
4078  {
4079  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
4080  {
4081  VMA_ASSERT(0);
4082  return false;
4083  }
4084  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
4085  {
4086  return false;
4087  }
4088  else // Last use time earlier than current time.
4089  {
4090  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
4091  {
4092  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
4093  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
4094  return true;
4095  }
4096  }
4097  }
4098 }
4099 
4100 #if VMA_STATS_STRING_ENABLED
4101 
4102 // Correspond to values of enum VmaSuballocationType.
4103 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
4104  "FREE",
4105  "UNKNOWN",
4106  "BUFFER",
4107  "IMAGE_UNKNOWN",
4108  "IMAGE_LINEAR",
4109  "IMAGE_OPTIMAL",
4110 };
4111 
4112 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
4113 {
4114  json.BeginObject();
4115 
4116  json.WriteString("Blocks");
4117  json.WriteNumber(stat.blockCount);
4118 
4119  json.WriteString("Allocations");
4120  json.WriteNumber(stat.allocationCount);
4121 
4122  json.WriteString("UnusedRanges");
4123  json.WriteNumber(stat.unusedRangeCount);
4124 
4125  json.WriteString("UsedBytes");
4126  json.WriteNumber(stat.usedBytes);
4127 
4128  json.WriteString("UnusedBytes");
4129  json.WriteNumber(stat.unusedBytes);
4130 
4131  if(stat.allocationCount > 1)
4132  {
4133  json.WriteString("AllocationSize");
4134  json.BeginObject(true);
4135  json.WriteString("Min");
4136  json.WriteNumber(stat.allocationSizeMin);
4137  json.WriteString("Avg");
4138  json.WriteNumber(stat.allocationSizeAvg);
4139  json.WriteString("Max");
4140  json.WriteNumber(stat.allocationSizeMax);
4141  json.EndObject();
4142  }
4143 
4144  if(stat.unusedRangeCount > 1)
4145  {
4146  json.WriteString("UnusedRangeSize");
4147  json.BeginObject(true);
4148  json.WriteString("Min");
4149  json.WriteNumber(stat.unusedRangeSizeMin);
4150  json.WriteString("Avg");
4151  json.WriteNumber(stat.unusedRangeSizeAvg);
4152  json.WriteString("Max");
4153  json.WriteNumber(stat.unusedRangeSizeMax);
4154  json.EndObject();
4155  }
4156 
4157  json.EndObject();
4158 }
4159 
4160 #endif // #if VMA_STATS_STRING_ENABLED
4161 
4162 struct VmaSuballocationItemSizeLess
4163 {
4164  bool operator()(
4165  const VmaSuballocationList::iterator lhs,
4166  const VmaSuballocationList::iterator rhs) const
4167  {
4168  return lhs->size < rhs->size;
4169  }
4170  bool operator()(
4171  const VmaSuballocationList::iterator lhs,
4172  VkDeviceSize rhsSize) const
4173  {
4174  return lhs->size < rhsSize;
4175  }
4176 };
4177 
4179 // class VmaBlockMetadata
4180 
4181 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
4182  m_Size(0),
4183  m_FreeCount(0),
4184  m_SumFreeSize(0),
4185  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
4186  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
4187 {
4188 }
4189 
4190 VmaBlockMetadata::~VmaBlockMetadata()
4191 {
4192 }
4193 
4194 void VmaBlockMetadata::Init(VkDeviceSize size)
4195 {
4196  m_Size = size;
4197  m_FreeCount = 1;
4198  m_SumFreeSize = size;
4199 
4200  VmaSuballocation suballoc = {};
4201  suballoc.offset = 0;
4202  suballoc.size = size;
4203  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4204  suballoc.hAllocation = VK_NULL_HANDLE;
4205 
4206  m_Suballocations.push_back(suballoc);
4207  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
4208  --suballocItem;
4209  m_FreeSuballocationsBySize.push_back(suballocItem);
4210 }
4211 
4212 bool VmaBlockMetadata::Validate() const
4213 {
4214  if(m_Suballocations.empty())
4215  {
4216  return false;
4217  }
4218 
4219  // Expected offset of new suballocation as calculates from previous ones.
4220  VkDeviceSize calculatedOffset = 0;
4221  // Expected number of free suballocations as calculated from traversing their list.
4222  uint32_t calculatedFreeCount = 0;
4223  // Expected sum size of free suballocations as calculated from traversing their list.
4224  VkDeviceSize calculatedSumFreeSize = 0;
4225  // Expected number of free suballocations that should be registered in
4226  // m_FreeSuballocationsBySize calculated from traversing their list.
4227  size_t freeSuballocationsToRegister = 0;
4228  // True if previous visisted suballocation was free.
4229  bool prevFree = false;
4230 
4231  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4232  suballocItem != m_Suballocations.cend();
4233  ++suballocItem)
4234  {
4235  const VmaSuballocation& subAlloc = *suballocItem;
4236 
4237  // Actual offset of this suballocation doesn't match expected one.
4238  if(subAlloc.offset != calculatedOffset)
4239  {
4240  return false;
4241  }
4242 
4243  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
4244  // Two adjacent free suballocations are invalid. They should be merged.
4245  if(prevFree && currFree)
4246  {
4247  return false;
4248  }
4249  prevFree = currFree;
4250 
4251  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
4252  {
4253  return false;
4254  }
4255 
4256  if(currFree)
4257  {
4258  calculatedSumFreeSize += subAlloc.size;
4259  ++calculatedFreeCount;
4260  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4261  {
4262  ++freeSuballocationsToRegister;
4263  }
4264  }
4265 
4266  calculatedOffset += subAlloc.size;
4267  }
4268 
4269  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
4270  // match expected one.
4271  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
4272  {
4273  return false;
4274  }
4275 
4276  VkDeviceSize lastSize = 0;
4277  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
4278  {
4279  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
4280 
4281  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
4282  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
4283  {
4284  return false;
4285  }
4286  // They must be sorted by size ascending.
4287  if(suballocItem->size < lastSize)
4288  {
4289  return false;
4290  }
4291 
4292  lastSize = suballocItem->size;
4293  }
4294 
4295  // Check if totals match calculacted values.
4296  return
4297  ValidateFreeSuballocationList() &&
4298  (calculatedOffset == m_Size) &&
4299  (calculatedSumFreeSize == m_SumFreeSize) &&
4300  (calculatedFreeCount == m_FreeCount);
4301 }
4302 
4303 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
4304 {
4305  if(!m_FreeSuballocationsBySize.empty())
4306  {
4307  return m_FreeSuballocationsBySize.back()->size;
4308  }
4309  else
4310  {
4311  return 0;
4312  }
4313 }
4314 
4315 bool VmaBlockMetadata::IsEmpty() const
4316 {
4317  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
4318 }
4319 
4320 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
4321 {
4322  outInfo.blockCount = 1;
4323 
4324  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4325  outInfo.allocationCount = rangeCount - m_FreeCount;
4326  outInfo.unusedRangeCount = m_FreeCount;
4327 
4328  outInfo.unusedBytes = m_SumFreeSize;
4329  outInfo.usedBytes = m_Size - outInfo.unusedBytes;
4330 
4331  outInfo.allocationSizeMin = UINT64_MAX;
4332  outInfo.allocationSizeMax = 0;
4333  outInfo.unusedRangeSizeMin = UINT64_MAX;
4334  outInfo.unusedRangeSizeMax = 0;
4335 
4336  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4337  suballocItem != m_Suballocations.cend();
4338  ++suballocItem)
4339  {
4340  const VmaSuballocation& suballoc = *suballocItem;
4341  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
4342  {
4343  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
4344  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
4345  }
4346  else
4347  {
4348  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
4349  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
4350  }
4351  }
4352 }
4353 
4354 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
4355 {
4356  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
4357 
4358  inoutStats.size += m_Size;
4359  inoutStats.unusedSize += m_SumFreeSize;
4360  inoutStats.allocationCount += rangeCount - m_FreeCount;
4361  inoutStats.unusedRangeCount += m_FreeCount;
4362  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
4363 }
4364 
4365 #if VMA_STATS_STRING_ENABLED
4366 
4367 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
4368 {
4369  json.BeginObject();
4370 
4371  json.WriteString("TotalBytes");
4372  json.WriteNumber(m_Size);
4373 
4374  json.WriteString("UnusedBytes");
4375  json.WriteNumber(m_SumFreeSize);
4376 
4377  json.WriteString("Allocations");
4378  json.WriteNumber(m_Suballocations.size() - m_FreeCount);
4379 
4380  json.WriteString("UnusedRanges");
4381  json.WriteNumber(m_FreeCount);
4382 
4383  json.WriteString("Suballocations");
4384  json.BeginArray();
4385  size_t i = 0;
4386  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
4387  suballocItem != m_Suballocations.cend();
4388  ++suballocItem, ++i)
4389  {
4390  json.BeginObject(true);
4391 
4392  json.WriteString("Type");
4393  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
4394 
4395  json.WriteString("Size");
4396  json.WriteNumber(suballocItem->size);
4397 
4398  json.WriteString("Offset");
4399  json.WriteNumber(suballocItem->offset);
4400 
4401  json.EndObject();
4402  }
4403  json.EndArray();
4404 
4405  json.EndObject();
4406 }
4407 
4408 #endif // #if VMA_STATS_STRING_ENABLED
4409 
4410 /*
4411 How many suitable free suballocations to analyze before choosing best one.
4412 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
4413  be chosen.
4414 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
4415  suballocations will be analized and best one will be chosen.
4416 - Any other value is also acceptable.
4417 */
4418 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
4419 
4420 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
4421 {
4422  VMA_ASSERT(IsEmpty());
4423  pAllocationRequest->offset = 0;
4424  pAllocationRequest->sumFreeSize = m_SumFreeSize;
4425  pAllocationRequest->sumItemSize = 0;
4426  pAllocationRequest->item = m_Suballocations.begin();
4427  pAllocationRequest->itemsToMakeLostCount = 0;
4428 }
4429 
4430 bool VmaBlockMetadata::CreateAllocationRequest(
4431  uint32_t currentFrameIndex,
4432  uint32_t frameInUseCount,
4433  VkDeviceSize bufferImageGranularity,
4434  VkDeviceSize allocSize,
4435  VkDeviceSize allocAlignment,
4436  VmaSuballocationType allocType,
4437  bool canMakeOtherLost,
4438  VmaAllocationRequest* pAllocationRequest)
4439 {
4440  VMA_ASSERT(allocSize > 0);
4441  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4442  VMA_ASSERT(pAllocationRequest != VMA_NULL);
4443  VMA_HEAVY_ASSERT(Validate());
4444 
4445  // There is not enough total free space in this block to fullfill the request: Early return.
4446  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
4447  {
4448  return false;
4449  }
4450 
4451  // New algorithm, efficiently searching freeSuballocationsBySize.
4452  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
4453  if(freeSuballocCount > 0)
4454  {
4455  if(VMA_BEST_FIT)
4456  {
4457  // Find first free suballocation with size not less than allocSize.
4458  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
4459  m_FreeSuballocationsBySize.data(),
4460  m_FreeSuballocationsBySize.data() + freeSuballocCount,
4461  allocSize,
4462  VmaSuballocationItemSizeLess());
4463  size_t index = it - m_FreeSuballocationsBySize.data();
4464  for(; index < freeSuballocCount; ++index)
4465  {
4466  if(CheckAllocation(
4467  currentFrameIndex,
4468  frameInUseCount,
4469  bufferImageGranularity,
4470  allocSize,
4471  allocAlignment,
4472  allocType,
4473  m_FreeSuballocationsBySize[index],
4474  false, // canMakeOtherLost
4475  &pAllocationRequest->offset,
4476  &pAllocationRequest->itemsToMakeLostCount,
4477  &pAllocationRequest->sumFreeSize,
4478  &pAllocationRequest->sumItemSize))
4479  {
4480  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4481  return true;
4482  }
4483  }
4484  }
4485  else
4486  {
4487  // Search staring from biggest suballocations.
4488  for(size_t index = freeSuballocCount; index--; )
4489  {
4490  if(CheckAllocation(
4491  currentFrameIndex,
4492  frameInUseCount,
4493  bufferImageGranularity,
4494  allocSize,
4495  allocAlignment,
4496  allocType,
4497  m_FreeSuballocationsBySize[index],
4498  false, // canMakeOtherLost
4499  &pAllocationRequest->offset,
4500  &pAllocationRequest->itemsToMakeLostCount,
4501  &pAllocationRequest->sumFreeSize,
4502  &pAllocationRequest->sumItemSize))
4503  {
4504  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
4505  return true;
4506  }
4507  }
4508  }
4509  }
4510 
4511  if(canMakeOtherLost)
4512  {
4513  // Brute-force algorithm. TODO: Come up with something better.
4514 
4515  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
4516  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
4517 
4518  VmaAllocationRequest tmpAllocRequest = {};
4519  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
4520  suballocIt != m_Suballocations.end();
4521  ++suballocIt)
4522  {
4523  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
4524  suballocIt->hAllocation->CanBecomeLost())
4525  {
4526  if(CheckAllocation(
4527  currentFrameIndex,
4528  frameInUseCount,
4529  bufferImageGranularity,
4530  allocSize,
4531  allocAlignment,
4532  allocType,
4533  suballocIt,
4534  canMakeOtherLost,
4535  &tmpAllocRequest.offset,
4536  &tmpAllocRequest.itemsToMakeLostCount,
4537  &tmpAllocRequest.sumFreeSize,
4538  &tmpAllocRequest.sumItemSize))
4539  {
4540  tmpAllocRequest.item = suballocIt;
4541 
4542  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
4543  {
4544  *pAllocationRequest = tmpAllocRequest;
4545  }
4546  }
4547  }
4548  }
4549 
4550  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
4551  {
4552  return true;
4553  }
4554  }
4555 
4556  return false;
4557 }
4558 
4559 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
4560  uint32_t currentFrameIndex,
4561  uint32_t frameInUseCount,
4562  VmaAllocationRequest* pAllocationRequest)
4563 {
4564  while(pAllocationRequest->itemsToMakeLostCount > 0)
4565  {
4566  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
4567  {
4568  ++pAllocationRequest->item;
4569  }
4570  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4571  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
4572  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
4573  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4574  {
4575  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
4576  --pAllocationRequest->itemsToMakeLostCount;
4577  }
4578  else
4579  {
4580  return false;
4581  }
4582  }
4583 
4584  VMA_HEAVY_ASSERT(Validate());
4585  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
4586  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
4587 
4588  return true;
4589 }
4590 
4591 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
4592 {
4593  uint32_t lostAllocationCount = 0;
4594  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
4595  it != m_Suballocations.end();
4596  ++it)
4597  {
4598  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
4599  it->hAllocation->CanBecomeLost() &&
4600  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
4601  {
4602  it = FreeSuballocation(it);
4603  ++lostAllocationCount;
4604  }
4605  }
4606  return lostAllocationCount;
4607 }
4608 
4609 void VmaBlockMetadata::Alloc(
4610  const VmaAllocationRequest& request,
4611  VmaSuballocationType type,
4612  VkDeviceSize allocSize,
4613  VmaAllocation hAllocation)
4614 {
4615  VMA_ASSERT(request.item != m_Suballocations.end());
4616  VmaSuballocation& suballoc = *request.item;
4617  // Given suballocation is a free block.
4618  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4619  // Given offset is inside this suballocation.
4620  VMA_ASSERT(request.offset >= suballoc.offset);
4621  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
4622  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
4623  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
4624 
4625  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
4626  // it to become used.
4627  UnregisterFreeSuballocation(request.item);
4628 
4629  suballoc.offset = request.offset;
4630  suballoc.size = allocSize;
4631  suballoc.type = type;
4632  suballoc.hAllocation = hAllocation;
4633 
4634  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
4635  if(paddingEnd)
4636  {
4637  VmaSuballocation paddingSuballoc = {};
4638  paddingSuballoc.offset = request.offset + allocSize;
4639  paddingSuballoc.size = paddingEnd;
4640  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4641  VmaSuballocationList::iterator next = request.item;
4642  ++next;
4643  const VmaSuballocationList::iterator paddingEndItem =
4644  m_Suballocations.insert(next, paddingSuballoc);
4645  RegisterFreeSuballocation(paddingEndItem);
4646  }
4647 
4648  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
4649  if(paddingBegin)
4650  {
4651  VmaSuballocation paddingSuballoc = {};
4652  paddingSuballoc.offset = request.offset - paddingBegin;
4653  paddingSuballoc.size = paddingBegin;
4654  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
4655  const VmaSuballocationList::iterator paddingBeginItem =
4656  m_Suballocations.insert(request.item, paddingSuballoc);
4657  RegisterFreeSuballocation(paddingBeginItem);
4658  }
4659 
4660  // Update totals.
4661  m_FreeCount = m_FreeCount - 1;
4662  if(paddingBegin > 0)
4663  {
4664  ++m_FreeCount;
4665  }
4666  if(paddingEnd > 0)
4667  {
4668  ++m_FreeCount;
4669  }
4670  m_SumFreeSize -= allocSize;
4671 }
4672 
4673 void VmaBlockMetadata::Free(const VmaAllocation allocation)
4674 {
4675  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
4676  suballocItem != m_Suballocations.end();
4677  ++suballocItem)
4678  {
4679  VmaSuballocation& suballoc = *suballocItem;
4680  if(suballoc.hAllocation == allocation)
4681  {
4682  FreeSuballocation(suballocItem);
4683  VMA_HEAVY_ASSERT(Validate());
4684  return;
4685  }
4686  }
4687  VMA_ASSERT(0 && "Not found!");
4688 }
4689 
4690 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
4691 {
4692  VkDeviceSize lastSize = 0;
4693  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
4694  {
4695  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
4696 
4697  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
4698  {
4699  VMA_ASSERT(0);
4700  return false;
4701  }
4702  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
4703  {
4704  VMA_ASSERT(0);
4705  return false;
4706  }
4707  if(it->size < lastSize)
4708  {
4709  VMA_ASSERT(0);
4710  return false;
4711  }
4712 
4713  lastSize = it->size;
4714  }
4715  return true;
4716 }
4717 
4718 bool VmaBlockMetadata::CheckAllocation(
4719  uint32_t currentFrameIndex,
4720  uint32_t frameInUseCount,
4721  VkDeviceSize bufferImageGranularity,
4722  VkDeviceSize allocSize,
4723  VkDeviceSize allocAlignment,
4724  VmaSuballocationType allocType,
4725  VmaSuballocationList::const_iterator suballocItem,
4726  bool canMakeOtherLost,
4727  VkDeviceSize* pOffset,
4728  size_t* itemsToMakeLostCount,
4729  VkDeviceSize* pSumFreeSize,
4730  VkDeviceSize* pSumItemSize) const
4731 {
4732  VMA_ASSERT(allocSize > 0);
4733  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
4734  VMA_ASSERT(suballocItem != m_Suballocations.cend());
4735  VMA_ASSERT(pOffset != VMA_NULL);
4736 
4737  *itemsToMakeLostCount = 0;
4738  *pSumFreeSize = 0;
4739  *pSumItemSize = 0;
4740 
4741  if(canMakeOtherLost)
4742  {
4743  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4744  {
4745  *pSumFreeSize = suballocItem->size;
4746  }
4747  else
4748  {
4749  if(suballocItem->hAllocation->CanBecomeLost() &&
4750  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4751  {
4752  ++*itemsToMakeLostCount;
4753  *pSumItemSize = suballocItem->size;
4754  }
4755  else
4756  {
4757  return false;
4758  }
4759  }
4760 
4761  // Remaining size is too small for this request: Early return.
4762  if(m_Size - suballocItem->offset < allocSize)
4763  {
4764  return false;
4765  }
4766 
4767  // Start from offset equal to beginning of this suballocation.
4768  *pOffset = suballocItem->offset;
4769 
4770  // Apply VMA_DEBUG_MARGIN at the beginning.
4771  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4772  {
4773  *pOffset += VMA_DEBUG_MARGIN;
4774  }
4775 
4776  // Apply alignment.
4777  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4778  *pOffset = VmaAlignUp(*pOffset, alignment);
4779 
4780  // Check previous suballocations for BufferImageGranularity conflicts.
4781  // Make bigger alignment if necessary.
4782  if(bufferImageGranularity > 1)
4783  {
4784  bool bufferImageGranularityConflict = false;
4785  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4786  while(prevSuballocItem != m_Suballocations.cbegin())
4787  {
4788  --prevSuballocItem;
4789  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4790  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4791  {
4792  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4793  {
4794  bufferImageGranularityConflict = true;
4795  break;
4796  }
4797  }
4798  else
4799  // Already on previous page.
4800  break;
4801  }
4802  if(bufferImageGranularityConflict)
4803  {
4804  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4805  }
4806  }
4807 
4808  // Now that we have final *pOffset, check if we are past suballocItem.
4809  // If yes, return false - this function should be called for another suballocItem as starting point.
4810  if(*pOffset >= suballocItem->offset + suballocItem->size)
4811  {
4812  return false;
4813  }
4814 
4815  // Calculate padding at the beginning based on current offset.
4816  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
4817 
4818  // Calculate required margin at the end if this is not last suballocation.
4819  VmaSuballocationList::const_iterator next = suballocItem;
4820  ++next;
4821  const VkDeviceSize requiredEndMargin =
4822  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4823 
4824  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
4825  // Another early return check.
4826  if(suballocItem->offset + totalSize > m_Size)
4827  {
4828  return false;
4829  }
4830 
4831  // Advance lastSuballocItem until desired size is reached.
4832  // Update itemsToMakeLostCount.
4833  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
4834  if(totalSize > suballocItem->size)
4835  {
4836  VkDeviceSize remainingSize = totalSize - suballocItem->size;
4837  while(remainingSize > 0)
4838  {
4839  ++lastSuballocItem;
4840  if(lastSuballocItem == m_Suballocations.cend())
4841  {
4842  return false;
4843  }
4844  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
4845  {
4846  *pSumFreeSize += lastSuballocItem->size;
4847  }
4848  else
4849  {
4850  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
4851  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
4852  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4853  {
4854  ++*itemsToMakeLostCount;
4855  *pSumItemSize += lastSuballocItem->size;
4856  }
4857  else
4858  {
4859  return false;
4860  }
4861  }
4862  remainingSize = (lastSuballocItem->size < remainingSize) ?
4863  remainingSize - lastSuballocItem->size : 0;
4864  }
4865  }
4866 
4867  // Check next suballocations for BufferImageGranularity conflicts.
4868  // If conflict exists, we must mark more allocations lost or fail.
4869  if(bufferImageGranularity > 1)
4870  {
4871  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
4872  ++nextSuballocItem;
4873  while(nextSuballocItem != m_Suballocations.cend())
4874  {
4875  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4876  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4877  {
4878  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4879  {
4880  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
4881  if(nextSuballoc.hAllocation->CanBecomeLost() &&
4882  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
4883  {
4884  ++*itemsToMakeLostCount;
4885  }
4886  else
4887  {
4888  return false;
4889  }
4890  }
4891  }
4892  else
4893  {
4894  // Already on next page.
4895  break;
4896  }
4897  ++nextSuballocItem;
4898  }
4899  }
4900  }
4901  else
4902  {
4903  const VmaSuballocation& suballoc = *suballocItem;
4904  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
4905 
4906  *pSumFreeSize = suballoc.size;
4907 
4908  // Size of this suballocation is too small for this request: Early return.
4909  if(suballoc.size < allocSize)
4910  {
4911  return false;
4912  }
4913 
4914  // Start from offset equal to beginning of this suballocation.
4915  *pOffset = suballoc.offset;
4916 
4917  // Apply VMA_DEBUG_MARGIN at the beginning.
4918  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
4919  {
4920  *pOffset += VMA_DEBUG_MARGIN;
4921  }
4922 
4923  // Apply alignment.
4924  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
4925  *pOffset = VmaAlignUp(*pOffset, alignment);
4926 
4927  // Check previous suballocations for BufferImageGranularity conflicts.
4928  // Make bigger alignment if necessary.
4929  if(bufferImageGranularity > 1)
4930  {
4931  bool bufferImageGranularityConflict = false;
4932  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
4933  while(prevSuballocItem != m_Suballocations.cbegin())
4934  {
4935  --prevSuballocItem;
4936  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
4937  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
4938  {
4939  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
4940  {
4941  bufferImageGranularityConflict = true;
4942  break;
4943  }
4944  }
4945  else
4946  // Already on previous page.
4947  break;
4948  }
4949  if(bufferImageGranularityConflict)
4950  {
4951  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
4952  }
4953  }
4954 
4955  // Calculate padding at the beginning based on current offset.
4956  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
4957 
4958  // Calculate required margin at the end if this is not last suballocation.
4959  VmaSuballocationList::const_iterator next = suballocItem;
4960  ++next;
4961  const VkDeviceSize requiredEndMargin =
4962  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
4963 
4964  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
4965  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
4966  {
4967  return false;
4968  }
4969 
4970  // Check next suballocations for BufferImageGranularity conflicts.
4971  // If conflict exists, allocation cannot be made here.
4972  if(bufferImageGranularity > 1)
4973  {
4974  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
4975  ++nextSuballocItem;
4976  while(nextSuballocItem != m_Suballocations.cend())
4977  {
4978  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
4979  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
4980  {
4981  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
4982  {
4983  return false;
4984  }
4985  }
4986  else
4987  {
4988  // Already on next page.
4989  break;
4990  }
4991  ++nextSuballocItem;
4992  }
4993  }
4994  }
4995 
4996  // All tests passed: Success. pOffset is already filled.
4997  return true;
4998 }
4999 
5000 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
5001 {
5002  VMA_ASSERT(item != m_Suballocations.end());
5003  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5004 
5005  VmaSuballocationList::iterator nextItem = item;
5006  ++nextItem;
5007  VMA_ASSERT(nextItem != m_Suballocations.end());
5008  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
5009 
5010  item->size += nextItem->size;
5011  --m_FreeCount;
5012  m_Suballocations.erase(nextItem);
5013 }
5014 
5015 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
5016 {
5017  // Change this suballocation to be marked as free.
5018  VmaSuballocation& suballoc = *suballocItem;
5019  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5020  suballoc.hAllocation = VK_NULL_HANDLE;
5021 
5022  // Update totals.
5023  ++m_FreeCount;
5024  m_SumFreeSize += suballoc.size;
5025 
5026  // Merge with previous and/or next suballocation if it's also free.
5027  bool mergeWithNext = false;
5028  bool mergeWithPrev = false;
5029 
5030  VmaSuballocationList::iterator nextItem = suballocItem;
5031  ++nextItem;
5032  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
5033  {
5034  mergeWithNext = true;
5035  }
5036 
5037  VmaSuballocationList::iterator prevItem = suballocItem;
5038  if(suballocItem != m_Suballocations.begin())
5039  {
5040  --prevItem;
5041  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5042  {
5043  mergeWithPrev = true;
5044  }
5045  }
5046 
5047  if(mergeWithNext)
5048  {
5049  UnregisterFreeSuballocation(nextItem);
5050  MergeFreeWithNext(suballocItem);
5051  }
5052 
5053  if(mergeWithPrev)
5054  {
5055  UnregisterFreeSuballocation(prevItem);
5056  MergeFreeWithNext(prevItem);
5057  RegisterFreeSuballocation(prevItem);
5058  return prevItem;
5059  }
5060  else
5061  {
5062  RegisterFreeSuballocation(suballocItem);
5063  return suballocItem;
5064  }
5065 }
5066 
5067 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
5068 {
5069  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5070  VMA_ASSERT(item->size > 0);
5071 
5072  // You may want to enable this validation at the beginning or at the end of
5073  // this function, depending on what do you want to check.
5074  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5075 
5076  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5077  {
5078  if(m_FreeSuballocationsBySize.empty())
5079  {
5080  m_FreeSuballocationsBySize.push_back(item);
5081  }
5082  else
5083  {
5084  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
5085  }
5086  }
5087 
5088  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5089 }
5090 
5091 
5092 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
5093 {
5094  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
5095  VMA_ASSERT(item->size > 0);
5096 
5097  // You may want to enable this validation at the beginning or at the end of
5098  // this function, depending on what do you want to check.
5099  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5100 
5101  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5102  {
5103  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5104  m_FreeSuballocationsBySize.data(),
5105  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
5106  item,
5107  VmaSuballocationItemSizeLess());
5108  for(size_t index = it - m_FreeSuballocationsBySize.data();
5109  index < m_FreeSuballocationsBySize.size();
5110  ++index)
5111  {
5112  if(m_FreeSuballocationsBySize[index] == item)
5113  {
5114  VmaVectorRemove(m_FreeSuballocationsBySize, index);
5115  return;
5116  }
5117  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
5118  }
5119  VMA_ASSERT(0 && "Not found.");
5120  }
5121 
5122  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
5123 }
5124 
5126 // class VmaDeviceMemoryBlock
5127 
5128 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
5129  m_MemoryTypeIndex(UINT32_MAX),
5130  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
5131  m_hMemory(VK_NULL_HANDLE),
5132  m_PersistentMap(false),
5133  m_pMappedData(VMA_NULL),
5134  m_Metadata(hAllocator)
5135 {
5136 }
5137 
5138 void VmaDeviceMemoryBlock::Init(
5139  uint32_t newMemoryTypeIndex,
5140  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
5141  VkDeviceMemory newMemory,
5142  VkDeviceSize newSize,
5143  bool persistentMap,
5144  void* pMappedData)
5145 {
5146  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
5147 
5148  m_MemoryTypeIndex = newMemoryTypeIndex;
5149  m_BlockVectorType = newBlockVectorType;
5150  m_hMemory = newMemory;
5151  m_PersistentMap = persistentMap;
5152  m_pMappedData = pMappedData;
5153 
5154  m_Metadata.Init(newSize);
5155 }
5156 
5157 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
5158 {
5159  // This is the most important assert in the entire library.
5160  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
5161  VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
5162 
5163  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
5164  if(m_pMappedData != VMA_NULL)
5165  {
5166  (allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, m_hMemory);
5167  m_pMappedData = VMA_NULL;
5168  }
5169 
5170  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
5171  m_hMemory = VK_NULL_HANDLE;
5172 }
5173 
5174 bool VmaDeviceMemoryBlock::Validate() const
5175 {
5176  if((m_hMemory == VK_NULL_HANDLE) ||
5177  (m_Metadata.GetSize() == 0))
5178  {
5179  return false;
5180  }
5181 
5182  return m_Metadata.Validate();
5183 }
5184 
5185 static void InitStatInfo(VmaStatInfo& outInfo)
5186 {
5187  memset(&outInfo, 0, sizeof(outInfo));
5188  outInfo.allocationSizeMin = UINT64_MAX;
5189  outInfo.unusedRangeSizeMin = UINT64_MAX;
5190 }
5191 
5192 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
5193 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
5194 {
5195  inoutInfo.blockCount += srcInfo.blockCount;
5196  inoutInfo.allocationCount += srcInfo.allocationCount;
5197  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
5198  inoutInfo.usedBytes += srcInfo.usedBytes;
5199  inoutInfo.unusedBytes += srcInfo.unusedBytes;
5200  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
5201  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
5202  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
5203  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
5204 }
5205 
5206 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
5207 {
5208  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
5209  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
5210  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
5211  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
5212 }
5213 
5214 VmaPool_T::VmaPool_T(
5215  VmaAllocator hAllocator,
5216  const VmaPoolCreateInfo& createInfo) :
5217  m_BlockVector(
5218  hAllocator,
5219  createInfo.memoryTypeIndex,
5220  (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ?
5221  VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
5222  createInfo.blockSize,
5223  createInfo.minBlockCount,
5224  createInfo.maxBlockCount,
5225  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
5226  createInfo.frameInUseCount,
5227  true) // isCustomPool
5228 {
5229 }
5230 
5231 VmaPool_T::~VmaPool_T()
5232 {
5233 }
5234 
5235 #if VMA_STATS_STRING_ENABLED
5236 
5237 #endif // #if VMA_STATS_STRING_ENABLED
5238 
5239 VmaBlockVector::VmaBlockVector(
5240  VmaAllocator hAllocator,
5241  uint32_t memoryTypeIndex,
5242  VMA_BLOCK_VECTOR_TYPE blockVectorType,
5243  VkDeviceSize preferredBlockSize,
5244  size_t minBlockCount,
5245  size_t maxBlockCount,
5246  VkDeviceSize bufferImageGranularity,
5247  uint32_t frameInUseCount,
5248  bool isCustomPool) :
5249  m_hAllocator(hAllocator),
5250  m_MemoryTypeIndex(memoryTypeIndex),
5251  m_BlockVectorType(blockVectorType),
5252  m_PreferredBlockSize(preferredBlockSize),
5253  m_MinBlockCount(minBlockCount),
5254  m_MaxBlockCount(maxBlockCount),
5255  m_BufferImageGranularity(bufferImageGranularity),
5256  m_FrameInUseCount(frameInUseCount),
5257  m_IsCustomPool(isCustomPool),
5258  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
5259  m_HasEmptyBlock(false),
5260  m_pDefragmentator(VMA_NULL)
5261 {
5262 }
5263 
5264 VmaBlockVector::~VmaBlockVector()
5265 {
5266  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
5267 
5268  for(size_t i = m_Blocks.size(); i--; )
5269  {
5270  m_Blocks[i]->Destroy(m_hAllocator);
5271  vma_delete(m_hAllocator, m_Blocks[i]);
5272  }
5273 }
5274 
5275 VkResult VmaBlockVector::CreateMinBlocks()
5276 {
5277  for(size_t i = 0; i < m_MinBlockCount; ++i)
5278  {
5279  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
5280  if(res != VK_SUCCESS)
5281  {
5282  return res;
5283  }
5284  }
5285  return VK_SUCCESS;
5286 }
5287 
5288 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
5289 {
5290  pStats->size = 0;
5291  pStats->unusedSize = 0;
5292  pStats->allocationCount = 0;
5293  pStats->unusedRangeCount = 0;
5294  pStats->unusedRangeSizeMax = 0;
5295 
5296  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5297 
5298  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5299  {
5300  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5301  VMA_ASSERT(pBlock);
5302  VMA_HEAVY_ASSERT(pBlock->Validate());
5303  pBlock->m_Metadata.AddPoolStats(*pStats);
5304  }
5305 }
5306 
5307 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
5308 
5309 VkResult VmaBlockVector::Allocate(
5310  VmaPool hCurrentPool,
5311  uint32_t currentFrameIndex,
5312  const VkMemoryRequirements& vkMemReq,
5313  const VmaAllocationCreateInfo& createInfo,
5314  VmaSuballocationType suballocType,
5315  VmaAllocation* pAllocation)
5316 {
5317  // Validate flags.
5318  if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) !=
5319  (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED))
5320  {
5321  VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT.");
5322  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5323  }
5324 
5325  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5326 
5327  // 1. Search existing allocations. Try to allocate without making other allocations lost.
5328  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5329  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5330  {
5331  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5332  VMA_ASSERT(pCurrBlock);
5333  VmaAllocationRequest currRequest = {};
5334  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5335  currentFrameIndex,
5336  m_FrameInUseCount,
5337  m_BufferImageGranularity,
5338  vkMemReq.size,
5339  vkMemReq.alignment,
5340  suballocType,
5341  false, // canMakeOtherLost
5342  &currRequest))
5343  {
5344  // Allocate from pCurrBlock.
5345  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
5346 
5347  // We no longer have an empty Allocation.
5348  if(pCurrBlock->m_Metadata.IsEmpty())
5349  {
5350  m_HasEmptyBlock = false;
5351  }
5352 
5353  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5354  pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
5355  (*pAllocation)->InitBlockAllocation(
5356  hCurrentPool,
5357  pCurrBlock,
5358  currRequest.offset,
5359  vkMemReq.alignment,
5360  vkMemReq.size,
5361  suballocType,
5362  createInfo.pUserData,
5363  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5364  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
5365  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5366  return VK_SUCCESS;
5367  }
5368  }
5369 
5370  const bool canCreateNewBlock =
5371  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
5372  (m_Blocks.size() < m_MaxBlockCount);
5373 
5374  // 2. Try to create new block.
5375  if(canCreateNewBlock)
5376  {
5377  // 2.1. Start with full preferredBlockSize.
5378  VkDeviceSize blockSize = m_PreferredBlockSize;
5379  size_t newBlockIndex = 0;
5380  VkResult res = CreateBlock(blockSize, &newBlockIndex);
5381  // Allocating blocks of other sizes is allowed only in default pools.
5382  // In custom pools block size is fixed.
5383  if(res < 0 && m_IsCustomPool == false)
5384  {
5385  // 2.2. Try half the size.
5386  blockSize /= 2;
5387  if(blockSize >= vkMemReq.size)
5388  {
5389  res = CreateBlock(blockSize, &newBlockIndex);
5390  if(res < 0)
5391  {
5392  // 2.3. Try quarter the size.
5393  blockSize /= 2;
5394  if(blockSize >= vkMemReq.size)
5395  {
5396  res = CreateBlock(blockSize, &newBlockIndex);
5397  }
5398  }
5399  }
5400  }
5401  if(res == VK_SUCCESS)
5402  {
5403  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
5404  VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
5405 
5406  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
5407  VmaAllocationRequest allocRequest;
5408  pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
5409  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5410  pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
5411  (*pAllocation)->InitBlockAllocation(
5412  hCurrentPool,
5413  pBlock,
5414  allocRequest.offset,
5415  vkMemReq.alignment,
5416  vkMemReq.size,
5417  suballocType,
5418  createInfo.pUserData,
5419  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5420  VMA_HEAVY_ASSERT(pBlock->Validate());
5421  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
5422 
5423  return VK_SUCCESS;
5424  }
5425  }
5426 
5427  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
5428 
5429  // 3. Try to allocate from existing blocks with making other allocations lost.
5430  if(canMakeOtherLost)
5431  {
5432  uint32_t tryIndex = 0;
5433  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
5434  {
5435  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
5436  VmaAllocationRequest bestRequest = {};
5437  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
5438 
5439  // 1. Search existing allocations.
5440  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
5441  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
5442  {
5443  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
5444  VMA_ASSERT(pCurrBlock);
5445  VmaAllocationRequest currRequest = {};
5446  if(pCurrBlock->m_Metadata.CreateAllocationRequest(
5447  currentFrameIndex,
5448  m_FrameInUseCount,
5449  m_BufferImageGranularity,
5450  vkMemReq.size,
5451  vkMemReq.alignment,
5452  suballocType,
5453  canMakeOtherLost,
5454  &currRequest))
5455  {
5456  const VkDeviceSize currRequestCost = currRequest.CalcCost();
5457  if(pBestRequestBlock == VMA_NULL ||
5458  currRequestCost < bestRequestCost)
5459  {
5460  pBestRequestBlock = pCurrBlock;
5461  bestRequest = currRequest;
5462  bestRequestCost = currRequestCost;
5463 
5464  if(bestRequestCost == 0)
5465  {
5466  break;
5467  }
5468  }
5469  }
5470  }
5471 
5472  if(pBestRequestBlock != VMA_NULL)
5473  {
5474  if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
5475  currentFrameIndex,
5476  m_FrameInUseCount,
5477  &bestRequest))
5478  {
5479  // We no longer have an empty Allocation.
5480  if(pBestRequestBlock->m_Metadata.IsEmpty())
5481  {
5482  m_HasEmptyBlock = false;
5483  }
5484  // Allocate from this pBlock.
5485  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
5486  pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
5487  (*pAllocation)->InitBlockAllocation(
5488  hCurrentPool,
5489  pBestRequestBlock,
5490  bestRequest.offset,
5491  vkMemReq.alignment,
5492  vkMemReq.size,
5493  suballocType,
5494  createInfo.pUserData,
5495  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
5496  VMA_HEAVY_ASSERT(pBlock->Validate());
5497  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
5498  return VK_SUCCESS;
5499  }
5500  // else: Some allocations must have been touched while we are here. Next try.
5501  }
5502  else
5503  {
5504  // Could not find place in any of the blocks - break outer loop.
5505  break;
5506  }
5507  }
5508  /* Maximum number of tries exceeded - a very unlike event when many other
5509  threads are simultaneously touching allocations making it impossible to make
5510  lost at the same time as we try to allocate. */
5511  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
5512  {
5513  return VK_ERROR_TOO_MANY_OBJECTS;
5514  }
5515  }
5516 
5517  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
5518 }
5519 
5520 void VmaBlockVector::Free(
5521  VmaAllocation hAllocation)
5522 {
5523  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
5524 
5525  // Scope for lock.
5526  {
5527  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5528 
5529  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
5530 
5531  pBlock->m_Metadata.Free(hAllocation);
5532  VMA_HEAVY_ASSERT(pBlock->Validate());
5533 
5534  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
5535 
5536  // pBlock became empty after this deallocation.
5537  if(pBlock->m_Metadata.IsEmpty())
5538  {
5539  // Already has empty Allocation. We don't want to have two, so delete this one.
5540  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
5541  {
5542  pBlockToDelete = pBlock;
5543  Remove(pBlock);
5544  }
5545  // We now have first empty Allocation.
5546  else
5547  {
5548  m_HasEmptyBlock = true;
5549  }
5550  }
5551  // Must be called after srcBlockIndex is used, because later it may become invalid!
5552  IncrementallySortBlocks();
5553  }
5554 
5555  // Destruction of a free Allocation. Deferred until this point, outside of mutex
5556  // lock, for performance reason.
5557  if(pBlockToDelete != VMA_NULL)
5558  {
5559  VMA_DEBUG_LOG(" Deleted empty allocation");
5560  pBlockToDelete->Destroy(m_hAllocator);
5561  vma_delete(m_hAllocator, pBlockToDelete);
5562  }
5563 }
5564 
5565 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
5566 {
5567  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5568  {
5569  if(m_Blocks[blockIndex] == pBlock)
5570  {
5571  VmaVectorRemove(m_Blocks, blockIndex);
5572  return;
5573  }
5574  }
5575  VMA_ASSERT(0);
5576 }
5577 
5578 void VmaBlockVector::IncrementallySortBlocks()
5579 {
5580  // Bubble sort only until first swap.
5581  for(size_t i = 1; i < m_Blocks.size(); ++i)
5582  {
5583  if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
5584  {
5585  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
5586  return;
5587  }
5588  }
5589 }
5590 
5591 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
5592 {
5593  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
5594  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
5595  allocInfo.allocationSize = blockSize;
5596  VkDeviceMemory mem = VK_NULL_HANDLE;
5597  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
5598  if(res < 0)
5599  {
5600  return res;
5601  }
5602 
5603  // New VkDeviceMemory successfully created.
5604 
5605  // Map memory if needed.
5606  void* pMappedData = VMA_NULL;
5607  const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED);
5608  if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0)
5609  {
5610  res = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5611  m_hAllocator->m_hDevice,
5612  mem,
5613  0,
5614  VK_WHOLE_SIZE,
5615  0,
5616  &pMappedData);
5617  if(res < 0)
5618  {
5619  VMA_DEBUG_LOG(" vkMapMemory FAILED");
5620  m_hAllocator->FreeVulkanMemory(m_MemoryTypeIndex, blockSize, mem);
5621  return res;
5622  }
5623  }
5624 
5625  // Create new Allocation for it.
5626  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
5627  pBlock->Init(
5628  m_MemoryTypeIndex,
5629  (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType,
5630  mem,
5631  allocInfo.allocationSize,
5632  persistentMap,
5633  pMappedData);
5634 
5635  m_Blocks.push_back(pBlock);
5636  if(pNewBlockIndex != VMA_NULL)
5637  {
5638  *pNewBlockIndex = m_Blocks.size() - 1;
5639  }
5640 
5641  return VK_SUCCESS;
5642 }
5643 
5644 #if VMA_STATS_STRING_ENABLED
5645 
5646 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
5647 {
5648  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5649 
5650  json.BeginObject();
5651 
5652  if(m_IsCustomPool)
5653  {
5654  json.WriteString("MemoryTypeIndex");
5655  json.WriteNumber(m_MemoryTypeIndex);
5656 
5657  if(m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
5658  {
5659  json.WriteString("Mapped");
5660  json.WriteBool(true);
5661  }
5662 
5663  json.WriteString("BlockSize");
5664  json.WriteNumber(m_PreferredBlockSize);
5665 
5666  json.WriteString("BlockCount");
5667  json.BeginObject(true);
5668  if(m_MinBlockCount > 0)
5669  {
5670  json.WriteString("Min");
5671  json.WriteNumber(m_MinBlockCount);
5672  }
5673  if(m_MaxBlockCount < SIZE_MAX)
5674  {
5675  json.WriteString("Max");
5676  json.WriteNumber(m_MaxBlockCount);
5677  }
5678  json.WriteString("Cur");
5679  json.WriteNumber(m_Blocks.size());
5680  json.EndObject();
5681 
5682  if(m_FrameInUseCount > 0)
5683  {
5684  json.WriteString("FrameInUseCount");
5685  json.WriteNumber(m_FrameInUseCount);
5686  }
5687  }
5688  else
5689  {
5690  json.WriteString("PreferredBlockSize");
5691  json.WriteNumber(m_PreferredBlockSize);
5692  }
5693 
5694  json.WriteString("Blocks");
5695  json.BeginArray();
5696  for(size_t i = 0; i < m_Blocks.size(); ++i)
5697  {
5698  m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
5699  }
5700  json.EndArray();
5701 
5702  json.EndObject();
5703 }
5704 
5705 #endif // #if VMA_STATS_STRING_ENABLED
5706 
5707 void VmaBlockVector::UnmapPersistentlyMappedMemory()
5708 {
5709  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5710 
5711  for(size_t i = m_Blocks.size(); i--; )
5712  {
5713  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5714  if(pBlock->m_pMappedData != VMA_NULL)
5715  {
5716  VMA_ASSERT(pBlock->m_PersistentMap != false);
5717  (m_hAllocator->GetVulkanFunctions().vkUnmapMemory)(m_hAllocator->m_hDevice, pBlock->m_hMemory);
5718  pBlock->m_pMappedData = VMA_NULL;
5719  }
5720  }
5721 }
5722 
5723 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
5724 {
5725  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5726 
5727  VkResult finalResult = VK_SUCCESS;
5728  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
5729  {
5730  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
5731  if(pBlock->m_PersistentMap)
5732  {
5733  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
5734  VkResult localResult = (*m_hAllocator->GetVulkanFunctions().vkMapMemory)(
5735  m_hAllocator->m_hDevice,
5736  pBlock->m_hMemory,
5737  0,
5738  VK_WHOLE_SIZE,
5739  0,
5740  &pBlock->m_pMappedData);
5741  if(localResult != VK_SUCCESS)
5742  {
5743  finalResult = localResult;
5744  }
5745  }
5746  }
5747  return finalResult;
5748 }
5749 
5750 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
5751  VmaAllocator hAllocator,
5752  uint32_t currentFrameIndex)
5753 {
5754  if(m_pDefragmentator == VMA_NULL)
5755  {
5756  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
5757  hAllocator,
5758  this,
5759  currentFrameIndex);
5760  }
5761 
5762  return m_pDefragmentator;
5763 }
5764 
5765 VkResult VmaBlockVector::Defragment(
5766  VmaDefragmentationStats* pDefragmentationStats,
5767  VkDeviceSize& maxBytesToMove,
5768  uint32_t& maxAllocationsToMove)
5769 {
5770  if(m_pDefragmentator == VMA_NULL)
5771  {
5772  return VK_SUCCESS;
5773  }
5774 
5775  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5776 
5777  // Defragment.
5778  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
5779 
5780  // Accumulate statistics.
5781  if(pDefragmentationStats != VMA_NULL)
5782  {
5783  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
5784  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
5785  pDefragmentationStats->bytesMoved += bytesMoved;
5786  pDefragmentationStats->allocationsMoved += allocationsMoved;
5787  VMA_ASSERT(bytesMoved <= maxBytesToMove);
5788  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
5789  maxBytesToMove -= bytesMoved;
5790  maxAllocationsToMove -= allocationsMoved;
5791  }
5792 
5793  // Free empty blocks.
5794  m_HasEmptyBlock = false;
5795  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
5796  {
5797  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
5798  if(pBlock->m_Metadata.IsEmpty())
5799  {
5800  if(m_Blocks.size() > m_MinBlockCount)
5801  {
5802  if(pDefragmentationStats != VMA_NULL)
5803  {
5804  ++pDefragmentationStats->deviceMemoryBlocksFreed;
5805  pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
5806  }
5807 
5808  VmaVectorRemove(m_Blocks, blockIndex);
5809  pBlock->Destroy(m_hAllocator);
5810  vma_delete(m_hAllocator, pBlock);
5811  }
5812  else
5813  {
5814  m_HasEmptyBlock = true;
5815  }
5816  }
5817  }
5818 
5819  return result;
5820 }
5821 
5822 void VmaBlockVector::DestroyDefragmentator()
5823 {
5824  if(m_pDefragmentator != VMA_NULL)
5825  {
5826  vma_delete(m_hAllocator, m_pDefragmentator);
5827  m_pDefragmentator = VMA_NULL;
5828  }
5829 }
5830 
5831 void VmaBlockVector::MakePoolAllocationsLost(
5832  uint32_t currentFrameIndex,
5833  size_t* pLostAllocationCount)
5834 {
5835  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5836 
5837  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5838  {
5839  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5840  VMA_ASSERT(pBlock);
5841  pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
5842  }
5843 }
5844 
5845 void VmaBlockVector::AddStats(VmaStats* pStats)
5846 {
5847  const uint32_t memTypeIndex = m_MemoryTypeIndex;
5848  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
5849 
5850  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
5851 
5852  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
5853  {
5854  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
5855  VMA_ASSERT(pBlock);
5856  VMA_HEAVY_ASSERT(pBlock->Validate());
5857  VmaStatInfo allocationStatInfo;
5858  pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
5859  VmaAddStatInfo(pStats->total, allocationStatInfo);
5860  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
5861  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
5862  }
5863 }
5864 
5866 // VmaDefragmentator members definition
5867 
5868 VmaDefragmentator::VmaDefragmentator(
5869  VmaAllocator hAllocator,
5870  VmaBlockVector* pBlockVector,
5871  uint32_t currentFrameIndex) :
5872  m_hAllocator(hAllocator),
5873  m_pBlockVector(pBlockVector),
5874  m_CurrentFrameIndex(currentFrameIndex),
5875  m_BytesMoved(0),
5876  m_AllocationsMoved(0),
5877  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
5878  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
5879 {
5880 }
5881 
5882 VmaDefragmentator::~VmaDefragmentator()
5883 {
5884  for(size_t i = m_Blocks.size(); i--; )
5885  {
5886  vma_delete(m_hAllocator, m_Blocks[i]);
5887  }
5888 }
5889 
5890 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
5891 {
5892  AllocationInfo allocInfo;
5893  allocInfo.m_hAllocation = hAlloc;
5894  allocInfo.m_pChanged = pChanged;
5895  m_Allocations.push_back(allocInfo);
5896 }
5897 
5898 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
5899 {
5900  // It has already been mapped for defragmentation.
5901  if(m_pMappedDataForDefragmentation)
5902  {
5903  *ppMappedData = m_pMappedDataForDefragmentation;
5904  return VK_SUCCESS;
5905  }
5906 
5907  // It is persistently mapped.
5908  if(m_pBlock->m_PersistentMap)
5909  {
5910  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
5911  *ppMappedData = m_pBlock->m_pMappedData;
5912  return VK_SUCCESS;
5913  }
5914 
5915  // Map on first usage.
5916  VkResult res = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5917  hAllocator->m_hDevice,
5918  m_pBlock->m_hMemory,
5919  0,
5920  VK_WHOLE_SIZE,
5921  0,
5922  &m_pMappedDataForDefragmentation);
5923  *ppMappedData = m_pMappedDataForDefragmentation;
5924  return res;
5925 }
5926 
5927 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
5928 {
5929  if(m_pMappedDataForDefragmentation != VMA_NULL)
5930  {
5931  (hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_pBlock->m_hMemory);
5932  }
5933 }
5934 
5935 VkResult VmaDefragmentator::DefragmentRound(
5936  VkDeviceSize maxBytesToMove,
5937  uint32_t maxAllocationsToMove)
5938 {
5939  if(m_Blocks.empty())
5940  {
5941  return VK_SUCCESS;
5942  }
5943 
5944  size_t srcBlockIndex = m_Blocks.size() - 1;
5945  size_t srcAllocIndex = SIZE_MAX;
5946  for(;;)
5947  {
5948  // 1. Find next allocation to move.
5949  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
5950  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
5951  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
5952  {
5953  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
5954  {
5955  // Finished: no more allocations to process.
5956  if(srcBlockIndex == 0)
5957  {
5958  return VK_SUCCESS;
5959  }
5960  else
5961  {
5962  --srcBlockIndex;
5963  srcAllocIndex = SIZE_MAX;
5964  }
5965  }
5966  else
5967  {
5968  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
5969  }
5970  }
5971 
5972  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
5973  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
5974 
5975  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
5976  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
5977  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
5978  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
5979 
5980  // 2. Try to find new place for this allocation in preceding or current block.
5981  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
5982  {
5983  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
5984  VmaAllocationRequest dstAllocRequest;
5985  if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
5986  m_CurrentFrameIndex,
5987  m_pBlockVector->GetFrameInUseCount(),
5988  m_pBlockVector->GetBufferImageGranularity(),
5989  size,
5990  alignment,
5991  suballocType,
5992  false, // canMakeOtherLost
5993  &dstAllocRequest) &&
5994  MoveMakesSense(
5995  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
5996  {
5997  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
5998 
5999  // Reached limit on number of allocations or bytes to move.
6000  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
6001  (m_BytesMoved + size > maxBytesToMove))
6002  {
6003  return VK_INCOMPLETE;
6004  }
6005 
6006  void* pDstMappedData = VMA_NULL;
6007  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
6008  if(res != VK_SUCCESS)
6009  {
6010  return res;
6011  }
6012 
6013  void* pSrcMappedData = VMA_NULL;
6014  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
6015  if(res != VK_SUCCESS)
6016  {
6017  return res;
6018  }
6019 
6020  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
6021  memcpy(
6022  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
6023  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
6024  static_cast<size_t>(size));
6025 
6026  pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
6027  pSrcBlockInfo->m_pBlock->m_Metadata.Free(allocInfo.m_hAllocation);
6028 
6029  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
6030 
6031  if(allocInfo.m_pChanged != VMA_NULL)
6032  {
6033  *allocInfo.m_pChanged = VK_TRUE;
6034  }
6035 
6036  ++m_AllocationsMoved;
6037  m_BytesMoved += size;
6038 
6039  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
6040 
6041  break;
6042  }
6043  }
6044 
6045  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
6046 
6047  if(srcAllocIndex > 0)
6048  {
6049  --srcAllocIndex;
6050  }
6051  else
6052  {
6053  if(srcBlockIndex > 0)
6054  {
6055  --srcBlockIndex;
6056  srcAllocIndex = SIZE_MAX;
6057  }
6058  else
6059  {
6060  return VK_SUCCESS;
6061  }
6062  }
6063  }
6064 }
6065 
6066 VkResult VmaDefragmentator::Defragment(
6067  VkDeviceSize maxBytesToMove,
6068  uint32_t maxAllocationsToMove)
6069 {
6070  if(m_Allocations.empty())
6071  {
6072  return VK_SUCCESS;
6073  }
6074 
6075  // Create block info for each block.
6076  const size_t blockCount = m_pBlockVector->m_Blocks.size();
6077  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6078  {
6079  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
6080  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
6081  m_Blocks.push_back(pBlockInfo);
6082  }
6083 
6084  // Sort them by m_pBlock pointer value.
6085  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
6086 
6087  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
6088  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
6089  {
6090  AllocationInfo& allocInfo = m_Allocations[blockIndex];
6091  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
6092  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6093  {
6094  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
6095  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
6096  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
6097  {
6098  (*it)->m_Allocations.push_back(allocInfo);
6099  }
6100  else
6101  {
6102  VMA_ASSERT(0);
6103  }
6104  }
6105  }
6106  m_Allocations.clear();
6107 
6108  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6109  {
6110  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
6111  pBlockInfo->CalcHasNonMovableAllocations();
6112  pBlockInfo->SortAllocationsBySizeDescecnding();
6113  }
6114 
6115  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
6116  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
6117 
6118  // Execute defragmentation rounds (the main part).
6119  VkResult result = VK_SUCCESS;
6120  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
6121  {
6122  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
6123  }
6124 
6125  // Unmap blocks that were mapped for defragmentation.
6126  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
6127  {
6128  m_Blocks[blockIndex]->Unmap(m_hAllocator);
6129  }
6130 
6131  return result;
6132 }
6133 
6134 bool VmaDefragmentator::MoveMakesSense(
6135  size_t dstBlockIndex, VkDeviceSize dstOffset,
6136  size_t srcBlockIndex, VkDeviceSize srcOffset)
6137 {
6138  if(dstBlockIndex < srcBlockIndex)
6139  {
6140  return true;
6141  }
6142  if(dstBlockIndex > srcBlockIndex)
6143  {
6144  return false;
6145  }
6146  if(dstOffset < srcOffset)
6147  {
6148  return true;
6149  }
6150  return false;
6151 }
6152 
6154 // VmaAllocator_T
6155 
6156 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
6157  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
6158  m_PhysicalDevice(pCreateInfo->physicalDevice),
6159  m_hDevice(pCreateInfo->device),
6160  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
6161  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
6162  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
6163  m_UnmapPersistentlyMappedMemoryCounter(0),
6164  m_PreferredLargeHeapBlockSize(0),
6165  m_PreferredSmallHeapBlockSize(0),
6166  m_CurrentFrameIndex(0),
6167  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
6168 {
6169  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
6170 
6171  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
6172  memset(&m_MemProps, 0, sizeof(m_MemProps));
6173  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
6174 
6175  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
6176  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
6177 
6178  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6179  {
6180  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
6181  }
6182 
6183  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
6184  {
6185  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
6186  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
6187  }
6188 
6189  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
6190 
6191  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
6192  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
6193 
6194  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
6195  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
6196  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
6197  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
6198 
6199  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
6200  {
6201  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
6202  {
6203  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
6204  if(limit != VK_WHOLE_SIZE)
6205  {
6206  m_HeapSizeLimit[heapIndex] = limit;
6207  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
6208  {
6209  m_MemProps.memoryHeaps[heapIndex].size = limit;
6210  }
6211  }
6212  }
6213  }
6214 
6215  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6216  {
6217  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
6218 
6219  for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex)
6220  {
6221  m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)(
6222  this,
6223  memTypeIndex,
6224  static_cast<VMA_BLOCK_VECTOR_TYPE>(blockVectorTypeIndex),
6225  preferredBlockSize,
6226  0,
6227  SIZE_MAX,
6228  GetBufferImageGranularity(),
6229  pCreateInfo->frameInUseCount,
6230  false); // isCustomPool
6231  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
6232  // becase minBlockCount is 0.
6233  m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
6234  }
6235  }
6236 }
6237 
6238 VmaAllocator_T::~VmaAllocator_T()
6239 {
6240  VMA_ASSERT(m_Pools.empty());
6241 
6242  for(size_t i = GetMemoryTypeCount(); i--; )
6243  {
6244  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
6245  {
6246  vma_delete(this, m_pOwnAllocations[i][j]);
6247  vma_delete(this, m_pBlockVectors[i][j]);
6248  }
6249  }
6250 }
6251 
6252 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
6253 {
6254 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6255  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
6256  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
6257  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
6258  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
6259  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
6260  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
6261  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
6262  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
6263  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
6264  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
6265  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
6266  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
6267  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
6268  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
6269 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
6270 
6271  if(pVulkanFunctions != VMA_NULL)
6272  {
6273  m_VulkanFunctions = *pVulkanFunctions;
6274  }
6275 
6276  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
6277  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
6278  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
6279  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
6280  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
6281  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
6282  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
6283  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
6284  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
6285  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
6286  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
6287  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
6288  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
6289  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
6290  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
6291  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
6292 }
6293 
6294 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
6295 {
6296  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6297  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
6298  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
6299  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
6300 }
6301 
6302 VkResult VmaAllocator_T::AllocateMemoryOfType(
6303  const VkMemoryRequirements& vkMemReq,
6304  const VmaAllocationCreateInfo& createInfo,
6305  uint32_t memTypeIndex,
6306  VmaSuballocationType suballocType,
6307  VmaAllocation* pAllocation)
6308 {
6309  VMA_ASSERT(pAllocation != VMA_NULL);
6310  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
6311 
6312  uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags);
6313  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6314  VMA_ASSERT(blockVector);
6315 
6316  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
6317  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
6318  const bool ownMemory =
6319  (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 ||
6320  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
6321  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
6322  vkMemReq.size > preferredBlockSize / 2);
6323 
6324  if(ownMemory)
6325  {
6326  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6327  {
6328  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6329  }
6330  else
6331  {
6332  return AllocateOwnMemory(
6333  vkMemReq.size,
6334  suballocType,
6335  memTypeIndex,
6336  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6337  createInfo.pUserData,
6338  pAllocation);
6339  }
6340  }
6341  else
6342  {
6343  VkResult res = blockVector->Allocate(
6344  VK_NULL_HANDLE, // hCurrentPool
6345  m_CurrentFrameIndex.load(),
6346  vkMemReq,
6347  createInfo,
6348  suballocType,
6349  pAllocation);
6350  if(res == VK_SUCCESS)
6351  {
6352  return res;
6353  }
6354 
6355  // 5. Try own memory.
6356  res = AllocateOwnMemory(
6357  vkMemReq.size,
6358  suballocType,
6359  memTypeIndex,
6360  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
6361  createInfo.pUserData,
6362  pAllocation);
6363  if(res == VK_SUCCESS)
6364  {
6365  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
6366  VMA_DEBUG_LOG(" Allocated as OwnMemory");
6367  return VK_SUCCESS;
6368  }
6369  else
6370  {
6371  // Everything failed: Return error code.
6372  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6373  return res;
6374  }
6375  }
6376 }
6377 
6378 VkResult VmaAllocator_T::AllocateOwnMemory(
6379  VkDeviceSize size,
6380  VmaSuballocationType suballocType,
6381  uint32_t memTypeIndex,
6382  bool map,
6383  void* pUserData,
6384  VmaAllocation* pAllocation)
6385 {
6386  VMA_ASSERT(pAllocation);
6387 
6388  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6389  allocInfo.memoryTypeIndex = memTypeIndex;
6390  allocInfo.allocationSize = size;
6391 
6392  // Allocate VkDeviceMemory.
6393  VkDeviceMemory hMemory = VK_NULL_HANDLE;
6394  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
6395  if(res < 0)
6396  {
6397  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
6398  return res;
6399  }
6400 
6401  void* pMappedData = nullptr;
6402  if(map)
6403  {
6404  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
6405  {
6406  res = (*m_VulkanFunctions.vkMapMemory)(
6407  m_hDevice,
6408  hMemory,
6409  0,
6410  VK_WHOLE_SIZE,
6411  0,
6412  &pMappedData);
6413  if(res < 0)
6414  {
6415  VMA_DEBUG_LOG(" vkMapMemory FAILED");
6416  FreeVulkanMemory(memTypeIndex, size, hMemory);
6417  return res;
6418  }
6419  }
6420  }
6421 
6422  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load());
6423  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
6424 
6425  // Register it in m_pOwnAllocations.
6426  {
6427  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6428  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
6429  VMA_ASSERT(pOwnAllocations);
6430  VmaVectorInsertSorted<VmaPointerLess>(*pOwnAllocations, *pAllocation);
6431  }
6432 
6433  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
6434 
6435  return VK_SUCCESS;
6436 }
6437 
6438 VkResult VmaAllocator_T::AllocateMemory(
6439  const VkMemoryRequirements& vkMemReq,
6440  const VmaAllocationCreateInfo& createInfo,
6441  VmaSuballocationType suballocType,
6442  VmaAllocation* pAllocation)
6443 {
6444  if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 &&
6445  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
6446  {
6447  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
6448  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6449  }
6450  if((createInfo.pool != VK_NULL_HANDLE) &&
6451  ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0))
6452  {
6453  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid.");
6454  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6455  }
6456 
6457  if(createInfo.pool != VK_NULL_HANDLE)
6458  {
6459  return createInfo.pool->m_BlockVector.Allocate(
6460  createInfo.pool,
6461  m_CurrentFrameIndex.load(),
6462  vkMemReq,
6463  createInfo,
6464  suballocType,
6465  pAllocation);
6466  }
6467  else
6468  {
6469  // Bit mask of memory Vulkan types acceptable for this allocation.
6470  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
6471  uint32_t memTypeIndex = UINT32_MAX;
6472  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6473  if(res == VK_SUCCESS)
6474  {
6475  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6476  // Succeeded on first try.
6477  if(res == VK_SUCCESS)
6478  {
6479  return res;
6480  }
6481  // Allocation from this memory type failed. Try other compatible memory types.
6482  else
6483  {
6484  for(;;)
6485  {
6486  // Remove old memTypeIndex from list of possibilities.
6487  memoryTypeBits &= ~(1u << memTypeIndex);
6488  // Find alternative memTypeIndex.
6489  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
6490  if(res == VK_SUCCESS)
6491  {
6492  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
6493  // Allocation from this alternative memory type succeeded.
6494  if(res == VK_SUCCESS)
6495  {
6496  return res;
6497  }
6498  // else: Allocation from this memory type failed. Try next one - next loop iteration.
6499  }
6500  // No other matching memory type index could be found.
6501  else
6502  {
6503  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
6504  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6505  }
6506  }
6507  }
6508  }
6509  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
6510  else
6511  return res;
6512  }
6513 }
6514 
6515 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
6516 {
6517  VMA_ASSERT(allocation);
6518 
6519  if(allocation->CanBecomeLost() == false ||
6520  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
6521  {
6522  switch(allocation->GetType())
6523  {
6524  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
6525  {
6526  VmaBlockVector* pBlockVector = VMA_NULL;
6527  VmaPool hPool = allocation->GetPool();
6528  if(hPool != VK_NULL_HANDLE)
6529  {
6530  pBlockVector = &hPool->m_BlockVector;
6531  }
6532  else
6533  {
6534  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
6535  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
6536  pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6537  }
6538  pBlockVector->Free(allocation);
6539  }
6540  break;
6541  case VmaAllocation_T::ALLOCATION_TYPE_OWN:
6542  FreeOwnMemory(allocation);
6543  break;
6544  default:
6545  VMA_ASSERT(0);
6546  }
6547  }
6548 
6549  vma_delete(this, allocation);
6550 }
6551 
6552 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
6553 {
6554  // Initialize.
6555  InitStatInfo(pStats->total);
6556  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
6557  InitStatInfo(pStats->memoryType[i]);
6558  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
6559  InitStatInfo(pStats->memoryHeap[i]);
6560 
6561  // Process default pools.
6562  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6563  {
6564  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6565  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6566  {
6567  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
6568  VMA_ASSERT(pBlockVector);
6569  pBlockVector->AddStats(pStats);
6570  }
6571  }
6572 
6573  // Process custom pools.
6574  {
6575  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6576  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6577  {
6578  m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
6579  }
6580  }
6581 
6582  // Process own allocations.
6583  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
6584  {
6585  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
6586  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6587  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
6588  {
6589  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
6590  VMA_ASSERT(pOwnAllocVector);
6591  for(size_t allocIndex = 0, allocCount = pOwnAllocVector->size(); allocIndex < allocCount; ++allocIndex)
6592  {
6593  VmaStatInfo allocationStatInfo;
6594  (*pOwnAllocVector)[allocIndex]->OwnAllocCalcStatsInfo(allocationStatInfo);
6595  VmaAddStatInfo(pStats->total, allocationStatInfo);
6596  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
6597  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
6598  }
6599  }
6600  }
6601 
6602  // Postprocess.
6603  VmaPostprocessCalcStatInfo(pStats->total);
6604  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
6605  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
6606  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
6607  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
6608 }
6609 
6610 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
6611 
6612 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
6613 {
6614  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
6615  {
6616  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6617  {
6618  for(uint32_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
6619  {
6620  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6621  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6622  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6623  {
6624  // Process OwnAllocations.
6625  {
6626  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6627  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6628  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
6629  {
6630  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
6631  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(this);
6632  }
6633  }
6634 
6635  // Process normal Allocations.
6636  {
6637  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6638  pBlockVector->UnmapPersistentlyMappedMemory();
6639  }
6640  }
6641  }
6642 
6643  // Process custom pools.
6644  {
6645  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6646  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6647  {
6648  m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory();
6649  }
6650  }
6651  }
6652  }
6653 }
6654 
6655 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
6656 {
6657  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
6658  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
6659  {
6660  VkResult finalResult = VK_SUCCESS;
6661  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
6662  {
6663  // Process custom pools.
6664  {
6665  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6666  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
6667  {
6668  m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory();
6669  }
6670  }
6671 
6672  for(uint32_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
6673  {
6674  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
6675  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
6676  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
6677  {
6678  // Process OwnAllocations.
6679  {
6680  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
6681  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6682  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
6683  {
6684  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
6685  hAlloc->OwnAllocMapPersistentlyMappedMemory(this);
6686  }
6687  }
6688 
6689  // Process normal Allocations.
6690  {
6691  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
6692  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
6693  if(localResult != VK_SUCCESS)
6694  {
6695  finalResult = localResult;
6696  }
6697  }
6698  }
6699  }
6700  }
6701  return finalResult;
6702  }
6703  else
6704  return VK_SUCCESS;
6705 }
6706 
6707 VkResult VmaAllocator_T::Defragment(
6708  VmaAllocation* pAllocations,
6709  size_t allocationCount,
6710  VkBool32* pAllocationsChanged,
6711  const VmaDefragmentationInfo* pDefragmentationInfo,
6712  VmaDefragmentationStats* pDefragmentationStats)
6713 {
6714  if(pAllocationsChanged != VMA_NULL)
6715  {
6716  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
6717  }
6718  if(pDefragmentationStats != VMA_NULL)
6719  {
6720  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
6721  }
6722 
6723  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
6724  {
6725  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
6726  return VK_ERROR_MEMORY_MAP_FAILED;
6727  }
6728 
6729  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
6730 
6731  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
6732 
6733  const size_t poolCount = m_Pools.size();
6734 
6735  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
6736  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
6737  {
6738  VmaAllocation hAlloc = pAllocations[allocIndex];
6739  VMA_ASSERT(hAlloc);
6740  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
6741  // OwnAlloc cannot be defragmented.
6742  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
6743  // Only HOST_VISIBLE memory types can be defragmented.
6744  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
6745  // Lost allocation cannot be defragmented.
6746  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
6747  {
6748  VmaBlockVector* pAllocBlockVector = nullptr;
6749 
6750  const VmaPool hAllocPool = hAlloc->GetPool();
6751  // This allocation belongs to custom pool.
6752  if(hAllocPool != VK_NULL_HANDLE)
6753  {
6754  pAllocBlockVector = &hAllocPool->GetBlockVector();
6755  }
6756  // This allocation belongs to general pool.
6757  else
6758  {
6759  pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()];
6760  }
6761 
6762  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
6763 
6764  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
6765  &pAllocationsChanged[allocIndex] : VMA_NULL;
6766  pDefragmentator->AddAllocation(hAlloc, pChanged);
6767  }
6768  }
6769 
6770  VkResult result = VK_SUCCESS;
6771 
6772  // ======== Main processing.
6773 
6774  VkDeviceSize maxBytesToMove = SIZE_MAX;
6775  uint32_t maxAllocationsToMove = UINT32_MAX;
6776  if(pDefragmentationInfo != VMA_NULL)
6777  {
6778  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
6779  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
6780  }
6781 
6782  // Process standard memory.
6783  for(uint32_t memTypeIndex = 0;
6784  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
6785  ++memTypeIndex)
6786  {
6787  // Only HOST_VISIBLE memory types can be defragmented.
6788  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6789  {
6790  for(uint32_t blockVectorType = 0;
6791  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
6792  ++blockVectorType)
6793  {
6794  result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment(
6795  pDefragmentationStats,
6796  maxBytesToMove,
6797  maxAllocationsToMove);
6798  }
6799  }
6800  }
6801 
6802  // Process custom pools.
6803  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
6804  {
6805  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
6806  pDefragmentationStats,
6807  maxBytesToMove,
6808  maxAllocationsToMove);
6809  }
6810 
6811  // ======== Destroy defragmentators.
6812 
6813  // Process custom pools.
6814  for(size_t poolIndex = poolCount; poolIndex--; )
6815  {
6816  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
6817  }
6818 
6819  // Process standard memory.
6820  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
6821  {
6822  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
6823  {
6824  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
6825  {
6826  m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator();
6827  }
6828  }
6829  }
6830 
6831  return result;
6832 }
6833 
6834 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
6835 {
6836  if(hAllocation->CanBecomeLost())
6837  {
6838  /*
6839  Warning: This is a carefully designed algorithm.
6840  Do not modify unless you really know what you're doing :)
6841  */
6842  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
6843  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
6844  for(;;)
6845  {
6846  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6847  {
6848  pAllocationInfo->memoryType = UINT32_MAX;
6849  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
6850  pAllocationInfo->offset = 0;
6851  pAllocationInfo->size = hAllocation->GetSize();
6852  pAllocationInfo->pMappedData = VMA_NULL;
6853  pAllocationInfo->pUserData = hAllocation->GetUserData();
6854  return;
6855  }
6856  else if(localLastUseFrameIndex == localCurrFrameIndex)
6857  {
6858  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6859  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6860  pAllocationInfo->offset = hAllocation->GetOffset();
6861  pAllocationInfo->size = hAllocation->GetSize();
6862  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6863  pAllocationInfo->pUserData = hAllocation->GetUserData();
6864  return;
6865  }
6866  else // Last use time earlier than current time.
6867  {
6868  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
6869  {
6870  localLastUseFrameIndex = localCurrFrameIndex;
6871  }
6872  }
6873  }
6874  }
6875  // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic.
6876  else
6877  {
6878  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
6879  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
6880  pAllocationInfo->offset = hAllocation->GetOffset();
6881  pAllocationInfo->size = hAllocation->GetSize();
6882  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
6883  pAllocationInfo->pUserData = hAllocation->GetUserData();
6884  }
6885 }
6886 
6887 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
6888 {
6889  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
6890 
6891  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
6892 
6893  if(newCreateInfo.maxBlockCount == 0)
6894  {
6895  newCreateInfo.maxBlockCount = SIZE_MAX;
6896  }
6897  if(newCreateInfo.blockSize == 0)
6898  {
6899  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
6900  }
6901 
6902  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
6903 
6904  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
6905  if(res != VK_SUCCESS)
6906  {
6907  vma_delete(this, *pPool);
6908  *pPool = VMA_NULL;
6909  return res;
6910  }
6911 
6912  // Add to m_Pools.
6913  {
6914  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6915  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
6916  }
6917 
6918  return VK_SUCCESS;
6919 }
6920 
6921 void VmaAllocator_T::DestroyPool(VmaPool pool)
6922 {
6923  // Remove from m_Pools.
6924  {
6925  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
6926  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
6927  VMA_ASSERT(success && "Pool not found in Allocator.");
6928  }
6929 
6930  vma_delete(this, pool);
6931 }
6932 
6933 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
6934 {
6935  pool->m_BlockVector.GetPoolStats(pPoolStats);
6936 }
6937 
6938 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
6939 {
6940  m_CurrentFrameIndex.store(frameIndex);
6941 }
6942 
6943 void VmaAllocator_T::MakePoolAllocationsLost(
6944  VmaPool hPool,
6945  size_t* pLostAllocationCount)
6946 {
6947  hPool->m_BlockVector.MakePoolAllocationsLost(
6948  m_CurrentFrameIndex.load(),
6949  pLostAllocationCount);
6950 }
6951 
6952 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
6953 {
6954  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST);
6955  (*pAllocation)->InitLost();
6956 }
6957 
6958 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
6959 {
6960  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
6961 
6962  VkResult res;
6963  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
6964  {
6965  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
6966  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
6967  {
6968  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6969  if(res == VK_SUCCESS)
6970  {
6971  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
6972  }
6973  }
6974  else
6975  {
6976  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
6977  }
6978  }
6979  else
6980  {
6981  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
6982  }
6983 
6984  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
6985  {
6986  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
6987  }
6988 
6989  return res;
6990 }
6991 
6992 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
6993 {
6994  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
6995  {
6996  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
6997  }
6998 
6999  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
7000 
7001  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
7002  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
7003  {
7004  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
7005  m_HeapSizeLimit[heapIndex] += size;
7006  }
7007 }
7008 
7009 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
7010 {
7011  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
7012 
7013  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7014  {
7015  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7016  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
7017  VMA_ASSERT(pOwnAllocations);
7018  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pOwnAllocations, allocation);
7019  VMA_ASSERT(success);
7020  }
7021 
7022  VkDeviceMemory hMemory = allocation->GetMemory();
7023 
7024  if(allocation->GetMappedData() != VMA_NULL)
7025  {
7026  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
7027  }
7028 
7029  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
7030 
7031  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
7032 }
7033 
7034 #if VMA_STATS_STRING_ENABLED
7035 
7036 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
7037 {
7038  bool ownAllocationsStarted = false;
7039  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7040  {
7041  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
7042  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7043  {
7044  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
7045  VMA_ASSERT(pOwnAllocVector);
7046  if(pOwnAllocVector->empty() == false)
7047  {
7048  if(ownAllocationsStarted == false)
7049  {
7050  ownAllocationsStarted = true;
7051  json.WriteString("OwnAllocations");
7052  json.BeginObject();
7053  }
7054 
7055  json.BeginString("Type ");
7056  json.ContinueString(memTypeIndex);
7057  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7058  {
7059  json.ContinueString(" Mapped");
7060  }
7061  json.EndString();
7062 
7063  json.BeginArray();
7064 
7065  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
7066  {
7067  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
7068  json.BeginObject(true);
7069 
7070  json.WriteString("Size");
7071  json.WriteNumber(hAlloc->GetSize());
7072 
7073  json.WriteString("Type");
7074  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
7075 
7076  json.EndObject();
7077  }
7078 
7079  json.EndArray();
7080  }
7081  }
7082  }
7083  if(ownAllocationsStarted)
7084  {
7085  json.EndObject();
7086  }
7087 
7088  {
7089  bool allocationsStarted = false;
7090  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7091  {
7092  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
7093  {
7094  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
7095  {
7096  if(allocationsStarted == false)
7097  {
7098  allocationsStarted = true;
7099  json.WriteString("DefaultPools");
7100  json.BeginObject();
7101  }
7102 
7103  json.BeginString("Type ");
7104  json.ContinueString(memTypeIndex);
7105  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
7106  {
7107  json.ContinueString(" Mapped");
7108  }
7109  json.EndString();
7110 
7111  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json);
7112  }
7113  }
7114  }
7115  if(allocationsStarted)
7116  {
7117  json.EndObject();
7118  }
7119  }
7120 
7121  {
7122  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7123  const size_t poolCount = m_Pools.size();
7124  if(poolCount > 0)
7125  {
7126  json.WriteString("Pools");
7127  json.BeginArray();
7128  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
7129  {
7130  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
7131  }
7132  json.EndArray();
7133  }
7134  }
7135 }
7136 
7137 #endif // #if VMA_STATS_STRING_ENABLED
7138 
7139 static VkResult AllocateMemoryForImage(
7140  VmaAllocator allocator,
7141  VkImage image,
7142  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7143  VmaSuballocationType suballocType,
7144  VmaAllocation* pAllocation)
7145 {
7146  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
7147 
7148  VkMemoryRequirements vkMemReq = {};
7149  (*allocator->GetVulkanFunctions().vkGetImageMemoryRequirements)(allocator->m_hDevice, image, &vkMemReq);
7150 
7151  return allocator->AllocateMemory(
7152  vkMemReq,
7153  *pAllocationCreateInfo,
7154  suballocType,
7155  pAllocation);
7156 }
7157 
7159 // Public interface
7160 
7161 VkResult vmaCreateAllocator(
7162  const VmaAllocatorCreateInfo* pCreateInfo,
7163  VmaAllocator* pAllocator)
7164 {
7165  VMA_ASSERT(pCreateInfo && pAllocator);
7166  VMA_DEBUG_LOG("vmaCreateAllocator");
7167  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
7168  return VK_SUCCESS;
7169 }
7170 
7171 void vmaDestroyAllocator(
7172  VmaAllocator allocator)
7173 {
7174  if(allocator != VK_NULL_HANDLE)
7175  {
7176  VMA_DEBUG_LOG("vmaDestroyAllocator");
7177  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
7178  vma_delete(&allocationCallbacks, allocator);
7179  }
7180 }
7181 
7182 void vmaGetPhysicalDeviceProperties(
7183  VmaAllocator allocator,
7184  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
7185 {
7186  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
7187  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
7188 }
7189 
7190 void vmaGetMemoryProperties(
7191  VmaAllocator allocator,
7192  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
7193 {
7194  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
7195  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
7196 }
7197 
7198 void vmaGetMemoryTypeProperties(
7199  VmaAllocator allocator,
7200  uint32_t memoryTypeIndex,
7201  VkMemoryPropertyFlags* pFlags)
7202 {
7203  VMA_ASSERT(allocator && pFlags);
7204  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
7205  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
7206 }
7207 
7208 void vmaSetCurrentFrameIndex(
7209  VmaAllocator allocator,
7210  uint32_t frameIndex)
7211 {
7212  VMA_ASSERT(allocator);
7213  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
7214 
7215  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7216 
7217  allocator->SetCurrentFrameIndex(frameIndex);
7218 }
7219 
7220 void vmaCalculateStats(
7221  VmaAllocator allocator,
7222  VmaStats* pStats)
7223 {
7224  VMA_ASSERT(allocator && pStats);
7225  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7226  allocator->CalculateStats(pStats);
7227 }
7228 
7229 #if VMA_STATS_STRING_ENABLED
7230 
7231 void vmaBuildStatsString(
7232  VmaAllocator allocator,
7233  char** ppStatsString,
7234  VkBool32 detailedMap)
7235 {
7236  VMA_ASSERT(allocator && ppStatsString);
7237  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7238 
7239  VmaStringBuilder sb(allocator);
7240  {
7241  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
7242  json.BeginObject();
7243 
7244  VmaStats stats;
7245  allocator->CalculateStats(&stats);
7246 
7247  json.WriteString("Total");
7248  VmaPrintStatInfo(json, stats.total);
7249 
7250  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
7251  {
7252  json.BeginString("Heap ");
7253  json.ContinueString(heapIndex);
7254  json.EndString();
7255  json.BeginObject();
7256 
7257  json.WriteString("Size");
7258  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
7259 
7260  json.WriteString("Flags");
7261  json.BeginArray(true);
7262  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
7263  {
7264  json.WriteString("DEVICE_LOCAL");
7265  }
7266  json.EndArray();
7267 
7268  if(stats.memoryHeap[heapIndex].blockCount > 0)
7269  {
7270  json.WriteString("Stats");
7271  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
7272  }
7273 
7274  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
7275  {
7276  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
7277  {
7278  json.BeginString("Type ");
7279  json.ContinueString(typeIndex);
7280  json.EndString();
7281 
7282  json.BeginObject();
7283 
7284  json.WriteString("Flags");
7285  json.BeginArray(true);
7286  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
7287  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
7288  {
7289  json.WriteString("DEVICE_LOCAL");
7290  }
7291  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
7292  {
7293  json.WriteString("HOST_VISIBLE");
7294  }
7295  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
7296  {
7297  json.WriteString("HOST_COHERENT");
7298  }
7299  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
7300  {
7301  json.WriteString("HOST_CACHED");
7302  }
7303  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
7304  {
7305  json.WriteString("LAZILY_ALLOCATED");
7306  }
7307  json.EndArray();
7308 
7309  if(stats.memoryType[typeIndex].blockCount > 0)
7310  {
7311  json.WriteString("Stats");
7312  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
7313  }
7314 
7315  json.EndObject();
7316  }
7317  }
7318 
7319  json.EndObject();
7320  }
7321  if(detailedMap == VK_TRUE)
7322  {
7323  allocator->PrintDetailedMap(json);
7324  }
7325 
7326  json.EndObject();
7327  }
7328 
7329  const size_t len = sb.GetLength();
7330  char* const pChars = vma_new_array(allocator, char, len + 1);
7331  if(len > 0)
7332  {
7333  memcpy(pChars, sb.GetData(), len);
7334  }
7335  pChars[len] = '\0';
7336  *ppStatsString = pChars;
7337 }
7338 
7339 void vmaFreeStatsString(
7340  VmaAllocator allocator,
7341  char* pStatsString)
7342 {
7343  if(pStatsString != VMA_NULL)
7344  {
7345  VMA_ASSERT(allocator);
7346  size_t len = strlen(pStatsString);
7347  vma_delete_array(allocator, pStatsString, len + 1);
7348  }
7349 }
7350 
7351 #endif // #if VMA_STATS_STRING_ENABLED
7352 
7355 VkResult vmaFindMemoryTypeIndex(
7356  VmaAllocator allocator,
7357  uint32_t memoryTypeBits,
7358  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7359  uint32_t* pMemoryTypeIndex)
7360 {
7361  VMA_ASSERT(allocator != VK_NULL_HANDLE);
7362  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
7363  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
7364 
7365  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
7366  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
7367  if(preferredFlags == 0)
7368  {
7369  preferredFlags = requiredFlags;
7370  }
7371  // preferredFlags, if not 0, must be a superset of requiredFlags.
7372  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
7373 
7374  // Convert usage to requiredFlags and preferredFlags.
7375  switch(pAllocationCreateInfo->usage)
7376  {
7377  case VMA_MEMORY_USAGE_UNKNOWN:
7378  break;
7379  case VMA_MEMORY_USAGE_GPU_ONLY:
7380  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7381  break;
7382  case VMA_MEMORY_USAGE_CPU_ONLY:
7383  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
7384  break;
7385  case VMA_MEMORY_USAGE_CPU_TO_GPU:
7386  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7387  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
7388  break;
7389  case VMA_MEMORY_USAGE_GPU_TO_CPU:
7390  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7391  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
7392  break;
7393  default:
7394  break;
7395  }
7396 
7397  if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0)
7398  {
7399  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
7400  }
7401 
7402  *pMemoryTypeIndex = UINT32_MAX;
7403  uint32_t minCost = UINT32_MAX;
7404  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
7405  memTypeIndex < allocator->GetMemoryTypeCount();
7406  ++memTypeIndex, memTypeBit <<= 1)
7407  {
7408  // This memory type is acceptable according to memoryTypeBits bitmask.
7409  if((memTypeBit & memoryTypeBits) != 0)
7410  {
7411  const VkMemoryPropertyFlags currFlags =
7412  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
7413  // This memory type contains requiredFlags.
7414  if((requiredFlags & ~currFlags) == 0)
7415  {
7416  // Calculate cost as number of bits from preferredFlags not present in this memory type.
7417  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
7418  // Remember memory type with lowest cost.
7419  if(currCost < minCost)
7420  {
7421  *pMemoryTypeIndex = memTypeIndex;
7422  if(currCost == 0)
7423  {
7424  return VK_SUCCESS;
7425  }
7426  minCost = currCost;
7427  }
7428  }
7429  }
7430  }
7431  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
7432 }
7433 
7434 VkResult vmaCreatePool(
7435  VmaAllocator allocator,
7436  const VmaPoolCreateInfo* pCreateInfo,
7437  VmaPool* pPool)
7438 {
7439  VMA_ASSERT(allocator && pCreateInfo && pPool);
7440 
7441  VMA_DEBUG_LOG("vmaCreatePool");
7442 
7443  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7444 
7445  return allocator->CreatePool(pCreateInfo, pPool);
7446 }
7447 
7448 void vmaDestroyPool(
7449  VmaAllocator allocator,
7450  VmaPool pool)
7451 {
7452  VMA_ASSERT(allocator && pool);
7453 
7454  VMA_DEBUG_LOG("vmaDestroyPool");
7455 
7456  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7457 
7458  allocator->DestroyPool(pool);
7459 }
7460 
7461 void vmaGetPoolStats(
7462  VmaAllocator allocator,
7463  VmaPool pool,
7464  VmaPoolStats* pPoolStats)
7465 {
7466  VMA_ASSERT(allocator && pool && pPoolStats);
7467 
7468  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7469 
7470  allocator->GetPoolStats(pool, pPoolStats);
7471 }
7472 
7473 void vmaMakePoolAllocationsLost(
7474  VmaAllocator allocator,
7475  VmaPool pool,
7476  size_t* pLostAllocationCount)
7477 {
7478  VMA_ASSERT(allocator && pool);
7479 
7480  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7481 
7482  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
7483 }
7484 
7485 VkResult vmaAllocateMemory(
7486  VmaAllocator allocator,
7487  const VkMemoryRequirements* pVkMemoryRequirements,
7488  const VmaAllocationCreateInfo* pCreateInfo,
7489  VmaAllocation* pAllocation,
7490  VmaAllocationInfo* pAllocationInfo)
7491 {
7492  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
7493 
7494  VMA_DEBUG_LOG("vmaAllocateMemory");
7495 
7496  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7497 
7498  VkResult result = allocator->AllocateMemory(
7499  *pVkMemoryRequirements,
7500  *pCreateInfo,
7501  VMA_SUBALLOCATION_TYPE_UNKNOWN,
7502  pAllocation);
7503 
7504  if(pAllocationInfo && result == VK_SUCCESS)
7505  {
7506  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7507  }
7508 
7509  return result;
7510 }
7511 
7512 VkResult vmaAllocateMemoryForBuffer(
7513  VmaAllocator allocator,
7514  VkBuffer buffer,
7515  const VmaAllocationCreateInfo* pCreateInfo,
7516  VmaAllocation* pAllocation,
7517  VmaAllocationInfo* pAllocationInfo)
7518 {
7519  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7520 
7521  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
7522 
7523  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7524 
7525  VkMemoryRequirements vkMemReq = {};
7526  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, buffer, &vkMemReq);
7527 
7528  VkResult result = allocator->AllocateMemory(
7529  vkMemReq,
7530  *pCreateInfo,
7531  VMA_SUBALLOCATION_TYPE_BUFFER,
7532  pAllocation);
7533 
7534  if(pAllocationInfo && result == VK_SUCCESS)
7535  {
7536  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7537  }
7538 
7539  return result;
7540 }
7541 
7542 VkResult vmaAllocateMemoryForImage(
7543  VmaAllocator allocator,
7544  VkImage image,
7545  const VmaAllocationCreateInfo* pCreateInfo,
7546  VmaAllocation* pAllocation,
7547  VmaAllocationInfo* pAllocationInfo)
7548 {
7549  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
7550 
7551  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
7552 
7553  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7554 
7555  VkResult result = AllocateMemoryForImage(
7556  allocator,
7557  image,
7558  pCreateInfo,
7559  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
7560  pAllocation);
7561 
7562  if(pAllocationInfo && result == VK_SUCCESS)
7563  {
7564  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7565  }
7566 
7567  return result;
7568 }
7569 
7570 void vmaFreeMemory(
7571  VmaAllocator allocator,
7572  VmaAllocation allocation)
7573 {
7574  VMA_ASSERT(allocator && allocation);
7575 
7576  VMA_DEBUG_LOG("vmaFreeMemory");
7577 
7578  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7579 
7580  allocator->FreeMemory(allocation);
7581 }
7582 
7583 void vmaGetAllocationInfo(
7584  VmaAllocator allocator,
7585  VmaAllocation allocation,
7586  VmaAllocationInfo* pAllocationInfo)
7587 {
7588  VMA_ASSERT(allocator && allocation && pAllocationInfo);
7589 
7590  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7591 
7592  allocator->GetAllocationInfo(allocation, pAllocationInfo);
7593 }
7594 
7595 void vmaSetAllocationUserData(
7596  VmaAllocator allocator,
7597  VmaAllocation allocation,
7598  void* pUserData)
7599 {
7600  VMA_ASSERT(allocator && allocation);
7601 
7602  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7603 
7604  allocation->SetUserData(pUserData);
7605 }
7606 
7607 void vmaCreateLostAllocation(
7608  VmaAllocator allocator,
7609  VmaAllocation* pAllocation)
7610 {
7611  VMA_ASSERT(allocator && pAllocation);
7612 
7613  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
7614 
7615  allocator->CreateLostAllocation(pAllocation);
7616 }
7617 
7618 VkResult vmaMapMemory(
7619  VmaAllocator allocator,
7620  VmaAllocation allocation,
7621  void** ppData)
7622 {
7623  VMA_ASSERT(allocator && allocation && ppData);
7624 
7625  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7626 
7627  return (*allocator->GetVulkanFunctions().vkMapMemory)(
7628  allocator->m_hDevice,
7629  allocation->GetMemory(),
7630  allocation->GetOffset(),
7631  allocation->GetSize(),
7632  0,
7633  ppData);
7634 }
7635 
7636 void vmaUnmapMemory(
7637  VmaAllocator allocator,
7638  VmaAllocation allocation)
7639 {
7640  VMA_ASSERT(allocator && allocation);
7641 
7642  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7643 
7644  (*allocator->GetVulkanFunctions().vkUnmapMemory)(allocator->m_hDevice, allocation->GetMemory());
7645 }
7646 
7647 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
7648 {
7649  VMA_ASSERT(allocator);
7650 
7651  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7652 
7653  allocator->UnmapPersistentlyMappedMemory();
7654 }
7655 
7656 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
7657 {
7658  VMA_ASSERT(allocator);
7659 
7660  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7661 
7662  return allocator->MapPersistentlyMappedMemory();
7663 }
7664 
7665 VkResult vmaDefragment(
7666  VmaAllocator allocator,
7667  VmaAllocation* pAllocations,
7668  size_t allocationCount,
7669  VkBool32* pAllocationsChanged,
7670  const VmaDefragmentationInfo *pDefragmentationInfo,
7671  VmaDefragmentationStats* pDefragmentationStats)
7672 {
7673  VMA_ASSERT(allocator && pAllocations);
7674 
7675  VMA_DEBUG_LOG("vmaDefragment");
7676 
7677  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7678 
7679  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
7680 }
7681 
7682 VkResult vmaCreateBuffer(
7683  VmaAllocator allocator,
7684  const VkBufferCreateInfo* pBufferCreateInfo,
7685  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7686  VkBuffer* pBuffer,
7687  VmaAllocation* pAllocation,
7688  VmaAllocationInfo* pAllocationInfo)
7689 {
7690  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
7691 
7692  VMA_DEBUG_LOG("vmaCreateBuffer");
7693 
7694  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7695 
7696  *pBuffer = VK_NULL_HANDLE;
7697  *pAllocation = VK_NULL_HANDLE;
7698 
7699  // 1. Create VkBuffer.
7700  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
7701  allocator->m_hDevice,
7702  pBufferCreateInfo,
7703  allocator->GetAllocationCallbacks(),
7704  pBuffer);
7705  if(res >= 0)
7706  {
7707  // 2. vkGetBufferMemoryRequirements.
7708  VkMemoryRequirements vkMemReq = {};
7709  (*allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements)(allocator->m_hDevice, *pBuffer, &vkMemReq);
7710 
7711  // 3. Allocate memory using allocator.
7712  res = allocator->AllocateMemory(
7713  vkMemReq,
7714  *pAllocationCreateInfo,
7715  VMA_SUBALLOCATION_TYPE_BUFFER,
7716  pAllocation);
7717  if(res >= 0)
7718  {
7719  // 3. Bind buffer with memory.
7720  res = (*allocator->GetVulkanFunctions().vkBindBufferMemory)(
7721  allocator->m_hDevice,
7722  *pBuffer,
7723  (*pAllocation)->GetMemory(),
7724  (*pAllocation)->GetOffset());
7725  if(res >= 0)
7726  {
7727  // All steps succeeded.
7728  if(pAllocationInfo != VMA_NULL)
7729  {
7730  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7731  }
7732  return VK_SUCCESS;
7733  }
7734  allocator->FreeMemory(*pAllocation);
7735  *pAllocation = VK_NULL_HANDLE;
7736  return res;
7737  }
7738  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
7739  *pBuffer = VK_NULL_HANDLE;
7740  return res;
7741  }
7742  return res;
7743 }
7744 
7745 void vmaDestroyBuffer(
7746  VmaAllocator allocator,
7747  VkBuffer buffer,
7748  VmaAllocation allocation)
7749 {
7750  if(buffer != VK_NULL_HANDLE)
7751  {
7752  VMA_ASSERT(allocator);
7753 
7754  VMA_DEBUG_LOG("vmaDestroyBuffer");
7755 
7756  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7757 
7758  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
7759 
7760  allocator->FreeMemory(allocation);
7761  }
7762 }
7763 
7764 VkResult vmaCreateImage(
7765  VmaAllocator allocator,
7766  const VkImageCreateInfo* pImageCreateInfo,
7767  const VmaAllocationCreateInfo* pAllocationCreateInfo,
7768  VkImage* pImage,
7769  VmaAllocation* pAllocation,
7770  VmaAllocationInfo* pAllocationInfo)
7771 {
7772  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
7773 
7774  VMA_DEBUG_LOG("vmaCreateImage");
7775 
7776  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7777 
7778  *pImage = VK_NULL_HANDLE;
7779  *pAllocation = VK_NULL_HANDLE;
7780 
7781  // 1. Create VkImage.
7782  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
7783  allocator->m_hDevice,
7784  pImageCreateInfo,
7785  allocator->GetAllocationCallbacks(),
7786  pImage);
7787  if(res >= 0)
7788  {
7789  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
7790  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
7791  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
7792 
7793  // 2. Allocate memory using allocator.
7794  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
7795  if(res >= 0)
7796  {
7797  // 3. Bind image with memory.
7798  res = (*allocator->GetVulkanFunctions().vkBindImageMemory)(
7799  allocator->m_hDevice,
7800  *pImage,
7801  (*pAllocation)->GetMemory(),
7802  (*pAllocation)->GetOffset());
7803  if(res >= 0)
7804  {
7805  // All steps succeeded.
7806  if(pAllocationInfo != VMA_NULL)
7807  {
7808  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
7809  }
7810  return VK_SUCCESS;
7811  }
7812  allocator->FreeMemory(*pAllocation);
7813  *pAllocation = VK_NULL_HANDLE;
7814  return res;
7815  }
7816  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
7817  *pImage = VK_NULL_HANDLE;
7818  return res;
7819  }
7820  return res;
7821 }
7822 
7823 void vmaDestroyImage(
7824  VmaAllocator allocator,
7825  VkImage image,
7826  VmaAllocation allocation)
7827 {
7828  if(image != VK_NULL_HANDLE)
7829  {
7830  VMA_ASSERT(allocator);
7831 
7832  VMA_DEBUG_LOG("vmaDestroyImage");
7833 
7834  VMA_DEBUG_GLOBAL_MUTEX_LOCK
7835 
7836  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
7837 
7838  allocator->FreeMemory(allocation);
7839  }
7840 }
7841 
7842 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:446
+
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:469
+
VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:800
vmaGetPoolStats
void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
Retrieves statistics of existing VmaPool object.
-
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:450
-
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for frequent writing on device and readback on host (download).
Definition: vk_mem_alloc.h:645
+
VmaVulkanFunctions::vkCreateBuffer
PFN_vkCreateBuffer vkCreateBuffer
Definition: vk_mem_alloc.h:456
+
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for frequent writing on device and readback on host (download).
Definition: vk_mem_alloc.h:651
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
-
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:444
-
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:922
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1075
+
VmaVulkanFunctions::vkMapMemory
PFN_vkMapMemory vkMapMemory
Definition: vk_mem_alloc.h:450
+
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:928
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:1081
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaCreateBuffer().
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation.
vmaUnmapPersistentlyMappedMemory
void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
Destroys Vulkan image and frees allocated memory.
-
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:846
+
VmaPoolStats::size
VkDeviceSize size
Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
Definition: vk_mem_alloc.h:852
VmaDefragmentationInfo
struct VmaDefragmentationInfo VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
-
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:694
-
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:727
-
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:403
+
VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
Definition: vk_mem_alloc.h:700
+
VmaAllocationCreateInfo::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:733
+
PFN_vmaFreeDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called before vkFreeMemory.
Definition: vk_mem_alloc.h:409
vmaMakePoolAllocationsLost
void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
-
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:475
-
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:796
-
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:522
-
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:457
-
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from small heaps <= 512 MB...
Definition: vk_mem_alloc.h:472
-
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:587
-
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:447
-
VmaAllocatorFlags
VkFlags VmaAllocatorFlags
Definition: vk_mem_alloc.h:433
-
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:586
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1079
-
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:492
-
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:596
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1087
-
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:710
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1070
-
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:448
+
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom CPU memory allocation callbacks.
Definition: vk_mem_alloc.h:481
+
VmaPoolCreateFlags
VkFlags VmaPoolCreateFlags
Definition: vk_mem_alloc.h:802
+
VmaAllocatorCreateInfo::pVulkanFunctions
const VmaVulkanFunctions * pVulkanFunctions
Pointers to Vulkan functions. Can be null if you leave define VMA_STATIC_VULKAN_FUNCTIONS 1...
Definition: vk_mem_alloc.h:528
+
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:463
+
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from small heaps <= 512 MB...
Definition: vk_mem_alloc.h:478
+
VmaStatInfo::allocationSizeMax
VkDeviceSize allocationSizeMax
Definition: vk_mem_alloc.h:593
+
VmaVulkanFunctions::vkBindImageMemory
PFN_vkBindImageMemory vkBindImageMemory
Definition: vk_mem_alloc.h:453
+
VmaAllocatorFlags
VkFlags VmaAllocatorFlags
Definition: vk_mem_alloc.h:439
+
VmaStatInfo::unusedBytes
VkDeviceSize unusedBytes
Total number of bytes occupied by unused ranges.
Definition: vk_mem_alloc.h:592
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:1085
+
VmaAllocatorCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:498
+
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:602
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:1093
+
VmaAllocationCreateInfo::flags
VmaAllocationCreateFlags flags
Use VmaAllocationCreateFlagBits enum.
Definition: vk_mem_alloc.h:716
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1076
+
VmaVulkanFunctions::vkGetBufferMemoryRequirements
PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements
Definition: vk_mem_alloc.h:454
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
-
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:466
-
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:800
+
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:472
+
VmaPoolCreateInfo
Describes parameter of created VmaPool.
Definition: vk_mem_alloc.h:806
VmaPoolStats
struct VmaPoolStats VmaPoolStats
Describes parameter of existing VmaPool.
-
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:932
+
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:938
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
-
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:445
+
VmaVulkanFunctions::vkUnmapMemory
PFN_vkUnmapMemory vkUnmapMemory
Definition: vk_mem_alloc.h:451
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
General purpose memory allocation.
-
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:729
-
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:816
-
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:852
-
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:803
+
VmaAllocationCreateInfo::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:735
+
VmaPoolCreateInfo::minBlockCount
size_t minBlockCount
Minimum number of blocks to be always allocated in this pool, even if they stay empty.
Definition: vk_mem_alloc.h:822
+
VmaPoolStats::allocationCount
size_t allocationCount
Number of VmaAllocation objects created from this pool that were not destroyed or lost...
Definition: vk_mem_alloc.h:858
+
VmaPoolCreateInfo::memoryTypeIndex
uint32_t memoryTypeIndex
Vulkan memory type index to allocate this pool from.
Definition: vk_mem_alloc.h:809
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
VmaVulkanFunctions
struct VmaVulkanFunctions VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
-
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:703
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1065
+
VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:709
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:1071
vmaCreatePool
VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
Allocates Vulkan device memory and creates VmaPool object.
-
VMA_POOL_CREATE_PERSISTENT_MAP_BIT
Definition: vk_mem_alloc.h:774
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1083
-
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:446
+
VMA_POOL_CREATE_PERSISTENT_MAP_BIT
Definition: vk_mem_alloc.h:780
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:1089
+
VmaVulkanFunctions::vkBindBufferMemory
PFN_vkBindBufferMemory vkBindBufferMemory
Definition: vk_mem_alloc.h:452
vmaSetCurrentFrameIndex
void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
Sets index of the current frame.
-
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:592
+
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:598
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
Function similar to vmaAllocateMemoryForBuffer().
-
VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:683
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1085
-
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:631
+
VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT
Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
Definition: vk_mem_alloc.h:689
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:1091
+
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:637
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
-
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:721
-
VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:429
+
VmaAllocationCreateInfo::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:727
+
VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:435
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
-
VmaAllocatorFlagBits
VmaAllocatorFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:424
+
VmaAllocatorFlagBits
VmaAllocatorFlagBits
Flags for created VmaAllocator.
Definition: vk_mem_alloc.h:430
vmaSetAllocationUserData
void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
Sets pUserData in given allocation to new value.
-
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:862
-
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:441
-
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:575
-
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:811
-
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:416
-
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:588
-
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:420
+
VmaPoolStats::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Size of the largest continuous free memory region.
Definition: vk_mem_alloc.h:868
+
VmaVulkanFunctions::vkGetPhysicalDeviceMemoryProperties
PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties
Definition: vk_mem_alloc.h:447
+
VmaStatInfo
Calculated statistics of memory usage in entire allocator.
Definition: vk_mem_alloc.h:581
+
VmaPoolCreateInfo::blockSize
VkDeviceSize blockSize
Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
Definition: vk_mem_alloc.h:817
+
VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
Definition: vk_mem_alloc.h:422
+
VmaStatInfo::unusedRangeSizeMin
VkDeviceSize unusedRangeSizeMin
Definition: vk_mem_alloc.h:594
+
VmaDeviceMemoryCallbacks::pfnFree
PFN_vmaFreeDeviceMemoryFunction pfnFree
Optional, can be null.
Definition: vk_mem_alloc.h:426
vmaMapPersistentlyMappedMemory
VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
-
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:806
+
VmaPoolCreateInfo::flags
VmaPoolCreateFlags flags
Use combination of VmaPoolCreateFlagBits.
Definition: vk_mem_alloc.h:812
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
-
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:397
-
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:716
-
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:707
-
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:578
-
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:443
-
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:824
-
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:478
-
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:855
-
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:734
-
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:510
-
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:594
-
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:587
+
PFN_vmaAllocateDeviceMemoryFunction
void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
Callback function called after successful vkAllocateMemory.
Definition: vk_mem_alloc.h:403
+
VmaAllocationCreateInfo::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:722
+
VmaAllocationCreateInfo
Definition: vk_mem_alloc.h:713
+
VmaStatInfo::blockCount
uint32_t blockCount
Number of VkDeviceMemory Vulkan memory blocks allocated.
Definition: vk_mem_alloc.h:584
+
VmaVulkanFunctions::vkFreeMemory
PFN_vkFreeMemory vkFreeMemory
Definition: vk_mem_alloc.h:449
+
VmaPoolCreateInfo::maxBlockCount
size_t maxBlockCount
Maximum number of blocks that can be allocated in this pool.
Definition: vk_mem_alloc.h:830
+
VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
Informative callbacks for vkAllocateMemory, vkFreeMemory.
Definition: vk_mem_alloc.h:484
+
VmaPoolStats::unusedRangeCount
size_t unusedRangeCount
Number of continuous memory ranges in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:861
+
VmaAllocationCreateInfo::pool
VmaPool pool
Pool that this allocation should be created in.
Definition: vk_mem_alloc.h:740
+
VmaAllocatorCreateInfo::pHeapSizeLimit
const VkDeviceSize * pHeapSizeLimit
Either NULL or a pointer to an array of limits on maximum number of bytes that can be allocated out o...
Definition: vk_mem_alloc.h:516
+
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:600
+
VmaStatInfo::allocationSizeMin
VkDeviceSize allocationSizeMin
Definition: vk_mem_alloc.h:593
VmaAllocationCreateInfo
struct VmaAllocationCreateInfo VmaAllocationCreateInfo
-
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:452
+
VmaVulkanFunctions::vkCreateImage
PFN_vkCreateImage vkCreateImage
Definition: vk_mem_alloc.h:458
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
-
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:418
-
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:701
-
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:451
-
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:838
-
VmaAllocatorCreateInfo::flags
VmaAllocatorFlags flags
Flags for created allocator. Use VmaAllocatorFlagBits enum.
Definition: vk_mem_alloc.h:460
+
VmaDeviceMemoryCallbacks::pfnAllocate
PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
Optional, can be null.
Definition: vk_mem_alloc.h:424
+
VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
Definition: vk_mem_alloc.h:707
+
VmaVulkanFunctions::vkDestroyBuffer
PFN_vkDestroyBuffer vkDestroyBuffer
Definition: vk_mem_alloc.h:457
+
VmaPoolCreateInfo::frameInUseCount
uint32_t frameInUseCount
Maximum number of additional frames that are in use at the same time as current frame.
Definition: vk_mem_alloc.h:844
+
VmaAllocatorCreateInfo::flags
VmaAllocatorFlags flags
Flags for created allocator. Use VmaAllocatorFlagBits enum.
Definition: vk_mem_alloc.h:466
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:943
-
VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:662
-
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps. ...
Definition: vk_mem_alloc.h:469
-
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:587
-
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:584
-
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:843
-
VMA_MEMORY_USAGE_CPU_ONLY
Memory will be mapped on host. Could be used for transfer to/from device.
Definition: vk_mem_alloc.h:639
+
VmaAllocationInfo::pUserData
void * pUserData
Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
Definition: vk_mem_alloc.h:949
+
VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:668
+
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Preferred size of a single VkDeviceMemory block to be allocated from large heaps. ...
Definition: vk_mem_alloc.h:475
+
VmaStatInfo::allocationSizeAvg
VkDeviceSize allocationSizeAvg
Definition: vk_mem_alloc.h:593
+
VmaStatInfo::usedBytes
VkDeviceSize usedBytes
Total number of bytes occupied by all allocations.
Definition: vk_mem_alloc.h:590
+
VmaPoolStats
Describes parameter of existing VmaPool.
Definition: vk_mem_alloc.h:849
+
VMA_MEMORY_USAGE_CPU_ONLY
Memory will be mapped on host. Could be used for transfer to/from device.
Definition: vk_mem_alloc.h:645
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
VmaStats
struct VmaStats VmaStats
General statistics from current state of Allocator.
-
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:927
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1081
+
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:933
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:1087
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
-
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:439
+
VmaVulkanFunctions
Pointers to some Vulkan functions - a subset used by the library.
Definition: vk_mem_alloc.h:445
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
-
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:582
-
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:705
-
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:580
-
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:449
-
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:453
-
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:765
-
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
Definition: vk_mem_alloc.h:938
+
VmaStatInfo::unusedRangeCount
uint32_t unusedRangeCount
Number of free ranges of memory between allocations.
Definition: vk_mem_alloc.h:588
+
VmaAllocationCreateFlags
VkFlags VmaAllocationCreateFlags
Definition: vk_mem_alloc.h:711
+
VmaStatInfo::allocationCount
uint32_t allocationCount
Number of VmaAllocation allocation objects allocated.
Definition: vk_mem_alloc.h:586
+
VmaVulkanFunctions::vkGetImageMemoryRequirements
PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements
Definition: vk_mem_alloc.h:455
+
VmaVulkanFunctions::vkDestroyImage
PFN_vkDestroyImage vkDestroyImage
Definition: vk_mem_alloc.h:459
+
VmaPoolCreateFlagBits
VmaPoolCreateFlagBits
Flags to be passed as VmaPoolCreateInfo::flags.
Definition: vk_mem_alloc.h:771
+
VmaAllocationInfo::pMappedData
void * pMappedData
Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
Definition: vk_mem_alloc.h:944
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
-
VMA_MEMORY_USAGE_UNKNOWN
No intended memory usage specified.
Definition: vk_mem_alloc.h:634
-
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:442
+
VMA_MEMORY_USAGE_UNKNOWN
No intended memory usage specified.
Definition: vk_mem_alloc.h:640
+
VmaVulkanFunctions::vkAllocateMemory
PFN_vkAllocateMemory vkAllocateMemory
Definition: vk_mem_alloc.h:448
vmaCreateLostAllocation
void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
Creates new allocation that is in lost state from the beginning.
-
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:646
-
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:908
-
VMA_MEMORY_USAGE_CPU_TO_GPU
Memory will be used for frequent (dynamic) updates from host and reads on device (upload).
Definition: vk_mem_alloc.h:642
-
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:650
-
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:588
-
VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:431
+
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:652
+
VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
Definition: vk_mem_alloc.h:914
+
VMA_MEMORY_USAGE_CPU_TO_GPU
Memory will be used for frequent (dynamic) updates from host and reads on device (upload).
Definition: vk_mem_alloc.h:648
+
VmaAllocationCreateFlagBits
VmaAllocationCreateFlagBits
Flags to be passed as VmaAllocationCreateInfo::flags.
Definition: vk_mem_alloc.h:656
+
VmaStatInfo::unusedRangeSizeAvg
VkDeviceSize unusedRangeSizeAvg
Definition: vk_mem_alloc.h:594
+
VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:437
VmaAllocationInfo
struct VmaAllocationInfo VmaAllocationInfo
Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
vmaGetMemoryTypeProperties
void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
Given Memory Type Index, returns Property Flags of this memory type.
-
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:673
-
VMA_MEMORY_USAGE_GPU_ONLY
Memory will be used on device only, so faster access from the device is preferred. No need to be mappable on host.
Definition: vk_mem_alloc.h:636
+
VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:679
+
VMA_MEMORY_USAGE_GPU_ONLY
Memory will be used on device only, so faster access from the device is preferred. No need to be mappable on host.
Definition: vk_mem_alloc.h:642
VmaStatInfo
struct VmaStatInfo VmaStatInfo
Calculated statistics of memory usage in entire allocator.
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
-
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:595
+
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:601
VmaDefragmentationStats
struct VmaDefragmentationStats VmaDefragmentationStats
Statistics returned by function vmaDefragment().
vmaDestroyPool
void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
Destroys VmaPool object and frees Vulkan device memory.
-
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:849
-
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:588
-
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:792
+
VmaPoolStats::unusedSize
VkDeviceSize unusedSize
Total number of bytes in the pool not used by any VmaAllocation.
Definition: vk_mem_alloc.h:855
+
VmaStatInfo::unusedRangeSizeMax
VkDeviceSize unusedRangeSizeMax
Definition: vk_mem_alloc.h:594
+
VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
Definition: vk_mem_alloc.h:798
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
Destroys Vulkan buffer and frees allocated memory.
-
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:913
+
VmaAllocationInfo::memoryType
uint32_t memoryType
Memory type index that this allocation was allocated from.
Definition: vk_mem_alloc.h:919
VmaPoolCreateInfo
struct VmaPoolCreateInfo VmaPoolCreateInfo
Describes parameter of created VmaPool.
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 73811d1..40e0cbf 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -33,9 +33,11 @@ extern "C" { Version 2.0.0-alpha.3 (2017-09-12) -Members grouped: see Modules. +Source repository: [VulkanMemoryAllocator project on GitHub](https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator) \n +Product page: [Vulkan Memory Allocator on GPUOpen](https://gpuopen.com/gaming-product/vulkan-memory-allocator/) -All members: see vk_mem_alloc.h. +Documentation of members grouped: Modules \n +Documentation of all members: vk_mem_alloc.h \section user_guide User guide