diff --git a/docs/html/group__layer1.html b/docs/html/group__layer1.html index a1f5fc5..80652d9 100644 --- a/docs/html/group__layer1.html +++ b/docs/html/group__layer1.html @@ -195,7 +195,7 @@ Functions

It makes no sense to set VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT and VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT at the same time.

VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT 

Set to use a memory that will be persistently mapped and retrieve pointer to it.

-

Pointer to mapped memory will be returned through ppMappedData. You cannot map the memory on your own as multiple maps of a single VkDeviceMemory are illegal.

+

Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. You cannot map the memory on your own as multiple maps of a single VkDeviceMemory are illegal.

VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM  diff --git a/docs/html/group__layer2.html b/docs/html/group__layer2.html index c03ac57..fb24f50 100644 --- a/docs/html/group__layer2.html +++ b/docs/html/group__layer2.html @@ -223,7 +223,7 @@ Functions
Parameters
- +
[out]pAllocationHandle to allocated memory.
[out]pAllocationInfoOptional. Information about allocated memory. It can be later fetched using function VmaGetAllocationInfo().
[out]pAllocationInfoOptional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
@@ -278,7 +278,7 @@ Functions
Parameters
- +
[out]pAllocationHandle to allocated memory.
[out]pAllocationInfoOptional. Information about allocated memory. It can be later fetched using function VmaGetAllocationInfo().
[out]pAllocationInfoOptional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
@@ -640,8 +640,11 @@ for(size_t i = 0; i < allocations.size(); ++i)

Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.

-

This is optional performance optimization. You should call it on Windows for time of call to vkQueueSubmit and vkQueuePresent, for performance reasons, because of the internal behavior of WDDM.

-

After this call VmaAllocationInfo::pMappedData of some allocations may become null.

+

This is optional performance optimization. On Windows you should call it before every call to vkQueueSubmit and vkQueuePresent. After which you can remap the allocations again using vmaMapPersistentlyMappedMemory(). This is because of the internal behavior of WDDM. Example:

+
vmaUnmapPersistentlyMappedMemory(allocator);
+vkQueueSubmit(...)
+vmaMapPersistentlyMappedMemory(allocator);
+

After this call VmaAllocationInfo::pMappedData of some allocations may become null.

This call is reference-counted. Memory is mapped again after you call vmaMapPersistentlyMappedMemory() same number of times that you called vmaUnmapPersistentlyMappedMemory().

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 461fcdd..733cebd 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,7 +62,7 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
161 #include <vulkan/vulkan.h>
162 
164 
168 VK_DEFINE_HANDLE(VmaAllocator)
169 
170 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
172  VmaAllocator allocator,
173  uint32_t memoryType,
174  VkDeviceMemory memory,
175  VkDeviceSize size);
177 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
178  VmaAllocator allocator,
179  uint32_t memoryType,
180  VkDeviceMemory memory,
181  VkDeviceSize size);
182 
188 typedef struct VmaDeviceMemoryCallbacks {
190  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
192  PFN_vmaFreeDeviceMemoryFunction pfnFree;
193 } VmaDeviceMemoryCallbacks;
194 
196 typedef enum VmaAllocatorFlagBits {
201  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
202 
203  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
204 } VmaAllocatorFlagBits;
205 typedef VkFlags VmaAllocatorFlags;
206 
208 typedef struct VmaAllocatorCreateInfo
209 {
211  VmaAllocatorFlags flags;
213 
214  VkPhysicalDevice physicalDevice;
216 
217  VkDevice device;
219 
220  VkDeviceSize preferredLargeHeapBlockSize;
222 
223  VkDeviceSize preferredSmallHeapBlockSize;
225 
226  const VkAllocationCallbacks* pAllocationCallbacks;
228 
229  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
230 } VmaAllocatorCreateInfo;
231 
233 VkResult vmaCreateAllocator(
234  const VmaAllocatorCreateInfo* pCreateInfo,
235  VmaAllocator* pAllocator);
236 
238 void vmaDestroyAllocator(
239  VmaAllocator allocator);
240 
245 void vmaGetPhysicalDeviceProperties(
246  VmaAllocator allocator,
247  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
248 
253 void vmaGetMemoryProperties(
254  VmaAllocator allocator,
255  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
256 
263 void vmaGetMemoryTypeProperties(
264  VmaAllocator allocator,
265  uint32_t memoryTypeIndex,
266  VkMemoryPropertyFlags* pFlags);
267 
268 typedef struct VmaStatInfo
269 {
270  uint32_t AllocationCount;
271  uint32_t SuballocationCount;
272  uint32_t UnusedRangeCount;
273  VkDeviceSize UsedBytes;
274  VkDeviceSize UnusedBytes;
275  VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax;
276  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
277 } VmaStatInfo;
278 
280 struct VmaStats
281 {
282  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
283  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
284  VmaStatInfo total;
285 };
286 
288 void vmaCalculateStats(
289  VmaAllocator allocator,
290  VmaStats* pStats);
291 
292 #define VMA_STATS_STRING_ENABLED 1
293 
294 #if VMA_STATS_STRING_ENABLED
295 
297 
299 void vmaBuildStatsString(
300  VmaAllocator allocator,
301  char** ppStatsString,
302  VkBool32 detailedMap);
303 
304 void vmaFreeStatsString(
305  VmaAllocator allocator,
306  char* pStatsString);
307 
308 #endif // #if VMA_STATS_STRING_ENABLED
309 
312 
317 typedef enum VmaMemoryUsage
318 {
320  VMA_MEMORY_USAGE_UNKNOWN = 0,
322  VMA_MEMORY_USAGE_GPU_ONLY = 1,
324 
325  VMA_MEMORY_USAGE_CPU_ONLY = 2,
327 
328  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
330 
331  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
332  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
333 } VmaMemoryUsage;
334 
336 typedef enum VmaMemoryRequirementFlagBits {
345  VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001,
346 
354  VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002,
361  VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004,
362 
363  VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
364 } VmaMemoryRequirementFlagBits;
365 typedef VkFlags VmaMemoryRequirementFlags;
366 
367 typedef struct VmaMemoryRequirements
368 {
369  VmaMemoryRequirementFlags flags;
374  VmaMemoryUsage usage;
378  VkMemoryPropertyFlags requiredFlags;
383  VkMemoryPropertyFlags preferredFlags;
385  void* pUserData;
386 } VmaMemoryRequirements;
387 
402 VkResult vmaFindMemoryTypeIndex(
403  VmaAllocator allocator,
404  uint32_t memoryTypeBits,
405  const VmaMemoryRequirements* pMemoryRequirements,
406  uint32_t* pMemoryTypeIndex);
407 
410 
415 VK_DEFINE_HANDLE(VmaAllocation)
416 
417 
419 typedef struct VmaAllocationInfo {
424  uint32_t memoryType;
431  VkDeviceMemory deviceMemory;
436  VkDeviceSize offset;
441  VkDeviceSize size;
447  void* pMappedData;
452  void* pUserData;
453 } VmaAllocationInfo;
454 
465 VkResult vmaAllocateMemory(
466  VmaAllocator allocator,
467  const VkMemoryRequirements* pVkMemoryRequirements,
468  const VmaMemoryRequirements* pVmaMemoryRequirements,
469  VmaAllocation* pAllocation,
470  VmaAllocationInfo* pAllocationInfo);
471 
478 VkResult vmaAllocateMemoryForBuffer(
479  VmaAllocator allocator,
480  VkBuffer buffer,
481  const VmaMemoryRequirements* pMemoryRequirements,
482  VmaAllocation* pAllocation,
483  VmaAllocationInfo* pAllocationInfo);
484 
486 VkResult vmaAllocateMemoryForImage(
487  VmaAllocator allocator,
488  VkImage image,
489  const VmaMemoryRequirements* pMemoryRequirements,
490  VmaAllocation* pAllocation,
491  VmaAllocationInfo* pAllocationInfo);
492 
494 void vmaFreeMemory(
495  VmaAllocator allocator,
496  VmaAllocation allocation);
497 
499 void vmaGetAllocationInfo(
500  VmaAllocator allocator,
501  VmaAllocation allocation,
502  VmaAllocationInfo* pAllocationInfo);
503 
505 void vmaSetAllocationUserData(
506  VmaAllocator allocator,
507  VmaAllocation allocation,
508  void* pUserData);
509 
518 VkResult vmaMapMemory(
519  VmaAllocator allocator,
520  VmaAllocation allocation,
521  void** ppData);
522 
523 void vmaUnmapMemory(
524  VmaAllocator allocator,
525  VmaAllocation allocation);
526 
539 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
540 
548 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
549 
551 typedef struct VmaDefragmentationInfo {
556  VkDeviceSize maxBytesToMove;
561  uint32_t maxAllocationsToMove;
562 } VmaDefragmentationInfo;
563 
565 typedef struct VmaDefragmentationStats {
567  VkDeviceSize bytesMoved;
569  VkDeviceSize bytesFreed;
571  uint32_t allocationsMoved;
573  uint32_t deviceMemoryBlocksFreed;
574 } VmaDefragmentationStats;
575 
646 VkResult vmaDefragment(
647  VmaAllocator allocator,
648  VmaAllocation* pAllocations,
649  size_t allocationCount,
650  VkBool32* pAllocationsChanged,
651  const VmaDefragmentationInfo *pDefragmentationInfo,
652  VmaDefragmentationStats* pDefragmentationStats);
653 
656 
676 VkResult vmaCreateBuffer(
677  VmaAllocator allocator,
678  const VkBufferCreateInfo* pCreateInfo,
679  const VmaMemoryRequirements* pMemoryRequirements,
680  VkBuffer* pBuffer,
681  VmaAllocation* pAllocation,
682  VmaAllocationInfo* pAllocationInfo);
683 
684 void vmaDestroyBuffer(
685  VmaAllocator allocator,
686  VkBuffer buffer,
687  VmaAllocation allocation);
688 
690 VkResult vmaCreateImage(
691  VmaAllocator allocator,
692  const VkImageCreateInfo* pCreateInfo,
693  const VmaMemoryRequirements* pMemoryRequirements,
694  VkImage* pImage,
695  VmaAllocation* pAllocation,
696  VmaAllocationInfo* pAllocationInfo);
697 
698 void vmaDestroyImage(
699  VmaAllocator allocator,
700  VkImage image,
701  VmaAllocation allocation);
702 
705 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
706 
707 #ifdef VMA_IMPLEMENTATION
708 #undef VMA_IMPLEMENTATION
709 
710 #include <cstdint>
711 #include <cstdlib>
712 
713 /*******************************************************************************
714 CONFIGURATION SECTION
715 
716 Define some of these macros before each #include of this header or change them
717 here if you need other then default behavior depending on your environment.
718 */
719 
720 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
721 //#define VMA_USE_STL_CONTAINERS 1
722 
723 /* Set this macro to 1 to make the library including and using STL containers:
724 std::pair, std::vector, std::list, std::unordered_map.
725 
726 Set it to 0 or undefined to make the library using its own implementation of
727 the containers.
728 */
729 #if VMA_USE_STL_CONTAINERS
730  #define VMA_USE_STL_VECTOR 1
731  #define VMA_USE_STL_UNORDERED_MAP 1
732  #define VMA_USE_STL_LIST 1
733 #endif
734 
735 #if VMA_USE_STL_VECTOR
736  #include <vector>
737 #endif
738 
739 #if VMA_USE_STL_UNORDERED_MAP
740  #include <unordered_map>
741 #endif
742 
743 #if VMA_USE_STL_LIST
744  #include <list>
745 #endif
746 
747 /*
748 Following headers are used in this CONFIGURATION section only, so feel free to
749 remove them if not needed.
750 */
751 #include <cassert> // for assert
752 #include <algorithm> // for min, max
753 #include <mutex> // for std::mutex
754 
755 #if !defined(_WIN32)
756  #include <malloc.h> // for aligned_alloc()
757 #endif
758 
759 // Normal assert to check for programmer's errors, especially in Debug configuration.
760 #ifndef VMA_ASSERT
761  #ifdef _DEBUG
762  #define VMA_ASSERT(expr) assert(expr)
763  #else
764  #define VMA_ASSERT(expr)
765  #endif
766 #endif
767 
768 // Assert that will be called very often, like inside data structures e.g. operator[].
769 // Making it non-empty can make program slow.
770 #ifndef VMA_HEAVY_ASSERT
771  #ifdef _DEBUG
772  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
773  #else
774  #define VMA_HEAVY_ASSERT(expr)
775  #endif
776 #endif
777 
778 #ifndef VMA_NULL
779  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
780  #define VMA_NULL nullptr
781 #endif
782 
783 #ifndef VMA_ALIGN_OF
784  #define VMA_ALIGN_OF(type) (__alignof(type))
785 #endif
786 
787 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
788  #if defined(_WIN32)
789  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
790  #else
791  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
792  #endif
793 #endif
794 
795 #ifndef VMA_SYSTEM_FREE
796  #if defined(_WIN32)
797  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
798  #else
799  #define VMA_SYSTEM_FREE(ptr) free(ptr)
800  #endif
801 #endif
802 
803 #ifndef VMA_MIN
804  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
805 #endif
806 
807 #ifndef VMA_MAX
808  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
809 #endif
810 
811 #ifndef VMA_SWAP
812  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
813 #endif
814 
815 #ifndef VMA_SORT
816  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
817 #endif
818 
819 #ifndef VMA_DEBUG_LOG
820  #define VMA_DEBUG_LOG(format, ...)
821  /*
822  #define VMA_DEBUG_LOG(format, ...) do { \
823  printf(format, __VA_ARGS__); \
824  printf("\n"); \
825  } while(false)
826  */
827 #endif
828 
829 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
830 #if VMA_STATS_STRING_ENABLED
831  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
832  {
833  _ultoa_s(num, outStr, strLen, 10);
834  }
835  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
836  {
837  _ui64toa_s(num, outStr, strLen, 10);
838  }
839 #endif
840 
841 #ifndef VMA_MUTEX
842  class VmaMutex
843  {
844  public:
845  VmaMutex() { }
846  ~VmaMutex() { }
847  void Lock() { m_Mutex.lock(); }
848  void Unlock() { m_Mutex.unlock(); }
849  private:
850  std::mutex m_Mutex;
851  };
852  #define VMA_MUTEX VmaMutex
853 #endif
854 
855 #ifndef VMA_BEST_FIT
856 
868  #define VMA_BEST_FIT (1)
869 #endif
870 
871 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
872 
876  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
877 #endif
878 
879 #ifndef VMA_DEBUG_ALIGNMENT
880 
884  #define VMA_DEBUG_ALIGNMENT (1)
885 #endif
886 
887 #ifndef VMA_DEBUG_MARGIN
888 
892  #define VMA_DEBUG_MARGIN (0)
893 #endif
894 
895 #ifndef VMA_DEBUG_GLOBAL_MUTEX
896 
900  #define VMA_DEBUG_GLOBAL_MUTEX (0)
901 #endif
902 
903 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
904 
908  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
909 #endif
910 
911 #ifndef VMA_SMALL_HEAP_MAX_SIZE
912  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
914 #endif
915 
916 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
917  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
919 #endif
920 
921 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
922  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
924 #endif
925 
926 /*******************************************************************************
927 END OF CONFIGURATION
928 */
929 
930 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
931  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
932 
933 // Returns number of bits set to 1 in (v).
934 static inline uint32_t CountBitsSet(uint32_t v)
935 {
936  uint32_t c = v - ((v >> 1) & 0x55555555);
937  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
938  c = ((c >> 4) + c) & 0x0F0F0F0F;
939  c = ((c >> 8) + c) & 0x00FF00FF;
940  c = ((c >> 16) + c) & 0x0000FFFF;
941  return c;
942 }
943 
944 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
945 // Use types like uint32_t, uint64_t as T.
946 template <typename T>
947 static inline T VmaAlignUp(T val, T align)
948 {
949  return (val + align - 1) / align * align;
950 }
951 
952 // Division with mathematical rounding to nearest number.
953 template <typename T>
954 inline T VmaRoundDiv(T x, T y)
955 {
956  return (x + (y / (T)2)) / y;
957 }
958 
959 #ifndef VMA_SORT
960 
961 template<typename Iterator, typename Compare>
962 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
963 {
964  Iterator centerValue = end; --centerValue;
965  Iterator insertIndex = beg;
966  for(Iterator i = beg; i < centerValue; ++i)
967  {
968  if(cmp(*i, *centerValue))
969  {
970  if(insertIndex != i)
971  {
972  VMA_SWAP(*i, *insertIndex);
973  }
974  ++insertIndex;
975  }
976  }
977  if(insertIndex != centerValue)
978  {
979  VMA_SWAP(*insertIndex, *centerValue);
980  }
981  return insertIndex;
982 }
983 
984 template<typename Iterator, typename Compare>
985 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
986 {
987  if(beg < end)
988  {
989  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
990  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
991  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
992  }
993 }
994 
995 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
996 
997 #endif // #ifndef VMA_SORT
998 
999 /*
1000 Returns true if two memory blocks occupy overlapping pages.
1001 ResourceA must be in less memory offset than ResourceB.
1002 
1003 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1004 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1005 */
1006 static inline bool VmaBlocksOnSamePage(
1007  VkDeviceSize resourceAOffset,
1008  VkDeviceSize resourceASize,
1009  VkDeviceSize resourceBOffset,
1010  VkDeviceSize pageSize)
1011 {
1012  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1013  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1014  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1015  VkDeviceSize resourceBStart = resourceBOffset;
1016  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1017  return resourceAEndPage == resourceBStartPage;
1018 }
1019 
1020 enum VmaSuballocationType
1021 {
1022  VMA_SUBALLOCATION_TYPE_FREE = 0,
1023  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1024  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1025  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1026  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1027  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1028  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1029 };
1030 
1031 /*
1032 Returns true if given suballocation types could conflict and must respect
1033 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1034 or linear image and another one is optimal image. If type is unknown, behave
1035 conservatively.
1036 */
1037 static inline bool VmaIsBufferImageGranularityConflict(
1038  VmaSuballocationType suballocType1,
1039  VmaSuballocationType suballocType2)
1040 {
1041  if(suballocType1 > suballocType2)
1042  {
1043  VMA_SWAP(suballocType1, suballocType2);
1044  }
1045 
1046  switch(suballocType1)
1047  {
1048  case VMA_SUBALLOCATION_TYPE_FREE:
1049  return false;
1050  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1051  return true;
1052  case VMA_SUBALLOCATION_TYPE_BUFFER:
1053  return
1054  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1055  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1056  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1057  return
1058  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1059  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1060  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1061  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1062  return
1063  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1064  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1065  return false;
1066  default:
1067  VMA_ASSERT(0);
1068  return true;
1069  }
1070 }
1071 
1072 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1073 struct VmaMutexLock
1074 {
1075 public:
1076  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1077  m_pMutex(useMutex ? &mutex : VMA_NULL)
1078  {
1079  if(m_pMutex)
1080  {
1081  m_pMutex->Lock();
1082  }
1083  }
1084 
1085  ~VmaMutexLock()
1086  {
1087  if(m_pMutex)
1088  {
1089  m_pMutex->Unlock();
1090  }
1091  }
1092 
1093 private:
1094  VMA_MUTEX* m_pMutex;
1095 };
1096 
1097 #if VMA_DEBUG_GLOBAL_MUTEX
1098  static VMA_MUTEX gDebugGlobalMutex;
1099  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex);
1100 #else
1101  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1102 #endif
1103 
1104 // Minimum size of a free suballocation to register it in the free suballocation collection.
1105 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1106 
1107 /*
1108 Performs binary search and returns iterator to first element that is greater or
1109 equal to (key), according to comparison (cmp).
1110 
1111 Cmp should return true if first argument is less than second argument.
1112 
1113 Returned value is the found element, if present in the collection or place where
1114 new element with value (key) should be inserted.
1115 */
1116 template <typename IterT, typename KeyT, typename CmpT>
1117 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1118 {
1119  size_t down = 0, up = (end - beg);
1120  while(down < up)
1121  {
1122  const size_t mid = (down + up) / 2;
1123  if(cmp(*(beg+mid), key))
1124  {
1125  down = mid + 1;
1126  }
1127  else
1128  {
1129  up = mid;
1130  }
1131  }
1132  return beg + down;
1133 }
1134 
1136 // Memory allocation
1137 
1138 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1139 {
1140  if((pAllocationCallbacks != VMA_NULL) &&
1141  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1142  {
1143  return (*pAllocationCallbacks->pfnAllocation)(
1144  pAllocationCallbacks->pUserData,
1145  size,
1146  alignment,
1147  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1148  }
1149  else
1150  {
1151  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1152  }
1153 }
1154 
1155 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1156 {
1157  if((pAllocationCallbacks != VMA_NULL) &&
1158  (pAllocationCallbacks->pfnFree != VMA_NULL))
1159  {
1160  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1161  }
1162  else
1163  {
1164  VMA_SYSTEM_FREE(ptr);
1165  }
1166 }
1167 
1168 template<typename T>
1169 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1170 {
1171  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1172 }
1173 
1174 template<typename T>
1175 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1176 {
1177  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1178 }
1179 
1180 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1181 
1182 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1183 
1184 template<typename T>
1185 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1186 {
1187  ptr->~T();
1188  VmaFree(pAllocationCallbacks, ptr);
1189 }
1190 
1191 template<typename T>
1192 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1193 {
1194  if(ptr != VMA_NULL)
1195  {
1196  for(size_t i = count; i--; )
1197  {
1198  ptr[i].~T();
1199  }
1200  VmaFree(pAllocationCallbacks, ptr);
1201  }
1202 }
1203 
1204 // STL-compatible allocator.
1205 template<typename T>
1206 class VmaStlAllocator
1207 {
1208 public:
1209  const VkAllocationCallbacks* const m_pCallbacks;
1210  typedef T value_type;
1211 
1212  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1213  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1214 
1215  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1216  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1217 
1218  template<typename U>
1219  bool operator==(const VmaStlAllocator<U>& rhs) const
1220  {
1221  return m_pCallbacks == rhs.m_pCallbacks;
1222  }
1223  template<typename U>
1224  bool operator!=(const VmaStlAllocator<U>& rhs) const
1225  {
1226  return m_pCallbacks != rhs.m_pCallbacks;
1227  }
1228 
1229  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1230 };
1231 
1232 #if VMA_USE_STL_VECTOR
1233 
1234 #define VmaVector std::vector
1235 
1236 template<typename T, typename allocatorT>
1237 static void VectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1238 {
1239  vec.insert(vec.begin() + index, item);
1240 }
1241 
1242 template<typename T, typename allocatorT>
1243 static void VectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1244 {
1245  vec.erase(vec.begin() + index);
1246 }
1247 
1248 #else // #if VMA_USE_STL_VECTOR
1249 
1250 /* Class with interface compatible with subset of std::vector.
1251 T must be POD because constructors and destructors are not called and memcpy is
1252 used for these objects. */
1253 template<typename T, typename AllocatorT>
1254 class VmaVector
1255 {
1256 public:
1257  VmaVector(const AllocatorT& allocator) :
1258  m_Allocator(allocator),
1259  m_pArray(VMA_NULL),
1260  m_Count(0),
1261  m_Capacity(0)
1262  {
1263  }
1264 
1265  VmaVector(size_t count, const AllocatorT& allocator) :
1266  m_Allocator(allocator),
1267  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, count) : VMA_NULL),
1268  m_Count(count),
1269  m_Capacity(count)
1270  {
1271  }
1272 
1273  VmaVector(const VmaVector<T, AllocatorT>& src) :
1274  m_Allocator(src.m_Allocator),
1275  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src->m_pCallbacks, src.m_Count) : VMA_NULL),
1276  m_Count(src.m_Count),
1277  m_Capacity(src.m_Count)
1278  {
1279  if(m_Count != 0)
1280  {
1281  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1282  }
1283  }
1284 
1285  ~VmaVector()
1286  {
1287  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1288  }
1289 
1290  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1291  {
1292  if(&rhs != this)
1293  {
1294  Resize(rhs.m_Count);
1295  if(m_Count != 0)
1296  {
1297  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1298  }
1299  }
1300  return *this;
1301  }
1302 
1303  bool empty() const { return m_Count == 0; }
1304  size_t size() const { return m_Count; }
1305  T* data() { return m_pArray; }
1306  const T* data() const { return m_pArray; }
1307 
1308  T& operator[](size_t index)
1309  {
1310  VMA_HEAVY_ASSERT(index < m_Count);
1311  return m_pArray[index];
1312  }
1313  const T& operator[](size_t index) const
1314  {
1315  VMA_HEAVY_ASSERT(index < m_Count);
1316  return m_pArray[index];
1317  }
1318 
1319  T& front()
1320  {
1321  VMA_HEAVY_ASSERT(m_Count > 0);
1322  return m_pArray[0];
1323  }
1324  const T& front() const
1325  {
1326  VMA_HEAVY_ASSERT(m_Count > 0);
1327  return m_pArray[0];
1328  }
1329  T& back()
1330  {
1331  VMA_HEAVY_ASSERT(m_Count > 0);
1332  return m_pArray[m_Count - 1];
1333  }
1334  const T& back() const
1335  {
1336  VMA_HEAVY_ASSERT(m_Count > 0);
1337  return m_pArray[m_Count - 1];
1338  }
1339 
1340  void reserve(size_t newCapacity, bool freeMemory = false)
1341  {
1342  newCapacity = VMA_MAX(newCapacity, m_Count);
1343 
1344  if((newCapacity < m_Capacity) && !freeMemory)
1345  {
1346  newCapacity = m_Capacity;
1347  }
1348 
1349  if(newCapacity != m_Capacity)
1350  {
1351  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1352  if(m_Count != 0)
1353  {
1354  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1355  }
1356  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1357  m_Capacity = newCapacity;
1358  m_pArray = newArray;
1359  }
1360  }
1361 
1362  void resize(size_t newCount, bool freeMemory = false)
1363  {
1364  size_t newCapacity = m_Capacity;
1365  if(newCount > m_Capacity)
1366  {
1367  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1368  }
1369  else if(freeMemory)
1370  {
1371  newCapacity = newCount;
1372  }
1373 
1374  if(newCapacity != m_Capacity)
1375  {
1376  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1377  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1378  if(elementsToCopy != 0)
1379  {
1380  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1381  }
1382  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1383  m_Capacity = newCapacity;
1384  m_pArray = newArray;
1385  }
1386 
1387  m_Count = newCount;
1388  }
1389 
1390  void clear(bool freeMemory = false)
1391  {
1392  resize(0, freeMemory);
1393  }
1394 
1395  void insert(size_t index, const T& src)
1396  {
1397  VMA_HEAVY_ASSERT(index <= m_Count);
1398  const size_t oldCount = size();
1399  resize(oldCount + 1);
1400  if(index < oldCount)
1401  {
1402  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1403  }
1404  m_pArray[index] = src;
1405  }
1406 
1407  void remove(size_t index)
1408  {
1409  VMA_HEAVY_ASSERT(index < m_Count);
1410  const size_t oldCount = size();
1411  if(index < oldCount - 1)
1412  {
1413  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1414  }
1415  resize(oldCount - 1);
1416  }
1417 
1418  void push_back(const T& src)
1419  {
1420  const size_t newIndex = size();
1421  resize(newIndex + 1);
1422  m_pArray[newIndex] = src;
1423  }
1424 
1425  void pop_back()
1426  {
1427  VMA_HEAVY_ASSERT(m_Count > 0);
1428  resize(size() - 1);
1429  }
1430 
1431  void push_front(const T& src)
1432  {
1433  insert(0, src);
1434  }
1435 
1436  void pop_front()
1437  {
1438  VMA_HEAVY_ASSERT(m_Count > 0);
1439  remove(0);
1440  }
1441 
1442  typedef T* iterator;
1443 
1444  iterator begin() { return m_pArray; }
1445  iterator end() { return m_pArray + m_Count; }
1446 
1447 private:
1448  AllocatorT m_Allocator;
1449  T* m_pArray;
1450  size_t m_Count;
1451  size_t m_Capacity;
1452 };
1453 
1454 template<typename T, typename allocatorT>
1455 static void VectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
1456 {
1457  vec.insert(index, item);
1458 }
1459 
1460 template<typename T, typename allocatorT>
1461 static void VectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
1462 {
1463  vec.remove(index);
1464 }
1465 
1466 #endif // #if VMA_USE_STL_VECTOR
1467 
1469 // class VmaPoolAllocator
1470 
1471 /*
1472 Allocator for objects of type T using a list of arrays (pools) to speed up
1473 allocation. Number of elements that can be allocated is not bounded because
1474 allocator can create multiple blocks.
1475 */
1476 template<typename T>
1477 class VmaPoolAllocator
1478 {
1479 public:
1480  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
1481  ~VmaPoolAllocator();
1482  void Clear();
1483  T* Alloc();
1484  void Free(T* ptr);
1485 
1486 private:
1487  union Item
1488  {
1489  uint32_t NextFreeIndex;
1490  T Value;
1491  };
1492 
1493  struct ItemBlock
1494  {
1495  Item* pItems;
1496  uint32_t FirstFreeIndex;
1497  };
1498 
1499  const VkAllocationCallbacks* m_pAllocationCallbacks;
1500  size_t m_ItemsPerBlock;
1501  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
1502 
1503  ItemBlock& CreateNewBlock();
1504 };
1505 
1506 template<typename T>
1507 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
1508  m_pAllocationCallbacks(pAllocationCallbacks),
1509  m_ItemsPerBlock(itemsPerBlock),
1510  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
1511 {
1512  VMA_ASSERT(itemsPerBlock > 0);
1513 }
1514 
1515 template<typename T>
1516 VmaPoolAllocator<T>::~VmaPoolAllocator()
1517 {
1518  Clear();
1519 }
1520 
1521 template<typename T>
1522 void VmaPoolAllocator<T>::Clear()
1523 {
1524  for(size_t i = m_ItemBlocks.size(); i--; )
1525  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
1526  m_ItemBlocks.clear();
1527 }
1528 
1529 template<typename T>
1530 T* VmaPoolAllocator<T>::Alloc()
1531 {
1532  for(size_t i = m_ItemBlocks.size(); i--; )
1533  {
1534  ItemBlock& block = m_ItemBlocks[i];
1535  // This block has some free items: Use first one.
1536  if(block.FirstFreeIndex != UINT32_MAX)
1537  {
1538  Item* const pItem = &block.pItems[block.FirstFreeIndex];
1539  block.FirstFreeIndex = pItem->NextFreeIndex;
1540  return &pItem->Value;
1541  }
1542  }
1543 
1544  // No block has free item: Create new one and use it.
1545  ItemBlock& newBlock = CreateNewBlock();
1546  Item* const pItem = &newBlock.pItems[0];
1547  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
1548  return &pItem->Value;
1549 }
1550 
1551 template<typename T>
1552 void VmaPoolAllocator<T>::Free(T* ptr)
1553 {
1554  // Search all memory blocks to find ptr.
1555  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
1556  {
1557  ItemBlock& block = m_ItemBlocks[i];
1558 
1559  // Casting to union.
1560  Item* pItemPtr;
1561  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
1562 
1563  // Check if pItemPtr is in address range of this block.
1564  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
1565  {
1566  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
1567  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
1568  block.FirstFreeIndex = index;
1569  return;
1570  }
1571  }
1572  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
1573 }
1574 
1575 template<typename T>
1576 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
1577 {
1578  ItemBlock newBlock = {
1579  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
1580 
1581  m_ItemBlocks.push_back(newBlock);
1582 
1583  // Setup singly-linked list of all free items in this block.
1584  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
1585  newBlock.pItems[i].NextFreeIndex = i + 1;
1586  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
1587  return m_ItemBlocks.back();
1588 }
1589 
1591 // class VmaRawList, VmaList
1592 
1593 #if VMA_USE_STL_LIST
1594 
1595 #define VmaList std::list
1596 
1597 #else // #if VMA_USE_STL_LIST
1598 
1599 template<typename T>
1600 struct VmaListItem
1601 {
1602  VmaListItem* pPrev;
1603  VmaListItem* pNext;
1604  T Value;
1605 };
1606 
1607 // Doubly linked list.
1608 template<typename T>
1609 class VmaRawList
1610 {
1611 public:
1612  typedef VmaListItem<T> ItemType;
1613 
1614  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
1615  ~VmaRawList();
1616  void Clear();
1617 
1618  size_t GetCount() const { return m_Count; }
1619  bool IsEmpty() const { return m_Count == 0; }
1620 
1621  ItemType* Front() { return m_pFront; }
1622  const ItemType* Front() const { return m_pFront; }
1623  ItemType* Back() { return m_pBack; }
1624  const ItemType* Back() const { return m_pBack; }
1625 
1626  ItemType* PushBack();
1627  ItemType* PushFront();
1628  ItemType* PushBack(const T& value);
1629  ItemType* PushFront(const T& value);
1630  void PopBack();
1631  void PopFront();
1632 
1633  // Item can be null - it means PushBack.
1634  ItemType* InsertBefore(ItemType* pItem);
1635  // Item can be null - it means PushFront.
1636  ItemType* InsertAfter(ItemType* pItem);
1637 
1638  ItemType* InsertBefore(ItemType* pItem, const T& value);
1639  ItemType* InsertAfter(ItemType* pItem, const T& value);
1640 
1641  void Remove(ItemType* pItem);
1642 
1643 private:
1644  const VkAllocationCallbacks* const m_pAllocationCallbacks;
1645  VmaPoolAllocator<ItemType> m_ItemAllocator;
1646  ItemType* m_pFront;
1647  ItemType* m_pBack;
1648  size_t m_Count;
1649 
1650  // Declared not defined, to block copy constructor and assignment operator.
1651  VmaRawList(const VmaRawList<T>& src);
1652  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
1653 };
1654 
1655 template<typename T>
1656 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
1657  m_pAllocationCallbacks(pAllocationCallbacks),
1658  m_ItemAllocator(pAllocationCallbacks, 128),
1659  m_pFront(VMA_NULL),
1660  m_pBack(VMA_NULL),
1661  m_Count(0)
1662 {
1663 }
1664 
1665 template<typename T>
1666 VmaRawList<T>::~VmaRawList()
1667 {
1668  // Intentionally not calling Clear, because that would be unnecessary
1669  // computations to return all items to m_ItemAllocator as free.
1670 }
1671 
1672 template<typename T>
1673 void VmaRawList<T>::Clear()
1674 {
1675  if(IsEmpty() == false)
1676  {
1677  ItemType* pItem = m_pBack;
1678  while(pItem != VMA_NULL)
1679  {
1680  ItemType* const pPrevItem = pItem->pPrev;
1681  m_ItemAllocator.Free(pItem);
1682  pItem = pPrevItem;
1683  }
1684  m_pFront = VMA_NULL;
1685  m_pBack = VMA_NULL;
1686  m_Count = 0;
1687  }
1688 }
1689 
1690 template<typename T>
1691 VmaListItem<T>* VmaRawList<T>::PushBack()
1692 {
1693  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1694  pNewItem->pNext = VMA_NULL;
1695  if(IsEmpty())
1696  {
1697  pNewItem->pPrev = VMA_NULL;
1698  m_pFront = pNewItem;
1699  m_pBack = pNewItem;
1700  m_Count = 1;
1701  }
1702  else
1703  {
1704  pNewItem->pPrev = m_pBack;
1705  m_pBack->pNext = pNewItem;
1706  m_pBack = pNewItem;
1707  ++m_Count;
1708  }
1709  return pNewItem;
1710 }
1711 
1712 template<typename T>
1713 VmaListItem<T>* VmaRawList<T>::PushFront()
1714 {
1715  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1716  pNewItem->pPrev = VMA_NULL;
1717  if(IsEmpty())
1718  {
1719  pNewItem->pNext = VMA_NULL;
1720  m_pFront = pNewItem;
1721  m_pBack = pNewItem;
1722  m_Count = 1;
1723  }
1724  else
1725  {
1726  pNewItem->pNext = m_pFront;
1727  m_pFront->pPrev = pNewItem;
1728  m_pFront = pNewItem;
1729  ++m_Count;
1730  }
1731  return pNewItem;
1732 }
1733 
1734 template<typename T>
1735 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
1736 {
1737  ItemType* const pNewItem = PushBack();
1738  pNewItem->Value = value;
1739  return pNewItem;
1740 }
1741 
1742 template<typename T>
1743 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
1744 {
1745  ItemType* const pNewItem = PushFront();
1746  pNewItem->Value = value;
1747  return pNewItem;
1748 }
1749 
1750 template<typename T>
1751 void VmaRawList<T>::PopBack()
1752 {
1753  VMA_HEAVY_ASSERT(m_Count > 0);
1754  ItemType* const pBackItem = m_pBack;
1755  ItemType* const pPrevItem = pBackItem->pPrev;
1756  if(pPrevItem != VMA_NULL)
1757  {
1758  pPrevItem->pNext = VMA_NULL;
1759  }
1760  m_pBack = pPrevItem;
1761  m_ItemAllocator.Free(pBackItem);
1762  --m_Count;
1763 }
1764 
1765 template<typename T>
1766 void VmaRawList<T>::PopFront()
1767 {
1768  VMA_HEAVY_ASSERT(m_Count > 0);
1769  ItemType* const pFrontItem = m_pFront;
1770  ItemType* const pNextItem = pFrontItem->pNext;
1771  if(pNextItem != VMA_NULL)
1772  {
1773  pNextItem->pPrev = VMA_NULL;
1774  }
1775  m_pFront = pNextItem;
1776  m_ItemAllocator.Free(pFrontItem);
1777  --m_Count;
1778 }
1779 
1780 template<typename T>
1781 void VmaRawList<T>::Remove(ItemType* pItem)
1782 {
1783  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
1784  VMA_HEAVY_ASSERT(m_Count > 0);
1785 
1786  if(pItem->pPrev != VMA_NULL)
1787  {
1788  pItem->pPrev->pNext = pItem->pNext;
1789  }
1790  else
1791  {
1792  VMA_HEAVY_ASSERT(m_pFront == pItem);
1793  m_pFront = pItem->pNext;
1794  }
1795 
1796  if(pItem->pNext != VMA_NULL)
1797  {
1798  pItem->pNext->pPrev = pItem->pPrev;
1799  }
1800  else
1801  {
1802  VMA_HEAVY_ASSERT(m_pBack == pItem);
1803  m_pBack = pItem->pPrev;
1804  }
1805 
1806  m_ItemAllocator.Free(pItem);
1807  --m_Count;
1808 }
1809 
1810 template<typename T>
1811 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
1812 {
1813  if(pItem != VMA_NULL)
1814  {
1815  ItemType* const prevItem = pItem->pPrev;
1816  ItemType* const newItem = m_ItemAllocator.Alloc();
1817  newItem->pPrev = prevItem;
1818  newItem->pNext = pItem;
1819  pItem->pPrev = newItem;
1820  if(prevItem != VMA_NULL)
1821  {
1822  prevItem->pNext = newItem;
1823  }
1824  else
1825  {
1826  VMA_HEAVY_ASSERT(m_pFront == pItem);
1827  m_pFront = newItem;
1828  }
1829  ++m_Count;
1830  return newItem;
1831  }
1832  else
1833  return PushBack();
1834 }
1835 
1836 template<typename T>
1837 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
1838 {
1839  if(pItem != VMA_NULL)
1840  {
1841  ItemType* const nextItem = pItem->pNext;
1842  ItemType* const newItem = m_ItemAllocator.Alloc();
1843  newItem->pNext = nextItem;
1844  newItem->pPrev = pItem;
1845  pItem->pNext = newItem;
1846  if(nextItem != VMA_NULL)
1847  {
1848  nextItem->pPrev = newItem;
1849  }
1850  else
1851  {
1852  VMA_HEAVY_ASSERT(m_pBack == pItem);
1853  m_pBack = newItem;
1854  }
1855  ++m_Count;
1856  return newItem;
1857  }
1858  else
1859  return PushFront();
1860 }
1861 
1862 template<typename T>
1863 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
1864 {
1865  ItemType* const newItem = InsertBefore(pItem);
1866  newItem->Value = value;
1867  return newItem;
1868 }
1869 
1870 template<typename T>
1871 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
1872 {
1873  ItemType* const newItem = InsertAfter(pItem);
1874  newItem->Value = value;
1875  return newItem;
1876 }
1877 
1878 template<typename T, typename AllocatorT>
1879 class VmaList
1880 {
1881 public:
1882  class iterator
1883  {
1884  public:
1885  iterator() :
1886  m_pList(VMA_NULL),
1887  m_pItem(VMA_NULL)
1888  {
1889  }
1890 
1891  T& operator*() const
1892  {
1893  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1894  return m_pItem->Value;
1895  }
1896  T* operator->() const
1897  {
1898  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1899  return &m_pItem->Value;
1900  }
1901 
1902  iterator& operator++()
1903  {
1904  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1905  m_pItem = m_pItem->pNext;
1906  return *this;
1907  }
1908  iterator& operator--()
1909  {
1910  if(m_pItem != VMA_NULL)
1911  {
1912  m_pItem = m_pItem->pPrev;
1913  }
1914  else
1915  {
1916  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
1917  m_pItem = m_pList->Back();
1918  }
1919  return *this;
1920  }
1921 
1922  iterator operator++(int)
1923  {
1924  iterator result = *this;
1925  ++*this;
1926  return result;
1927  }
1928  iterator operator--(int)
1929  {
1930  iterator result = *this;
1931  --*this;
1932  return result;
1933  }
1934 
1935  bool operator==(const iterator& rhs) const
1936  {
1937  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1938  return m_pItem == rhs.m_pItem;
1939  }
1940  bool operator!=(const iterator& rhs) const
1941  {
1942  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1943  return m_pItem != rhs.m_pItem;
1944  }
1945 
1946  private:
1947  VmaRawList<T>* m_pList;
1948  VmaListItem<T>* m_pItem;
1949 
1950  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
1951  m_pList(pList),
1952  m_pItem(pItem)
1953  {
1954  }
1955 
1956  friend class VmaList<T, AllocatorT>;
1957  friend class VmaList<T, AllocatorT>:: const_iterator;
1958  };
1959 
1960  class const_iterator
1961  {
1962  public:
1963  const_iterator() :
1964  m_pList(VMA_NULL),
1965  m_pItem(VMA_NULL)
1966  {
1967  }
1968 
1969  const_iterator(const iterator& src) :
1970  m_pList(src.m_pList),
1971  m_pItem(src.m_pItem)
1972  {
1973  }
1974 
1975  const T& operator*() const
1976  {
1977  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1978  return m_pItem->Value;
1979  }
1980  const T* operator->() const
1981  {
1982  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1983  return &m_pItem->Value;
1984  }
1985 
1986  const_iterator& operator++()
1987  {
1988  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1989  m_pItem = m_pItem->pNext;
1990  return *this;
1991  }
1992  const_iterator& operator--()
1993  {
1994  if(m_pItem != VMA_NULL)
1995  {
1996  m_pItem = m_pItem->pPrev;
1997  }
1998  else
1999  {
2000  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2001  m_pItem = m_pList->Back();
2002  }
2003  return *this;
2004  }
2005 
2006  const_iterator operator++(int)
2007  {
2008  const_iterator result = *this;
2009  ++*this;
2010  return result;
2011  }
2012  const_iterator operator--(int)
2013  {
2014  const_iterator result = *this;
2015  --*this;
2016  return result;
2017  }
2018 
2019  bool operator==(const const_iterator& rhs) const
2020  {
2021  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2022  return m_pItem == rhs.m_pItem;
2023  }
2024  bool operator!=(const const_iterator& rhs) const
2025  {
2026  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2027  return m_pItem != rhs.m_pItem;
2028  }
2029 
2030  private:
2031  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2032  m_pList(pList),
2033  m_pItem(pItem)
2034  {
2035  }
2036 
2037  const VmaRawList<T>* m_pList;
2038  const VmaListItem<T>* m_pItem;
2039 
2040  friend class VmaList<T, AllocatorT>;
2041  };
2042 
2043  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2044 
2045  bool empty() const { return m_RawList.IsEmpty(); }
2046  size_t size() const { return m_RawList.GetCount(); }
2047 
2048  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2049  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2050 
2051  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2052  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2053 
2054  void clear() { m_RawList.Clear(); }
2055  void push_back(const T& value) { m_RawList.PushBack(value); }
2056  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2057  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2058 
2059 private:
2060  VmaRawList<T> m_RawList;
2061 };
2062 
2063 #endif // #if VMA_USE_STL_LIST
2064 
2066 // class VmaMap
2067 
2068 #if VMA_USE_STL_UNORDERED_MAP
2069 
2070 #define VmaPair std::pair
2071 
2072 #define VMA_MAP_TYPE(KeyT, ValueT) \
2073  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2074 
2075 #else // #if VMA_USE_STL_UNORDERED_MAP
2076 
2077 template<typename T1, typename T2>
2078 struct VmaPair
2079 {
2080  T1 first;
2081  T2 second;
2082 
2083  VmaPair() : first(), second() { }
2084  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2085 };
2086 
2087 /* Class compatible with subset of interface of std::unordered_map.
2088 KeyT, ValueT must be POD because they will be stored in VmaVector.
2089 */
2090 template<typename KeyT, typename ValueT>
2091 class VmaMap
2092 {
2093 public:
2094  typedef VmaPair<KeyT, ValueT> PairType;
2095  typedef PairType* iterator;
2096 
2097  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2098 
2099  iterator begin() { return m_Vector.begin(); }
2100  iterator end() { return m_Vector.end(); }
2101 
2102  void insert(const PairType& pair);
2103  iterator find(const KeyT& key);
2104  void erase(iterator it);
2105 
2106 private:
2107  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2108 };
2109 
2110 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2111 
2112 template<typename FirstT, typename SecondT>
2113 struct VmaPairFirstLess
2114 {
2115  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2116  {
2117  return lhs.first < rhs.first;
2118  }
2119  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2120  {
2121  return lhs.first < rhsFirst;
2122  }
2123 };
2124 
2125 template<typename KeyT, typename ValueT>
2126 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2127 {
2128  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2129  m_Vector.data(),
2130  m_Vector.data() + m_Vector.size(),
2131  pair,
2132  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2133  VectorInsert(m_Vector, indexToInsert, pair);
2134 }
2135 
2136 template<typename KeyT, typename ValueT>
2137 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2138 {
2139  PairType* it = VmaBinaryFindFirstNotLess(
2140  m_Vector.data(),
2141  m_Vector.data() + m_Vector.size(),
2142  key,
2143  VmaPairFirstLess<KeyT, ValueT>());
2144  if((it != m_Vector.end()) && (it->first == key))
2145  {
2146  return it;
2147  }
2148  else
2149  {
2150  return m_Vector.end();
2151  }
2152 }
2153 
2154 template<typename KeyT, typename ValueT>
2155 void VmaMap<KeyT, ValueT>::erase(iterator it)
2156 {
2157  VectorRemove(m_Vector, it - m_Vector.begin());
2158 }
2159 
2160 #endif // #if VMA_USE_STL_UNORDERED_MAP
2161 
2163 
2164 class VmaBlock;
2165 
2166 enum VMA_BLOCK_VECTOR_TYPE
2167 {
2168  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2169  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2170  VMA_BLOCK_VECTOR_TYPE_COUNT
2171 };
2172 
2173 static VMA_BLOCK_VECTOR_TYPE VmaMemoryRequirementFlagsToBlockVectorType(VmaMemoryRequirementFlags flags)
2174 {
2175  return (flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0 ?
2176  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2177  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2178 }
2179 
2180 struct VmaAllocation_T
2181 {
2182 public:
2183  enum ALLOCATION_TYPE
2184  {
2185  ALLOCATION_TYPE_NONE,
2186  ALLOCATION_TYPE_BLOCK,
2187  ALLOCATION_TYPE_OWN,
2188  };
2189 
2190  VmaAllocation_T()
2191  {
2192  memset(this, 0, sizeof(VmaAllocation_T));
2193  }
2194 
2195  void InitBlockAllocation(
2196  VmaBlock* block,
2197  VkDeviceSize offset,
2198  VkDeviceSize alignment,
2199  VkDeviceSize size,
2200  VmaSuballocationType suballocationType,
2201  void* pUserData)
2202  {
2203  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2204  VMA_ASSERT(block != VMA_NULL);
2205  m_Type = ALLOCATION_TYPE_BLOCK;
2206  m_Alignment = alignment;
2207  m_Size = size;
2208  m_pUserData = pUserData;
2209  m_SuballocationType = suballocationType;
2210  m_BlockAllocation.m_Block = block;
2211  m_BlockAllocation.m_Offset = offset;
2212  }
2213 
2214  void ChangeBlockAllocation(
2215  VmaBlock* block,
2216  VkDeviceSize offset)
2217  {
2218  VMA_ASSERT(block != VMA_NULL);
2219  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2220  m_BlockAllocation.m_Block = block;
2221  m_BlockAllocation.m_Offset = offset;
2222  }
2223 
2224  void InitOwnAllocation(
2225  uint32_t memoryTypeIndex,
2226  VkDeviceMemory hMemory,
2227  VmaSuballocationType suballocationType,
2228  bool persistentMap,
2229  void* pMappedData,
2230  VkDeviceSize size,
2231  void* pUserData)
2232  {
2233  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2234  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2235  m_Type = ALLOCATION_TYPE_OWN;
2236  m_Alignment = 0;
2237  m_Size = size;
2238  m_pUserData = pUserData;
2239  m_SuballocationType = suballocationType;
2240  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2241  m_OwnAllocation.m_hMemory = hMemory;
2242  m_OwnAllocation.m_PersistentMap = persistentMap;
2243  m_OwnAllocation.m_pMappedData = pMappedData;
2244  }
2245 
2246  ALLOCATION_TYPE GetType() const { return m_Type; }
2247  VkDeviceSize GetAlignment() const { return m_Alignment; }
2248  VkDeviceSize GetSize() const { return m_Size; }
2249  void* GetUserData() const { return m_pUserData; }
2250  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2251  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2252 
2253  VmaBlock* GetBlock() const
2254  {
2255  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2256  return m_BlockAllocation.m_Block;
2257  }
2258  VkDeviceSize GetOffset() const
2259  {
2260  return (m_Type == ALLOCATION_TYPE_BLOCK) ? m_BlockAllocation.m_Offset : 0;
2261  }
2262  VkDeviceMemory GetMemory() const;
2263  uint32_t GetMemoryTypeIndex() const;
2264  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2265  void* GetMappedData() const;
2266 
2267  VkResult OwnAllocMapPersistentlyMappedMemory(VkDevice hDevice)
2268  {
2269  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2270  if(m_OwnAllocation.m_PersistentMap)
2271  {
2272  return vkMapMemory(hDevice, m_OwnAllocation.m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_OwnAllocation.m_pMappedData);
2273  }
2274  return VK_SUCCESS;
2275  }
2276  void OwnAllocUnmapPersistentlyMappedMemory(VkDevice hDevice)
2277  {
2278  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2279  if(m_OwnAllocation.m_pMappedData)
2280  {
2281  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
2282  vkUnmapMemory(hDevice, m_OwnAllocation.m_hMemory);
2283  m_OwnAllocation.m_pMappedData = VMA_NULL;
2284  }
2285  }
2286 
2287 private:
2288  VkDeviceSize m_Alignment;
2289  VkDeviceSize m_Size;
2290  void* m_pUserData;
2291  ALLOCATION_TYPE m_Type;
2292  VmaSuballocationType m_SuballocationType;
2293 
2294  union
2295  {
2296  // Allocation out of VmaBlock.
2297  struct BlockAllocation
2298  {
2299  VmaBlock* m_Block;
2300  VkDeviceSize m_Offset;
2301  } m_BlockAllocation;
2302 
2303  // Allocation for an object that has its own private VkDeviceMemory.
2304  struct OwnAllocation
2305  {
2306  uint32_t m_MemoryTypeIndex;
2307  VkDeviceMemory m_hMemory;
2308  bool m_PersistentMap;
2309  void* m_pMappedData;
2310  } m_OwnAllocation;
2311  };
2312 };
2313 
2314 /*
2315 Represents a region of VmaBlock that is either assigned and returned as
2316 allocated memory block or free.
2317 */
2318 struct VmaSuballocation
2319 {
2320  VkDeviceSize offset;
2321  VkDeviceSize size;
2322  VmaSuballocationType type;
2323 };
2324 
2325 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
2326 
2327 // Parameters of an allocation.
2328 struct VmaAllocationRequest
2329 {
2330  VmaSuballocationList::iterator freeSuballocationItem;
2331  VkDeviceSize offset;
2332 };
2333 
2334 /* Single block of memory - VkDeviceMemory with all the data about its regions
2335 assigned or free. */
2336 class VmaBlock
2337 {
2338 public:
2339  uint32_t m_MemoryTypeIndex;
2340  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
2341  VkDeviceMemory m_hMemory;
2342  VkDeviceSize m_Size;
2343  bool m_PersistentMap;
2344  void* m_pMappedData;
2345  uint32_t m_FreeCount;
2346  VkDeviceSize m_SumFreeSize;
2347  VmaSuballocationList m_Suballocations;
2348  // Suballocations that are free and have size greater than certain threshold.
2349  // Sorted by size, ascending.
2350  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
2351 
2352  VmaBlock(VmaAllocator hAllocator);
2353 
2354  ~VmaBlock()
2355  {
2356  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2357  }
2358 
2359  // Always call after construction.
2360  void Init(
2361  uint32_t newMemoryTypeIndex,
2362  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
2363  VkDeviceMemory newMemory,
2364  VkDeviceSize newSize,
2365  bool persistentMap,
2366  void* pMappedData);
2367  // Always call before destruction.
2368  void Destroy(VmaAllocator allocator);
2369 
2370  // Validates all data structures inside this object. If not valid, returns false.
2371  bool Validate() const;
2372 
2373  // Tries to find a place for suballocation with given parameters inside this allocation.
2374  // If succeeded, fills pAllocationRequest and returns true.
2375  // If failed, returns false.
2376  bool CreateAllocationRequest(
2377  VkDeviceSize bufferImageGranularity,
2378  VkDeviceSize allocSize,
2379  VkDeviceSize allocAlignment,
2380  VmaSuballocationType allocType,
2381  VmaAllocationRequest* pAllocationRequest);
2382 
2383  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
2384  // If yes, fills pOffset and returns true. If no, returns false.
2385  bool CheckAllocation(
2386  VkDeviceSize bufferImageGranularity,
2387  VkDeviceSize allocSize,
2388  VkDeviceSize allocAlignment,
2389  VmaSuballocationType allocType,
2390  VmaSuballocationList::const_iterator freeSuballocItem,
2391  VkDeviceSize* pOffset) const;
2392 
2393  // Returns true if this allocation is empty - contains only single free suballocation.
2394  bool IsEmpty() const;
2395 
2396  // Makes actual allocation based on request. Request must already be checked
2397  // and valid.
2398  void Alloc(
2399  const VmaAllocationRequest& request,
2400  VmaSuballocationType type,
2401  VkDeviceSize allocSize);
2402 
2403  // Frees suballocation assigned to given memory region.
2404  void Free(const VmaAllocation allocation);
2405 
2406 #if VMA_STATS_STRING_ENABLED
2407  void PrintDetailedMap(class VmaStringBuilder& sb) const;
2408 #endif
2409 
2410 private:
2411  // Given free suballocation, it merges it with following one, which must also be free.
2412  void MergeFreeWithNext(VmaSuballocationList::iterator item);
2413  // Releases given suballocation, making it free. Merges it with adjacent free
2414  // suballocations if applicable.
2415  void FreeSuballocation(VmaSuballocationList::iterator suballocItem);
2416  // Given free suballocation, it inserts it into sorted list of
2417  // m_FreeSuballocationsBySize if it's suitable.
2418  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
2419  // Given free suballocation, it removes it from sorted list of
2420  // m_FreeSuballocationsBySize if it's suitable.
2421  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
2422 };
2423 
2424 struct VmaPointerLess
2425 {
2426  bool operator()(const void* lhs, const void* rhs) const
2427  {
2428  return lhs < rhs;
2429  }
2430 };
2431 
2432 /* Sequence of VmaBlock. Represents memory blocks allocated for a specific
2433 Vulkan memory type. */
2434 struct VmaBlockVector
2435 {
2436  // Incrementally sorted by sumFreeSize, ascending.
2437  VmaVector< VmaBlock*, VmaStlAllocator<VmaBlock*> > m_Blocks;
2438 
2439  VmaBlockVector(VmaAllocator hAllocator);
2440  ~VmaBlockVector();
2441 
2442  bool IsEmpty() const { return m_Blocks.empty(); }
2443 
2444  // Finds and removes given block from vector.
2445  void Remove(VmaBlock* pBlock);
2446 
2447  // Performs single step in sorting m_Blocks. They may not be fully sorted
2448  // after this call.
2449  void IncrementallySortBlocks();
2450 
2451  // Adds statistics of this BlockVector to pStats.
2452  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const;
2453 
2454 #if VMA_STATS_STRING_ENABLED
2455  void PrintDetailedMap(class VmaStringBuilder& sb) const;
2456 #endif
2457 
2458  void UnmapPersistentlyMappedMemory();
2459  VkResult MapPersistentlyMappedMemory();
2460 
2461 private:
2462  VmaAllocator m_hAllocator;
2463 };
2464 
2465 // Main allocator object.
2466 struct VmaAllocator_T
2467 {
2468  bool m_UseMutex;
2469  VkDevice m_hDevice;
2470  bool m_AllocationCallbacksSpecified;
2471  VkAllocationCallbacks m_AllocationCallbacks;
2472  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
2473  VkDeviceSize m_PreferredLargeHeapBlockSize;
2474  VkDeviceSize m_PreferredSmallHeapBlockSize;
2475  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
2476  // Counter to allow nested calls to these functions.
2477  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
2478 
2479  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
2480  VkPhysicalDeviceMemoryProperties m_MemProps;
2481 
2482  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
2483  /* There can be at most one allocation that is completely empty - a
2484  hysteresis to avoid pessimistic case of alternating creation and destruction
2485  of a VkDeviceMemory. */
2486  bool m_HasEmptyBlock[VK_MAX_MEMORY_TYPES];
2487  VMA_MUTEX m_BlocksMutex[VK_MAX_MEMORY_TYPES];
2488 
2489  // Each vector is sorted by memory (handle value).
2490  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
2491  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
2492  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
2493 
2494  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
2495  ~VmaAllocator_T();
2496 
2497  const VkAllocationCallbacks* GetAllocationCallbacks() const
2498  {
2499  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
2500  }
2501 
2502  VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const;
2503 
2504  VkDeviceSize GetBufferImageGranularity() const
2505  {
2506  return VMA_MAX(
2507  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
2508  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
2509  }
2510 
2511  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
2512  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
2513 
2514  // Main allocation function.
2515  VkResult AllocateMemory(
2516  const VkMemoryRequirements& vkMemReq,
2517  const VmaMemoryRequirements& vmaMemReq,
2518  VmaSuballocationType suballocType,
2519  VmaAllocation* pAllocation);
2520 
2521  // Main deallocation function.
2522  void FreeMemory(const VmaAllocation allocation);
2523 
2524  void CalculateStats(VmaStats* pStats);
2525 
2526 #if VMA_STATS_STRING_ENABLED
2527  void PrintDetailedMap(class VmaStringBuilder& sb);
2528 #endif
2529 
2530  void UnmapPersistentlyMappedMemory();
2531  VkResult MapPersistentlyMappedMemory();
2532 
2533  VkResult Defragment(
2534  VmaAllocation* pAllocations,
2535  size_t allocationCount,
2536  VkBool32* pAllocationsChanged,
2537  const VmaDefragmentationInfo* pDefragmentationInfo,
2538  VmaDefragmentationStats* pDefragmentationStats);
2539 
2540  static void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
2541 
2542 private:
2543  VkPhysicalDevice m_PhysicalDevice;
2544 
2545  VkResult AllocateMemoryOfType(
2546  const VkMemoryRequirements& vkMemReq,
2547  const VmaMemoryRequirements& vmaMemReq,
2548  uint32_t memTypeIndex,
2549  VmaSuballocationType suballocType,
2550  VmaAllocation* pAllocation);
2551 
2552  // Allocates and registers new VkDeviceMemory specifically for single allocation.
2553  VkResult AllocateOwnMemory(
2554  VkDeviceSize size,
2555  VmaSuballocationType suballocType,
2556  uint32_t memTypeIndex,
2557  bool map,
2558  void* pUserData,
2559  VmaAllocation* pAllocation);
2560 
2561  // Tries to free pMemory as Own Memory. Returns true if found and freed.
2562  void FreeOwnMemory(VmaAllocation allocation);
2563 };
2564 
2566 // Memory allocation #2 after VmaAllocator_T definition
2567 
2568 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
2569 {
2570  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
2571 }
2572 
2573 static void VmaFree(VmaAllocator hAllocator, void* ptr)
2574 {
2575  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
2576 }
2577 
2578 template<typename T>
2579 static T* VmaAllocate(VmaAllocator hAllocator)
2580 {
2581  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
2582 }
2583 
2584 template<typename T>
2585 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
2586 {
2587  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
2588 }
2589 
2590 template<typename T>
2591 static void vma_delete(VmaAllocator hAllocator, T* ptr)
2592 {
2593  if(ptr != VMA_NULL)
2594  {
2595  ptr->~T();
2596  VmaFree(hAllocator, ptr);
2597  }
2598 }
2599 
2600 template<typename T>
2601 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
2602 {
2603  if(ptr != VMA_NULL)
2604  {
2605  for(size_t i = count; i--; )
2606  ptr[i].~T();
2607  VmaFree(hAllocator, ptr);
2608  }
2609 }
2610 
2612 // VmaStringBuilder
2613 
2614 #if VMA_STATS_STRING_ENABLED
2615 
2616 class VmaStringBuilder
2617 {
2618 public:
2619  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
2620  size_t GetLength() const { return m_Data.size(); }
2621  const char* GetData() const { return m_Data.data(); }
2622 
2623  void Add(char ch) { m_Data.push_back(ch); }
2624  void Add(const char* pStr);
2625  void AddNewLine() { Add('\n'); }
2626  void AddNumber(uint32_t num);
2627  void AddNumber(uint64_t num);
2628  void AddBool(bool b) { Add(b ? "true" : "false"); }
2629  void AddNull() { Add("null"); }
2630  void AddString(const char* pStr);
2631 
2632 private:
2633  VmaVector< char, VmaStlAllocator<char> > m_Data;
2634 };
2635 
2636 void VmaStringBuilder::Add(const char* pStr)
2637 {
2638  const size_t strLen = strlen(pStr);
2639  if(strLen > 0)
2640  {
2641  const size_t oldCount = m_Data.size();
2642  m_Data.resize(oldCount + strLen);
2643  memcpy(m_Data.data() + oldCount, pStr, strLen);
2644  }
2645 }
2646 
2647 void VmaStringBuilder::AddNumber(uint32_t num)
2648 {
2649  char buf[11];
2650  VmaUint32ToStr(buf, sizeof(buf), num);
2651  Add(buf);
2652 }
2653 
2654 void VmaStringBuilder::AddNumber(uint64_t num)
2655 {
2656  char buf[21];
2657  VmaUint64ToStr(buf, sizeof(buf), num);
2658  Add(buf);
2659 }
2660 
2661 void VmaStringBuilder::AddString(const char* pStr)
2662 {
2663  Add('"');
2664  const size_t strLen = strlen(pStr);
2665  for(size_t i = 0; i < strLen; ++i)
2666  {
2667  char ch = pStr[i];
2668  if(ch == '\'')
2669  {
2670  Add("\\\\");
2671  }
2672  else if(ch == '"')
2673  {
2674  Add("\\\"");
2675  }
2676  else if(ch >= 32)
2677  {
2678  Add(ch);
2679  }
2680  else switch(ch)
2681  {
2682  case '\n':
2683  Add("\\n");
2684  break;
2685  case '\r':
2686  Add("\\r");
2687  break;
2688  case '\t':
2689  Add("\\t");
2690  break;
2691  default:
2692  VMA_ASSERT(0 && "Character not currently supported.");
2693  break;
2694  }
2695  }
2696  Add('"');
2697 }
2698 
2700 
2701 VkDeviceMemory VmaAllocation_T::GetMemory() const
2702 {
2703  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2704  m_BlockAllocation.m_Block->m_hMemory : m_OwnAllocation.m_hMemory;
2705 }
2706 
2707 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
2708 {
2709  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2710  m_BlockAllocation.m_Block->m_MemoryTypeIndex : m_OwnAllocation.m_MemoryTypeIndex;
2711 }
2712 
2713 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
2714 {
2715  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2716  m_BlockAllocation.m_Block->m_BlockVectorType :
2717  (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
2718 }
2719 
2720 void* VmaAllocation_T::GetMappedData() const
2721 {
2722  switch(m_Type)
2723  {
2724  case ALLOCATION_TYPE_BLOCK:
2725  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
2726  {
2727  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
2728  }
2729  else
2730  {
2731  return VMA_NULL;
2732  }
2733  break;
2734  case ALLOCATION_TYPE_OWN:
2735  return m_OwnAllocation.m_pMappedData;
2736  default:
2737  VMA_ASSERT(0);
2738  return VMA_NULL;
2739  }
2740 }
2741 
2742 // Correspond to values of enum VmaSuballocationType.
2743 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
2744  "FREE",
2745  "UNKNOWN",
2746  "BUFFER",
2747  "IMAGE_UNKNOWN",
2748  "IMAGE_LINEAR",
2749  "IMAGE_OPTIMAL",
2750 };
2751 
2752 static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat)
2753 {
2754  sb.Add("{ \"Allocations\": ");
2755  sb.AddNumber(stat.AllocationCount);
2756  sb.Add(", \"Suballocations\": ");
2757  sb.AddNumber(stat.SuballocationCount);
2758  sb.Add(", \"UnusedRanges\": ");
2759  sb.AddNumber(stat.UnusedRangeCount);
2760  sb.Add(", \"UsedBytes\": ");
2761  sb.AddNumber(stat.UsedBytes);
2762  sb.Add(", \"UnusedBytes\": ");
2763  sb.AddNumber(stat.UnusedBytes);
2764  sb.Add(", \"SuballocationSize\": { \"Min\": ");
2765  sb.AddNumber(stat.SuballocationSizeMin);
2766  sb.Add(", \"Avg\": ");
2767  sb.AddNumber(stat.SuballocationSizeAvg);
2768  sb.Add(", \"Max\": ");
2769  sb.AddNumber(stat.SuballocationSizeMax);
2770  sb.Add(" }, \"UnusedRangeSize\": { \"Min\": ");
2771  sb.AddNumber(stat.UnusedRangeSizeMin);
2772  sb.Add(", \"Avg\": ");
2773  sb.AddNumber(stat.UnusedRangeSizeAvg);
2774  sb.Add(", \"Max\": ");
2775  sb.AddNumber(stat.UnusedRangeSizeMax);
2776  sb.Add(" } }");
2777 }
2778 
2779 #endif // #if VMA_STATS_STRING_ENABLED
2780 
2781 struct VmaSuballocationItemSizeLess
2782 {
2783  bool operator()(
2784  const VmaSuballocationList::iterator lhs,
2785  const VmaSuballocationList::iterator rhs) const
2786  {
2787  return lhs->size < rhs->size;
2788  }
2789  bool operator()(
2790  const VmaSuballocationList::iterator lhs,
2791  VkDeviceSize rhsSize) const
2792  {
2793  return lhs->size < rhsSize;
2794  }
2795 };
2796 
2797 VmaBlock::VmaBlock(VmaAllocator hAllocator) :
2798  m_MemoryTypeIndex(UINT32_MAX),
2799  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
2800  m_hMemory(VK_NULL_HANDLE),
2801  m_Size(0),
2802  m_PersistentMap(false),
2803  m_pMappedData(VMA_NULL),
2804  m_FreeCount(0),
2805  m_SumFreeSize(0),
2806  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
2807  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
2808 {
2809 }
2810 
2811 void VmaBlock::Init(
2812  uint32_t newMemoryTypeIndex,
2813  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
2814  VkDeviceMemory newMemory,
2815  VkDeviceSize newSize,
2816  bool persistentMap,
2817  void* pMappedData)
2818 {
2819  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2820 
2821  m_MemoryTypeIndex = newMemoryTypeIndex;
2822  m_BlockVectorType = newBlockVectorType;
2823  m_hMemory = newMemory;
2824  m_Size = newSize;
2825  m_PersistentMap = persistentMap;
2826  m_pMappedData = pMappedData;
2827  m_FreeCount = 1;
2828  m_SumFreeSize = newSize;
2829 
2830  m_Suballocations.clear();
2831  m_FreeSuballocationsBySize.clear();
2832 
2833  VmaSuballocation suballoc = {};
2834  suballoc.offset = 0;
2835  suballoc.size = newSize;
2836  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2837 
2838  m_Suballocations.push_back(suballoc);
2839  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
2840  --suballocItem;
2841  m_FreeSuballocationsBySize.push_back(suballocItem);
2842 }
2843 
2844 void VmaBlock::Destroy(VmaAllocator allocator)
2845 {
2846  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
2847  if(m_pMappedData != VMA_NULL)
2848  {
2849  vkUnmapMemory(allocator->m_hDevice, m_hMemory);
2850  m_pMappedData = VMA_NULL;
2851  }
2852 
2853  // Callback.
2854  if(allocator->m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
2855  {
2856  (*allocator->m_DeviceMemoryCallbacks.pfnFree)(allocator, m_MemoryTypeIndex, m_hMemory, m_Size);
2857  }
2858 
2859  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
2860  m_hMemory = VK_NULL_HANDLE;
2861 }
2862 
2863 bool VmaBlock::Validate() const
2864 {
2865  if((m_hMemory == VK_NULL_HANDLE) ||
2866  (m_Size == 0) ||
2867  m_Suballocations.empty())
2868  {
2869  return false;
2870  }
2871 
2872  // Expected offset of new suballocation as calculates from previous ones.
2873  VkDeviceSize calculatedOffset = 0;
2874  // Expected number of free suballocations as calculated from traversing their list.
2875  uint32_t calculatedFreeCount = 0;
2876  // Expected sum size of free suballocations as calculated from traversing their list.
2877  VkDeviceSize calculatedSumFreeSize = 0;
2878  // Expected number of free suballocations that should be registered in
2879  // m_FreeSuballocationsBySize calculated from traversing their list.
2880  size_t freeSuballocationsToRegister = 0;
2881  // True if previous visisted suballocation was free.
2882  bool prevFree = false;
2883 
2884  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2885  suballocItem != m_Suballocations.cend();
2886  ++suballocItem)
2887  {
2888  const VmaSuballocation& subAlloc = *suballocItem;
2889 
2890  // Actual offset of this suballocation doesn't match expected one.
2891  if(subAlloc.offset != calculatedOffset)
2892  {
2893  return false;
2894  }
2895 
2896  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
2897  // Two adjacent free suballocations are invalid. They should be merged.
2898  if(prevFree && currFree)
2899  {
2900  return false;
2901  }
2902  prevFree = currFree;
2903 
2904  if(currFree)
2905  {
2906  calculatedSumFreeSize += subAlloc.size;
2907  ++calculatedFreeCount;
2908  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2909  {
2910  ++freeSuballocationsToRegister;
2911  }
2912  }
2913 
2914  calculatedOffset += subAlloc.size;
2915  }
2916 
2917  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
2918  // match expected one.
2919  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
2920  {
2921  return false;
2922  }
2923 
2924  VkDeviceSize lastSize = 0;
2925  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
2926  {
2927  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
2928 
2929  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
2930  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
2931  {
2932  return false;
2933  }
2934  // They must be sorted by size ascending.
2935  if(suballocItem->size < lastSize)
2936  {
2937  return false;
2938  }
2939 
2940  lastSize = suballocItem->size;
2941  }
2942 
2943  // Check if totals match calculacted values.
2944  return
2945  (calculatedOffset == m_Size) &&
2946  (calculatedSumFreeSize == m_SumFreeSize) &&
2947  (calculatedFreeCount == m_FreeCount);
2948 }
2949 
2950 /*
2951 How many suitable free suballocations to analyze before choosing best one.
2952 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
2953  be chosen.
2954 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
2955  suballocations will be analized and best one will be chosen.
2956 - Any other value is also acceptable.
2957 */
2958 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
2959 
2960 bool VmaBlock::CreateAllocationRequest(
2961  VkDeviceSize bufferImageGranularity,
2962  VkDeviceSize allocSize,
2963  VkDeviceSize allocAlignment,
2964  VmaSuballocationType allocType,
2965  VmaAllocationRequest* pAllocationRequest)
2966 {
2967  VMA_ASSERT(allocSize > 0);
2968  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2969  VMA_ASSERT(pAllocationRequest != VMA_NULL);
2970  VMA_HEAVY_ASSERT(Validate());
2971 
2972  // There is not enough total free space in this allocation to fullfill the request: Early return.
2973  if(m_SumFreeSize < allocSize)
2974  {
2975  return false;
2976  }
2977 
2978  // Old brute-force algorithm, linearly searching suballocations.
2979  /*
2980  uint32_t suitableSuballocationsFound = 0;
2981  for(VmaSuballocationList::iterator suballocItem = suballocations.Front();
2982  suballocItem != VMA_NULL &&
2983  suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK;
2984  suballocItem = suballocItem->Next)
2985  {
2986  if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE)
2987  {
2988  VkDeviceSize offset = 0, cost = 0;
2989  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost))
2990  {
2991  ++suitableSuballocationsFound;
2992  if(cost < costLimit)
2993  {
2994  pAllocationRequest->freeSuballocationItem = suballocItem;
2995  pAllocationRequest->offset = offset;
2996  pAllocationRequest->cost = cost;
2997  if(cost == 0)
2998  return true;
2999  costLimit = cost;
3000  betterSuballocationFound = true;
3001  }
3002  }
3003  }
3004  }
3005  */
3006 
3007  // New algorithm, efficiently searching freeSuballocationsBySize.
3008  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
3009  if(freeSuballocCount > 0)
3010  {
3011  if(VMA_BEST_FIT)
3012  {
3013  // Find first free suballocation with size not less than allocSize.
3014  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3015  m_FreeSuballocationsBySize.data(),
3016  m_FreeSuballocationsBySize.data() + freeSuballocCount,
3017  allocSize,
3018  VmaSuballocationItemSizeLess());
3019  size_t index = it - m_FreeSuballocationsBySize.data();
3020  for(; index < freeSuballocCount; ++index)
3021  {
3022  VkDeviceSize offset = 0;
3023  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
3024  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
3025  {
3026  pAllocationRequest->freeSuballocationItem = suballocItem;
3027  pAllocationRequest->offset = offset;
3028  return true;
3029  }
3030  }
3031  }
3032  else
3033  {
3034  // Search staring from biggest suballocations.
3035  for(size_t index = freeSuballocCount; index--; )
3036  {
3037  VkDeviceSize offset = 0;
3038  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
3039  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
3040  {
3041  pAllocationRequest->freeSuballocationItem = suballocItem;
3042  pAllocationRequest->offset = offset;
3043  return true;
3044  }
3045  }
3046  }
3047  }
3048 
3049  return false;
3050 }
3051 
3052 bool VmaBlock::CheckAllocation(
3053  VkDeviceSize bufferImageGranularity,
3054  VkDeviceSize allocSize,
3055  VkDeviceSize allocAlignment,
3056  VmaSuballocationType allocType,
3057  VmaSuballocationList::const_iterator freeSuballocItem,
3058  VkDeviceSize* pOffset) const
3059 {
3060  VMA_ASSERT(allocSize > 0);
3061  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
3062  VMA_ASSERT(freeSuballocItem != m_Suballocations.cend());
3063  VMA_ASSERT(pOffset != VMA_NULL);
3064 
3065  const VmaSuballocation& suballoc = *freeSuballocItem;
3066  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
3067 
3068  // Size of this suballocation is too small for this request: Early return.
3069  if(suballoc.size < allocSize)
3070  {
3071  return false;
3072  }
3073 
3074  // Start from offset equal to beginning of this suballocation.
3075  *pOffset = suballoc.offset;
3076 
3077  // Apply VMA_DEBUG_MARGIN at the beginning.
3078  if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin())
3079  {
3080  *pOffset += VMA_DEBUG_MARGIN;
3081  }
3082 
3083  // Apply alignment.
3084  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
3085  *pOffset = VmaAlignUp(*pOffset, alignment);
3086 
3087  // Check previous suballocations for BufferImageGranularity conflicts.
3088  // Make bigger alignment if necessary.
3089  if(bufferImageGranularity > 1)
3090  {
3091  bool bufferImageGranularityConflict = false;
3092  VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem;
3093  while(prevSuballocItem != m_Suballocations.cbegin())
3094  {
3095  --prevSuballocItem;
3096  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
3097  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
3098  {
3099  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
3100  {
3101  bufferImageGranularityConflict = true;
3102  break;
3103  }
3104  }
3105  else
3106  // Already on previous page.
3107  break;
3108  }
3109  if(bufferImageGranularityConflict)
3110  {
3111  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
3112  }
3113  }
3114 
3115  // Calculate padding at the beginning based on current offset.
3116  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
3117 
3118  // Calculate required margin at the end if this is not last suballocation.
3119  VmaSuballocationList::const_iterator next = freeSuballocItem;
3120  ++next;
3121  const VkDeviceSize requiredEndMargin =
3122  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
3123 
3124  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
3125  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
3126  {
3127  return false;
3128  }
3129 
3130  // Check next suballocations for BufferImageGranularity conflicts.
3131  // If conflict exists, allocation cannot be made here.
3132  if(bufferImageGranularity > 1)
3133  {
3134  VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem;
3135  ++nextSuballocItem;
3136  while(nextSuballocItem != m_Suballocations.cend())
3137  {
3138  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
3139  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
3140  {
3141  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
3142  {
3143  return false;
3144  }
3145  }
3146  else
3147  {
3148  // Already on next page.
3149  break;
3150  }
3151  ++nextSuballocItem;
3152  }
3153  }
3154 
3155  // All tests passed: Success. pOffset is already filled.
3156  return true;
3157 }
3158 
3159 bool VmaBlock::IsEmpty() const
3160 {
3161  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
3162 }
3163 
3164 void VmaBlock::Alloc(
3165  const VmaAllocationRequest& request,
3166  VmaSuballocationType type,
3167  VkDeviceSize allocSize)
3168 {
3169  VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end());
3170  VmaSuballocation& suballoc = *request.freeSuballocationItem;
3171  // Given suballocation is a free block.
3172  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
3173  // Given offset is inside this suballocation.
3174  VMA_ASSERT(request.offset >= suballoc.offset);
3175  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
3176  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
3177  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
3178 
3179  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
3180  // it to become used.
3181  UnregisterFreeSuballocation(request.freeSuballocationItem);
3182 
3183  suballoc.offset = request.offset;
3184  suballoc.size = allocSize;
3185  suballoc.type = type;
3186 
3187  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
3188  if(paddingEnd)
3189  {
3190  VmaSuballocation paddingSuballoc = {};
3191  paddingSuballoc.offset = request.offset + allocSize;
3192  paddingSuballoc.size = paddingEnd;
3193  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3194  VmaSuballocationList::iterator next = request.freeSuballocationItem;
3195  ++next;
3196  const VmaSuballocationList::iterator paddingEndItem =
3197  m_Suballocations.insert(next, paddingSuballoc);
3198  RegisterFreeSuballocation(paddingEndItem);
3199  }
3200 
3201  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
3202  if(paddingBegin)
3203  {
3204  VmaSuballocation paddingSuballoc = {};
3205  paddingSuballoc.offset = request.offset - paddingBegin;
3206  paddingSuballoc.size = paddingBegin;
3207  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3208  const VmaSuballocationList::iterator paddingBeginItem =
3209  m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc);
3210  RegisterFreeSuballocation(paddingBeginItem);
3211  }
3212 
3213  // Update totals.
3214  m_FreeCount = m_FreeCount - 1;
3215  if(paddingBegin > 0)
3216  {
3217  ++m_FreeCount;
3218  }
3219  if(paddingEnd > 0)
3220  {
3221  ++m_FreeCount;
3222  }
3223  m_SumFreeSize -= allocSize;
3224 }
3225 
3226 void VmaBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
3227 {
3228  // Change this suballocation to be marked as free.
3229  VmaSuballocation& suballoc = *suballocItem;
3230  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3231 
3232  // Update totals.
3233  ++m_FreeCount;
3234  m_SumFreeSize += suballoc.size;
3235 
3236  // Merge with previous and/or next suballocation if it's also free.
3237  bool mergeWithNext = false;
3238  bool mergeWithPrev = false;
3239 
3240  VmaSuballocationList::iterator nextItem = suballocItem;
3241  ++nextItem;
3242  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
3243  {
3244  mergeWithNext = true;
3245  }
3246 
3247  VmaSuballocationList::iterator prevItem = suballocItem;
3248  if(suballocItem != m_Suballocations.begin())
3249  {
3250  --prevItem;
3251  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
3252  {
3253  mergeWithPrev = true;
3254  }
3255  }
3256 
3257  if(mergeWithNext)
3258  {
3259  UnregisterFreeSuballocation(nextItem);
3260  MergeFreeWithNext(suballocItem);
3261  }
3262 
3263  if(mergeWithPrev)
3264  {
3265  UnregisterFreeSuballocation(prevItem);
3266  MergeFreeWithNext(prevItem);
3267  RegisterFreeSuballocation(prevItem);
3268  }
3269  else
3270  RegisterFreeSuballocation(suballocItem);
3271 }
3272 
3273 void VmaBlock::Free(const VmaAllocation allocation)
3274 {
3275  const VkDeviceSize allocationOffset = allocation->GetOffset();
3276  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
3277  suballocItem != m_Suballocations.end();
3278  ++suballocItem)
3279  {
3280  VmaSuballocation& suballoc = *suballocItem;
3281  if(suballoc.offset == allocationOffset)
3282  {
3283  FreeSuballocation(suballocItem);
3284  VMA_HEAVY_ASSERT(Validate());
3285  return;
3286  }
3287  }
3288  VMA_ASSERT(0 && "Not found!");
3289 }
3290 
3291 #if VMA_STATS_STRING_ENABLED
3292 
3293 void VmaBlock::PrintDetailedMap(class VmaStringBuilder& sb) const
3294 {
3295  sb.Add("{\n\t\t\t\"Bytes\": ");
3296  sb.AddNumber(m_Size);
3297  sb.Add(",\n\t\t\t\"FreeBytes\": ");
3298  sb.AddNumber(m_SumFreeSize);
3299  sb.Add(",\n\t\t\t\"Suballocations\": ");
3300  sb.AddNumber(m_Suballocations.size());
3301  sb.Add(",\n\t\t\t\"FreeSuballocations\": ");
3302  sb.AddNumber(m_FreeCount);
3303  sb.Add(",\n\t\t\t\"SuballocationList\": [");
3304 
3305  size_t i = 0;
3306  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
3307  suballocItem != m_Suballocations.cend();
3308  ++suballocItem, ++i)
3309  {
3310  if(i > 0)
3311  {
3312  sb.Add(",\n\t\t\t\t{ \"Type\": ");
3313  }
3314  else
3315  {
3316  sb.Add("\n\t\t\t\t{ \"Type\": ");
3317  }
3318  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
3319  sb.Add(", \"Size\": ");
3320  sb.AddNumber(suballocItem->size);
3321  sb.Add(", \"Offset\": ");
3322  sb.AddNumber(suballocItem->offset);
3323  sb.Add(" }");
3324  }
3325 
3326  sb.Add("\n\t\t\t]\n\t\t}");
3327 }
3328 
3329 #endif // #if VMA_STATS_STRING_ENABLED
3330 
3331 void VmaBlock::MergeFreeWithNext(VmaSuballocationList::iterator item)
3332 {
3333  VMA_ASSERT(item != m_Suballocations.end());
3334  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3335 
3336  VmaSuballocationList::iterator nextItem = item;
3337  ++nextItem;
3338  VMA_ASSERT(nextItem != m_Suballocations.end());
3339  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
3340 
3341  item->size += nextItem->size;
3342  --m_FreeCount;
3343  m_Suballocations.erase(nextItem);
3344 }
3345 
3346 void VmaBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
3347 {
3348  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3349  VMA_ASSERT(item->size > 0);
3350 
3351  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
3352  {
3353  if(m_FreeSuballocationsBySize.empty())
3354  {
3355  m_FreeSuballocationsBySize.push_back(item);
3356  }
3357  else
3358  {
3359  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3360  m_FreeSuballocationsBySize.data(),
3361  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
3362  item,
3363  VmaSuballocationItemSizeLess());
3364  size_t index = it - m_FreeSuballocationsBySize.data();
3365  VectorInsert(m_FreeSuballocationsBySize, index, item);
3366  }
3367  }
3368 }
3369 
3370 void VmaBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
3371 {
3372  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3373  VMA_ASSERT(item->size > 0);
3374 
3375  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
3376  {
3377  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3378  m_FreeSuballocationsBySize.data(),
3379  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
3380  item,
3381  VmaSuballocationItemSizeLess());
3382  for(size_t index = it - m_FreeSuballocationsBySize.data();
3383  index < m_FreeSuballocationsBySize.size();
3384  ++index)
3385  {
3386  if(m_FreeSuballocationsBySize[index] == item)
3387  {
3388  VectorRemove(m_FreeSuballocationsBySize, index);
3389  return;
3390  }
3391  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
3392  }
3393  VMA_ASSERT(0 && "Not found.");
3394  }
3395 }
3396 
3397 static void InitStatInfo(VmaStatInfo& outInfo)
3398 {
3399  memset(&outInfo, 0, sizeof(outInfo));
3400  outInfo.SuballocationSizeMin = UINT64_MAX;
3401  outInfo.UnusedRangeSizeMin = UINT64_MAX;
3402 }
3403 
3404 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc)
3405 {
3406  outInfo.AllocationCount = 1;
3407 
3408  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
3409  outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount;
3410  outInfo.UnusedRangeCount = alloc.m_FreeCount;
3411 
3412  outInfo.UnusedBytes = alloc.m_SumFreeSize;
3413  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
3414 
3415  outInfo.SuballocationSizeMin = UINT64_MAX;
3416  outInfo.SuballocationSizeMax = 0;
3417  outInfo.UnusedRangeSizeMin = UINT64_MAX;
3418  outInfo.UnusedRangeSizeMax = 0;
3419 
3420  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
3421  suballocItem != alloc.m_Suballocations.cend();
3422  ++suballocItem)
3423  {
3424  const VmaSuballocation& suballoc = *suballocItem;
3425  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
3426  {
3427  outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size);
3428  outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size);
3429  }
3430  else
3431  {
3432  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
3433  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
3434  }
3435  }
3436 }
3437 
3438 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
3439 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
3440 {
3441  inoutInfo.AllocationCount += srcInfo.AllocationCount;
3442  inoutInfo.SuballocationCount += srcInfo.SuballocationCount;
3443  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
3444  inoutInfo.UsedBytes += srcInfo.UsedBytes;
3445  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
3446  inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin);
3447  inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax);
3448  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
3449  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
3450 }
3451 
3452 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
3453 {
3454  inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ?
3455  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0;
3456  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
3457  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
3458 }
3459 
3460 VmaBlockVector::VmaBlockVector(VmaAllocator hAllocator) :
3461  m_hAllocator(hAllocator),
3462  m_Blocks(VmaStlAllocator<VmaBlock*>(hAllocator->GetAllocationCallbacks()))
3463 {
3464 }
3465 
3466 VmaBlockVector::~VmaBlockVector()
3467 {
3468  for(size_t i = m_Blocks.size(); i--; )
3469  {
3470  m_Blocks[i]->Destroy(m_hAllocator);
3471  vma_delete(m_hAllocator, m_Blocks[i]);
3472  }
3473 }
3474 
3475 void VmaBlockVector::Remove(VmaBlock* pBlock)
3476 {
3477  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
3478  {
3479  if(m_Blocks[blockIndex] == pBlock)
3480  {
3481  VectorRemove(m_Blocks, blockIndex);
3482  return;
3483  }
3484  }
3485  VMA_ASSERT(0);
3486 }
3487 
3488 void VmaBlockVector::IncrementallySortBlocks()
3489 {
3490  // Bubble sort only until first swap.
3491  for(size_t i = 1; i < m_Blocks.size(); ++i)
3492  {
3493  if(m_Blocks[i - 1]->m_SumFreeSize > m_Blocks[i]->m_SumFreeSize)
3494  {
3495  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
3496  return;
3497  }
3498  }
3499 }
3500 
3501 #if VMA_STATS_STRING_ENABLED
3502 
3503 void VmaBlockVector::PrintDetailedMap(class VmaStringBuilder& sb) const
3504 {
3505  for(size_t i = 0; i < m_Blocks.size(); ++i)
3506  {
3507  if(i > 0)
3508  {
3509  sb.Add(",\n\t\t");
3510  }
3511  else
3512  {
3513  sb.Add("\n\t\t");
3514  }
3515  m_Blocks[i]->PrintDetailedMap(sb);
3516  }
3517 }
3518 
3519 #endif // #if VMA_STATS_STRING_ENABLED
3520 
3521 void VmaBlockVector::UnmapPersistentlyMappedMemory()
3522 {
3523  for(size_t i = m_Blocks.size(); i--; )
3524  {
3525  VmaBlock* pBlock = m_Blocks[i];
3526  if(pBlock->m_pMappedData != VMA_NULL)
3527  {
3528  VMA_ASSERT(pBlock->m_PersistentMap != false);
3529  vkUnmapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory);
3530  pBlock->m_pMappedData = VMA_NULL;
3531  }
3532  }
3533 }
3534 
3535 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
3536 {
3537  VkResult finalResult = VK_SUCCESS;
3538  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
3539  {
3540  VmaBlock* pBlock = m_Blocks[i];
3541  if(pBlock->m_PersistentMap)
3542  {
3543  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
3544  VkResult localResult = vkMapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &pBlock->m_pMappedData);
3545  if(localResult != VK_SUCCESS)
3546  {
3547  finalResult = localResult;
3548  }
3549  }
3550  }
3551  return finalResult;
3552 }
3553 
3554 void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const
3555 {
3556  for(uint32_t allocIndex = 0; allocIndex < m_Blocks.size(); ++allocIndex)
3557  {
3558  const VmaBlock* const pBlock = m_Blocks[allocIndex];
3559  VMA_ASSERT(pBlock);
3560  VMA_HEAVY_ASSERT(pBlock->Validate());
3561  VmaStatInfo allocationStatInfo;
3562  CalcAllocationStatInfo(allocationStatInfo, *pBlock);
3563  VmaAddStatInfo(pStats->total, allocationStatInfo);
3564  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
3565  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
3566  }
3567 }
3568 
3570 // VmaDefragmentator
3571 
3572 class VmaDefragmentator
3573 {
3574  VkDevice m_hDevice;
3575  const VkAllocationCallbacks* m_pAllocationCallbacks;
3576  VkDeviceSize m_BufferImageGranularity;
3577  uint32_t m_MemTypeIndex;
3578  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3579  VkDeviceSize m_BytesMoved;
3580  uint32_t m_AllocationsMoved;
3581 
3582  struct AllocationInfo
3583  {
3584  VmaAllocation m_hAllocation;
3585  VkBool32* m_pChanged;
3586 
3587  AllocationInfo() :
3588  m_hAllocation(VK_NULL_HANDLE),
3589  m_pChanged(VMA_NULL)
3590  {
3591  }
3592  };
3593 
3594  struct AllocationInfoSizeGreater
3595  {
3596  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3597  {
3598  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3599  }
3600  };
3601 
3602  // Used between AddAllocation and Defragment.
3603  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3604 
3605  struct BlockInfo
3606  {
3607  VmaBlock* m_pBlock;
3608  bool m_HasNonMovableAllocations;
3609  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3610 
3611  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3612  m_pBlock(VMA_NULL),
3613  m_HasNonMovableAllocations(true),
3614  m_Allocations(pAllocationCallbacks),
3615  m_pMappedDataForDefragmentation(VMA_NULL)
3616  {
3617  }
3618 
3619  void CalcHasNonMovableAllocations()
3620  {
3621  const size_t blockAllocCount =
3622  m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount;
3623  const size_t defragmentAllocCount = m_Allocations.size();
3624  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3625  }
3626 
3627  void SortAllocationsBySizeDescecnding()
3628  {
3629  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3630  }
3631 
3632  VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData)
3633  {
3634  // It has already been mapped for defragmentation.
3635  if(m_pMappedDataForDefragmentation)
3636  {
3637  *ppMappedData = m_pMappedDataForDefragmentation;
3638  return VK_SUCCESS;
3639  }
3640 
3641  // It is persistently mapped.
3642  if(m_pBlock->m_PersistentMap)
3643  {
3644  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
3645  *ppMappedData = m_pBlock->m_pMappedData;
3646  return VK_SUCCESS;
3647  }
3648 
3649  // Map on first usage.
3650  VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation);
3651  *ppMappedData = m_pMappedDataForDefragmentation;
3652  return res;
3653  }
3654 
3655  void Unmap(VkDevice hDevice)
3656  {
3657  if(m_pMappedDataForDefragmentation != VMA_NULL)
3658  {
3659  vkUnmapMemory(hDevice, m_pBlock->m_hMemory);
3660  }
3661  }
3662 
3663  private:
3664  // Not null if mapped for defragmentation only, not persistently mapped.
3665  void* m_pMappedDataForDefragmentation;
3666  };
3667 
3668  struct BlockPointerLess
3669  {
3670  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaBlock* pRhsBlock) const
3671  {
3672  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3673  }
3674  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3675  {
3676  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3677  }
3678  };
3679 
3680  // 1. Blocks with some non-movable allocations go first.
3681  // 2. Blocks with smaller sumFreeSize go first.
3682  struct BlockInfoCompareMoveDestination
3683  {
3684  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3685  {
3686  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3687  {
3688  return true;
3689  }
3690  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3691  {
3692  return false;
3693  }
3694  if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize)
3695  {
3696  return true;
3697  }
3698  return false;
3699  }
3700  };
3701 
3702  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3703  BlockInfoVector m_Blocks;
3704 
3705  VkResult DefragmentRound(
3706  VkDeviceSize maxBytesToMove,
3707  uint32_t maxAllocationsToMove);
3708 
3709  static bool MoveMakesSense(
3710  size_t dstBlockIndex, VkDeviceSize dstOffset,
3711  size_t srcBlockIndex, VkDeviceSize srcOffset);
3712 
3713 public:
3714  VmaDefragmentator(
3715  VkDevice hDevice,
3716  const VkAllocationCallbacks* pAllocationCallbacks,
3717  VkDeviceSize bufferImageGranularity,
3718  uint32_t memTypeIndex,
3719  VMA_BLOCK_VECTOR_TYPE blockVectorType);
3720 
3721  ~VmaDefragmentator();
3722 
3723  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3724  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3725 
3726  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3727 
3728  VkResult Defragment(
3729  VmaBlockVector* pBlockVector,
3730  VkDeviceSize maxBytesToMove,
3731  uint32_t maxAllocationsToMove);
3732 };
3733 
3734 VmaDefragmentator::VmaDefragmentator(
3735  VkDevice hDevice,
3736  const VkAllocationCallbacks* pAllocationCallbacks,
3737  VkDeviceSize bufferImageGranularity,
3738  uint32_t memTypeIndex,
3739  VMA_BLOCK_VECTOR_TYPE blockVectorType) :
3740  m_hDevice(hDevice),
3741  m_pAllocationCallbacks(pAllocationCallbacks),
3742  m_BufferImageGranularity(bufferImageGranularity),
3743  m_MemTypeIndex(memTypeIndex),
3744  m_BlockVectorType(blockVectorType),
3745  m_BytesMoved(0),
3746  m_AllocationsMoved(0),
3747  m_Allocations(VmaStlAllocator<AllocationInfo>(pAllocationCallbacks)),
3748  m_Blocks(VmaStlAllocator<BlockInfo*>(pAllocationCallbacks))
3749 {
3750 }
3751 
3752 VmaDefragmentator::~VmaDefragmentator()
3753 {
3754  for(size_t i = m_Blocks.size(); i--; )
3755  {
3756  vma_delete(m_pAllocationCallbacks, m_Blocks[i]);
3757  }
3758 }
3759 
3760 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
3761 {
3762  AllocationInfo allocInfo;
3763  allocInfo.m_hAllocation = hAlloc;
3764  allocInfo.m_pChanged = pChanged;
3765  m_Allocations.push_back(allocInfo);
3766 }
3767 
3768 VkResult VmaDefragmentator::DefragmentRound(
3769  VkDeviceSize maxBytesToMove,
3770  uint32_t maxAllocationsToMove)
3771 {
3772  if(m_Blocks.empty())
3773  {
3774  return VK_SUCCESS;
3775  }
3776 
3777  size_t srcBlockIndex = m_Blocks.size() - 1;
3778  size_t srcAllocIndex = SIZE_MAX;
3779  for(;;)
3780  {
3781  // 1. Find next allocation to move.
3782  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
3783  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
3784  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
3785  {
3786  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
3787  {
3788  // Finished: no more allocations to process.
3789  if(srcBlockIndex == 0)
3790  {
3791  return VK_SUCCESS;
3792  }
3793  else
3794  {
3795  --srcBlockIndex;
3796  srcAllocIndex = SIZE_MAX;
3797  }
3798  }
3799  else
3800  {
3801  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
3802  }
3803  }
3804 
3805  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
3806  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
3807 
3808  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
3809  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
3810  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
3811  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
3812 
3813  // 2. Try to find new place for this allocation in preceding or current block.
3814  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
3815  {
3816  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
3817  VmaAllocationRequest dstAllocRequest;
3818  if(pDstBlockInfo->m_pBlock->CreateAllocationRequest(
3819  m_BufferImageGranularity,
3820  size,
3821  alignment,
3822  suballocType,
3823  &dstAllocRequest) &&
3824  MoveMakesSense(
3825  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
3826  {
3827  // Reached limit on number of allocations or bytes to move.
3828  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
3829  (m_BytesMoved + size > maxBytesToMove))
3830  {
3831  return VK_INCOMPLETE;
3832  }
3833 
3834  void* pDstMappedData = VMA_NULL;
3835  VkResult res = pDstBlockInfo->EnsureMapping(m_hDevice, &pDstMappedData);
3836  if(res != VK_SUCCESS)
3837  {
3838  return res;
3839  }
3840 
3841  void* pSrcMappedData = VMA_NULL;
3842  res = pSrcBlockInfo->EnsureMapping(m_hDevice, &pSrcMappedData);
3843  if(res != VK_SUCCESS)
3844  {
3845  return res;
3846  }
3847 
3848  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
3849  memcpy(
3850  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
3851  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
3852  size);
3853 
3854  pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size);
3855  pSrcBlockInfo->m_pBlock->Free(allocInfo.m_hAllocation);
3856 
3857  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
3858 
3859  if(allocInfo.m_pChanged != VMA_NULL)
3860  {
3861  *allocInfo.m_pChanged = VK_TRUE;
3862  }
3863 
3864  ++m_AllocationsMoved;
3865  m_BytesMoved += size;
3866 
3867  VectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
3868 
3869  break;
3870  }
3871  }
3872 
3873  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
3874 
3875  if(srcAllocIndex > 0)
3876  {
3877  --srcAllocIndex;
3878  }
3879  else
3880  {
3881  if(srcBlockIndex > 0)
3882  {
3883  --srcBlockIndex;
3884  srcAllocIndex = SIZE_MAX;
3885  }
3886  else
3887  {
3888  return VK_SUCCESS;
3889  }
3890  }
3891  }
3892 }
3893 
3894 VkResult VmaDefragmentator::Defragment(
3895  VmaBlockVector* pBlockVector,
3896  VkDeviceSize maxBytesToMove,
3897  uint32_t maxAllocationsToMove)
3898 {
3899  if(m_Allocations.empty())
3900  {
3901  return VK_SUCCESS;
3902  }
3903 
3904  // Create block info for each block.
3905  const size_t blockCount = pBlockVector->m_Blocks.size();
3906  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3907  {
3908  BlockInfo* pBlockInfo = vma_new(m_pAllocationCallbacks, BlockInfo)(m_pAllocationCallbacks);
3909  pBlockInfo->m_pBlock = pBlockVector->m_Blocks[blockIndex];
3910  m_Blocks.push_back(pBlockInfo);
3911  }
3912 
3913  // Sort them by m_pBlock pointer value.
3914  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
3915 
3916  // Move allocation infos from m_Allocations to appropriate m_Blocks[i].m_Allocations.
3917  for(size_t allocIndex = 0, allocCount = m_Allocations.size(); allocIndex < allocCount; ++allocIndex)
3918  {
3919  AllocationInfo& allocInfo = m_Allocations[allocIndex];
3920  VmaBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
3921  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
3922  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
3923  {
3924  (*it)->m_Allocations.push_back(allocInfo);
3925  }
3926  else
3927  {
3928  VMA_ASSERT(0);
3929  }
3930  }
3931  m_Allocations.clear();
3932 
3933  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3934  {
3935  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
3936  pBlockInfo->CalcHasNonMovableAllocations();
3937  pBlockInfo->SortAllocationsBySizeDescecnding();
3938  }
3939 
3940  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
3941  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
3942 
3943  // Execute defragmentation round (the main part).
3944  VkResult result = VK_SUCCESS;
3945  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
3946  {
3947  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
3948  }
3949 
3950  // Unmap blocks that were mapped for defragmentation.
3951  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3952  {
3953  m_Blocks[blockIndex]->Unmap(m_hDevice);
3954  }
3955 
3956  return result;
3957 }
3958 
3959 bool VmaDefragmentator::MoveMakesSense(
3960  size_t dstBlockIndex, VkDeviceSize dstOffset,
3961  size_t srcBlockIndex, VkDeviceSize srcOffset)
3962 {
3963  if(dstBlockIndex < srcBlockIndex)
3964  {
3965  return true;
3966  }
3967  if(dstBlockIndex > srcBlockIndex)
3968  {
3969  return false;
3970  }
3971  if(dstOffset < srcOffset)
3972  {
3973  return true;
3974  }
3975  return false;
3976 }
3977 
3979 // VmaAllocator_T
3980 
3981 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
3982  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
3983  m_PhysicalDevice(pCreateInfo->physicalDevice),
3984  m_hDevice(pCreateInfo->device),
3985  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
3986  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
3987  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
3988  m_PreferredLargeHeapBlockSize(0),
3989  m_PreferredSmallHeapBlockSize(0),
3990  m_UnmapPersistentlyMappedMemoryCounter(0)
3991 {
3992  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
3993 
3994  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
3995  memset(&m_MemProps, 0, sizeof(m_MemProps));
3996  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
3997 
3998  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
3999  memset(&m_HasEmptyBlock, 0, sizeof(m_HasEmptyBlock));
4000  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
4001 
4002  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
4003  {
4004  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
4005  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
4006  }
4007 
4008  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
4009  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
4010  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
4011  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
4012 
4013  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
4014  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
4015 
4016  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
4017  {
4018  for(size_t j = 0; j < VMA_BLOCK_VECTOR_TYPE_COUNT; ++j)
4019  {
4020  m_pBlockVectors[i][j] = vma_new(this, VmaBlockVector)(this);
4021  m_pOwnAllocations[i][j] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
4022  }
4023  }
4024 }
4025 
4026 VmaAllocator_T::~VmaAllocator_T()
4027 {
4028  for(size_t i = GetMemoryTypeCount(); i--; )
4029  {
4030  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
4031  {
4032  vma_delete(this, m_pOwnAllocations[i][j]);
4033  vma_delete(this, m_pBlockVectors[i][j]);
4034  }
4035  }
4036 }
4037 
4038 VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const
4039 {
4040  VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
4041  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
4042  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
4043 }
4044 
4045 VkResult VmaAllocator_T::AllocateMemoryOfType(
4046  const VkMemoryRequirements& vkMemReq,
4047  const VmaMemoryRequirements& vmaMemReq,
4048  uint32_t memTypeIndex,
4049  VmaSuballocationType suballocType,
4050  VmaAllocation* pAllocation)
4051 {
4052  VMA_ASSERT(pAllocation != VMA_NULL);
4053  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
4054 
4055  const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex);
4056  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
4057  const bool ownMemory =
4058  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 ||
4059  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
4060  ((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) == 0 &&
4061  vkMemReq.size > preferredBlockSize / 2);
4062 
4063  if(ownMemory)
4064  {
4065  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4066  {
4067  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4068  }
4069  else
4070  {
4071  return AllocateOwnMemory(
4072  vkMemReq.size,
4073  suballocType,
4074  memTypeIndex,
4075  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
4076  vmaMemReq.pUserData,
4077  pAllocation);
4078  }
4079  }
4080  else
4081  {
4082  uint32_t blockVectorType = VmaMemoryRequirementFlagsToBlockVectorType(vmaMemReq.flags);
4083 
4084  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4085  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4086  VMA_ASSERT(blockVector);
4087 
4088  // 1. Search existing allocations.
4089  // Forward order - prefer blocks with smallest amount of free space.
4090  for(size_t allocIndex = 0; allocIndex < blockVector->m_Blocks.size(); ++allocIndex )
4091  {
4092  VmaBlock* const pBlock = blockVector->m_Blocks[allocIndex];
4093  VMA_ASSERT(pBlock);
4094  VmaAllocationRequest allocRequest = {};
4095  // Check if can allocate from pBlock.
4096  if(pBlock->CreateAllocationRequest(
4097  GetBufferImageGranularity(),
4098  vkMemReq.size,
4099  vkMemReq.alignment,
4100  suballocType,
4101  &allocRequest))
4102  {
4103  // We no longer have an empty Allocation.
4104  if(pBlock->IsEmpty())
4105  {
4106  m_HasEmptyBlock[memTypeIndex] = false;
4107  }
4108  // Allocate from this pBlock.
4109  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
4110  *pAllocation = vma_new(this, VmaAllocation_T)();
4111  (*pAllocation)->InitBlockAllocation(
4112  pBlock,
4113  allocRequest.offset,
4114  vkMemReq.alignment,
4115  vkMemReq.size,
4116  suballocType,
4117  vmaMemReq.pUserData);
4118  VMA_HEAVY_ASSERT(pBlock->Validate());
4119  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex);
4120  return VK_SUCCESS;
4121  }
4122  }
4123 
4124  // 2. Create new Allocation.
4125  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4126  {
4127  VMA_DEBUG_LOG(" FAILED due to VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT");
4128  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4129  }
4130  else
4131  {
4132  // Start with full preferredBlockSize.
4133  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
4134  allocInfo.memoryTypeIndex = memTypeIndex;
4135  allocInfo.allocationSize = preferredBlockSize;
4136  VkDeviceMemory mem = VK_NULL_HANDLE;
4137  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4138  if(res < 0)
4139  {
4140  // 3. Try half the size.
4141  allocInfo.allocationSize /= 2;
4142  if(allocInfo.allocationSize >= vkMemReq.size)
4143  {
4144  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4145  if(res < 0)
4146  {
4147  // 4. Try quarter the size.
4148  allocInfo.allocationSize /= 2;
4149  if(allocInfo.allocationSize >= vkMemReq.size)
4150  {
4151  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4152  }
4153  }
4154  }
4155  }
4156  if(res < 0)
4157  {
4158  // 5. Try OwnAlloc.
4159  res = AllocateOwnMemory(
4160  vkMemReq.size,
4161  suballocType,
4162  memTypeIndex,
4163  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
4164  vmaMemReq.pUserData,
4165  pAllocation);
4166  if(res == VK_SUCCESS)
4167  {
4168  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
4169  VMA_DEBUG_LOG(" Allocated as OwnMemory");
4170  return VK_SUCCESS;
4171  }
4172  else
4173  {
4174  // Everything failed: Return error code.
4175  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
4176  return res;
4177  }
4178  }
4179 
4180  // New VkDeviceMemory successfully created.
4181 
4182  // Map memory if needed.
4183  void* pMappedData = VMA_NULL;
4184  const bool persistentMap = (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0;
4185  if(persistentMap && m_UnmapPersistentlyMappedMemoryCounter == 0)
4186  {
4187  res = vkMapMemory(m_hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData);
4188  if(res < 0)
4189  {
4190  VMA_DEBUG_LOG(" vkMapMemory FAILED");
4191  vkFreeMemory(m_hDevice, mem, GetAllocationCallbacks());
4192  return res;
4193  }
4194  }
4195 
4196  // Callback.
4197  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
4198  {
4199  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, mem, allocInfo.allocationSize);
4200  }
4201 
4202  // Create new Allocation for it.
4203  VmaBlock* const pBlock = vma_new(this, VmaBlock)(this);
4204  pBlock->Init(
4205  memTypeIndex,
4206  (VMA_BLOCK_VECTOR_TYPE)blockVectorType,
4207  mem,
4208  allocInfo.allocationSize,
4209  persistentMap,
4210  pMappedData);
4211 
4212  blockVector->m_Blocks.push_back(pBlock);
4213 
4214  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
4215  VmaAllocationRequest allocRequest = {};
4216  allocRequest.freeSuballocationItem = pBlock->m_Suballocations.begin();
4217  allocRequest.offset = 0;
4218  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
4219  *pAllocation = vma_new(this, VmaAllocation_T)();
4220  (*pAllocation)->InitBlockAllocation(
4221  pBlock,
4222  allocRequest.offset,
4223  vkMemReq.alignment,
4224  vkMemReq.size,
4225  suballocType,
4226  vmaMemReq.pUserData);
4227  VMA_HEAVY_ASSERT(pBlock->Validate());
4228  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
4229  return VK_SUCCESS;
4230  }
4231  }
4232 }
4233 
4234 VkResult VmaAllocator_T::AllocateOwnMemory(
4235  VkDeviceSize size,
4236  VmaSuballocationType suballocType,
4237  uint32_t memTypeIndex,
4238  bool map,
4239  void* pUserData,
4240  VmaAllocation* pAllocation)
4241 {
4242  VMA_ASSERT(pAllocation);
4243 
4244  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
4245  allocInfo.memoryTypeIndex = memTypeIndex;
4246  allocInfo.allocationSize = size;
4247 
4248  // Allocate VkDeviceMemory.
4249  VkDeviceMemory hMemory = VK_NULL_HANDLE;
4250  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &hMemory);
4251  if(res < 0)
4252  {
4253  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
4254  return res;
4255  }
4256 
4257  void* pMappedData = nullptr;
4258  if(map)
4259  {
4260  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
4261  {
4262  res = vkMapMemory(m_hDevice, hMemory, 0, VK_WHOLE_SIZE, 0, &pMappedData);
4263  if(res < 0)
4264  {
4265  VMA_DEBUG_LOG(" vkMapMemory FAILED");
4266  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
4267  return res;
4268  }
4269  }
4270  }
4271 
4272  // Callback.
4273  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
4274  {
4275  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, hMemory, size);
4276  }
4277 
4278  *pAllocation = vma_new(this, VmaAllocation_T)();
4279  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
4280 
4281  // Register it in m_pOwnAllocations.
4282  {
4283  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4284  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
4285  VMA_ASSERT(pOwnAllocations);
4286  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
4287  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
4288  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4289  pOwnAllocationsBeg,
4290  pOwnAllocationsEnd,
4291  *pAllocation,
4292  VmaPointerLess()) - pOwnAllocationsBeg;
4293  VectorInsert(*pOwnAllocations, indexToInsert, *pAllocation);
4294  }
4295 
4296  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
4297 
4298  return VK_SUCCESS;
4299 }
4300 
4301 VkResult VmaAllocator_T::AllocateMemory(
4302  const VkMemoryRequirements& vkMemReq,
4303  const VmaMemoryRequirements& vmaMemReq,
4304  VmaSuballocationType suballocType,
4305  VmaAllocation* pAllocation)
4306 {
4307  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 &&
4308  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4309  {
4310  VMA_ASSERT(0 && "Specifying VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT together with VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT makes no sense.");
4311  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4312  }
4313 
4314  // Bit mask of memory Vulkan types acceptable for this allocation.
4315  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
4316  uint32_t memTypeIndex = UINT32_MAX;
4317  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
4318  if(res == VK_SUCCESS)
4319  {
4320  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
4321  // Succeeded on first try.
4322  if(res == VK_SUCCESS)
4323  {
4324  return res;
4325  }
4326  // Allocation from this memory type failed. Try other compatible memory types.
4327  else
4328  {
4329  for(;;)
4330  {
4331  // Remove old memTypeIndex from list of possibilities.
4332  memoryTypeBits &= ~(1u << memTypeIndex);
4333  // Find alternative memTypeIndex.
4334  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
4335  if(res == VK_SUCCESS)
4336  {
4337  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
4338  // Allocation from this alternative memory type succeeded.
4339  if(res == VK_SUCCESS)
4340  {
4341  return res;
4342  }
4343  // else: Allocation from this memory type failed. Try next one - next loop iteration.
4344  }
4345  // No other matching memory type index could be found.
4346  else
4347  {
4348  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
4349  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4350  }
4351  }
4352  }
4353  }
4354  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
4355  else
4356  return res;
4357 }
4358 
4359 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
4360 {
4361  VMA_ASSERT(allocation);
4362 
4363  if(allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
4364  {
4365  VmaBlock* pBlockToDelete = VMA_NULL;
4366 
4367  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
4368  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
4369  {
4370  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4371 
4372  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4373  VmaBlock* pBlock = allocation->GetBlock();
4374 
4375  pBlock->Free(allocation);
4376  VMA_HEAVY_ASSERT(pBlock->Validate());
4377 
4378  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
4379 
4380  // pBlock became empty after this deallocation.
4381  if(pBlock->IsEmpty())
4382  {
4383  // Already has empty Allocation. We don't want to have two, so delete this one.
4384  if(m_HasEmptyBlock[memTypeIndex])
4385  {
4386  pBlockToDelete = pBlock;
4387  pBlockVector->Remove(pBlock);
4388  }
4389  // We now have first empty Allocation.
4390  else
4391  {
4392  m_HasEmptyBlock[memTypeIndex] = true;
4393  }
4394  }
4395  // Must be called after srcBlockIndex is used, because later it may become invalid!
4396  pBlockVector->IncrementallySortBlocks();
4397  }
4398  // Destruction of a free Allocation. Deferred until this point, outside of mutex
4399  // lock, for performance reason.
4400  if(pBlockToDelete != VMA_NULL)
4401  {
4402  VMA_DEBUG_LOG(" Deleted empty allocation");
4403  pBlockToDelete->Destroy(this);
4404  vma_delete(this, pBlockToDelete);
4405  }
4406 
4407  vma_delete(this, allocation);
4408  }
4409  else // VmaAllocation_T::ALLOCATION_TYPE_OWN
4410  {
4411  FreeOwnMemory(allocation);
4412  }
4413 }
4414 
4415 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
4416 {
4417  InitStatInfo(pStats->total);
4418  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
4419  InitStatInfo(pStats->memoryType[i]);
4420  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
4421  InitStatInfo(pStats->memoryHeap[i]);
4422 
4423  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4424  {
4425  VmaMutexLock allocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4426  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4427  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4428  {
4429  const VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4430  VMA_ASSERT(pBlockVector);
4431  pBlockVector->AddStats(pStats, memTypeIndex, heapIndex);
4432  }
4433  }
4434 
4435  VmaPostprocessCalcStatInfo(pStats->total);
4436  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
4437  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
4438  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
4439  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
4440 }
4441 
4442 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
4443 
4444 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
4445 {
4446  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
4447  {
4448  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
4449  {
4450  for(size_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
4451  {
4452  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
4453  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
4454  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4455  {
4456  // Process OwnAllocations.
4457  {
4458  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4459  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4460  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
4461  {
4462  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
4463  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(m_hDevice);
4464  }
4465  }
4466 
4467  // Process normal Allocations.
4468  {
4469  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4470  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4471  pBlockVector->UnmapPersistentlyMappedMemory();
4472  }
4473  }
4474  }
4475  }
4476  }
4477 }
4478 
4479 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
4480 {
4481  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
4482  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
4483  {
4484  VkResult finalResult = VK_SUCCESS;
4485  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
4486  {
4487  for(size_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
4488  {
4489  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
4490  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
4491  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4492  {
4493  // Process OwnAllocations.
4494  {
4495  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4496  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4497  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
4498  {
4499  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
4500  hAlloc->OwnAllocMapPersistentlyMappedMemory(m_hDevice);
4501  }
4502  }
4503 
4504  // Process normal Allocations.
4505  {
4506  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4507  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4508  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
4509  if(localResult != VK_SUCCESS)
4510  {
4511  finalResult = localResult;
4512  }
4513  }
4514  }
4515  }
4516  }
4517  return finalResult;
4518  }
4519  else
4520  return VK_SUCCESS;
4521 }
4522 
4523 VkResult VmaAllocator_T::Defragment(
4524  VmaAllocation* pAllocations,
4525  size_t allocationCount,
4526  VkBool32* pAllocationsChanged,
4527  const VmaDefragmentationInfo* pDefragmentationInfo,
4528  VmaDefragmentationStats* pDefragmentationStats)
4529 {
4530  if(pAllocationsChanged != VMA_NULL)
4531  {
4532  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
4533  }
4534  if(pDefragmentationStats != VMA_NULL)
4535  {
4536  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
4537  }
4538 
4539  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
4540  {
4541  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
4542  return VK_ERROR_MEMORY_MAP_FAILED;
4543  }
4544 
4545  // Initialize defragmentators per memory type.
4546  const VkDeviceSize bufferImageGranularity = GetBufferImageGranularity();
4547  VmaDefragmentator* pDefragmentators[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
4548  memset(pDefragmentators, 0, sizeof(pDefragmentators));
4549  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4550  {
4551  // Only HOST_VISIBLE memory types can be defragmented.
4552  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4553  {
4554  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4555  {
4556  pDefragmentators[memTypeIndex][blockVectorType] = vma_new(this, VmaDefragmentator)(
4557  m_hDevice,
4558  GetAllocationCallbacks(),
4559  bufferImageGranularity,
4560  memTypeIndex,
4561  (VMA_BLOCK_VECTOR_TYPE)blockVectorType);
4562  }
4563  }
4564  }
4565 
4566  // Dispatch pAllocations among defragmentators.
4567  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
4568  {
4569  VmaAllocation hAlloc = pAllocations[allocIndex];
4570  VMA_ASSERT(hAlloc);
4571  if(hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
4572  {
4573  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
4574  // Only HOST_VISIBLE memory types can be defragmented.
4575  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4576  {
4577  const VMA_BLOCK_VECTOR_TYPE blockVectorType = hAlloc->GetBlockVectorType();
4578  VkBool32* pChanged = (pAllocationsChanged != VMA_NULL) ?
4579  &pAllocationsChanged[allocIndex] : VMA_NULL;
4580  pDefragmentators[memTypeIndex][blockVectorType]->AddAllocation(hAlloc, pChanged);
4581  }
4582  // else: skip this allocation, cannot move it.
4583  }
4584  // else ALLOCATION_TYPE_OWN: skip this allocation, nothing to defragment.
4585  }
4586 
4587  VkResult result = VK_SUCCESS;
4588 
4589  // Main processing.
4590  VkDeviceSize maxBytesToMove = SIZE_MAX;
4591  uint32_t maxAllocationsToMove = UINT32_MAX;
4592  if(pDefragmentationInfo != VMA_NULL)
4593  {
4594  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
4595  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
4596  }
4597  for(uint32_t memTypeIndex = 0;
4598  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
4599  ++memTypeIndex)
4600  {
4601  // Only HOST_VISIBLE memory types can be defragmented.
4602  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4603  {
4604  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4605 
4606  for(uint32_t blockVectorType = 0;
4607  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
4608  ++blockVectorType)
4609  {
4610  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4611 
4612  // Defragment.
4613  result = pDefragmentators[memTypeIndex][blockVectorType]->Defragment(pBlockVector, maxBytesToMove, maxAllocationsToMove);
4614 
4615  // Accumulate statistics.
4616  if(pDefragmentationStats != VMA_NULL)
4617  {
4618  const VkDeviceSize bytesMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetBytesMoved();
4619  const uint32_t allocationsMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetAllocationsMoved();
4620  pDefragmentationStats->bytesMoved += bytesMoved;
4621  pDefragmentationStats->allocationsMoved += allocationsMoved;
4622  VMA_ASSERT(bytesMoved <= maxBytesToMove);
4623  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
4624  maxBytesToMove -= bytesMoved;
4625  maxAllocationsToMove -= allocationsMoved;
4626  }
4627 
4628  // Free empty blocks.
4629  for(size_t blockIndex = pBlockVector->m_Blocks.size(); blockIndex--; )
4630  {
4631  VmaBlock* pBlock = pBlockVector->m_Blocks[blockIndex];
4632  if(pBlock->IsEmpty())
4633  {
4634  if(pDefragmentationStats != VMA_NULL)
4635  {
4636  ++pDefragmentationStats->deviceMemoryBlocksFreed;
4637  pDefragmentationStats->bytesFreed += pBlock->m_Size;
4638  }
4639 
4640  VectorRemove(pBlockVector->m_Blocks, blockIndex);
4641  pBlock->Destroy(this);
4642  vma_delete(this, pBlock);
4643  }
4644  }
4645 
4646  // All block vector types processed: we can be sure that all empty allocations have been freed.
4647  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_COUNT - 1)
4648  {
4649  m_HasEmptyBlock[memTypeIndex] = false;
4650  }
4651  }
4652  }
4653  }
4654 
4655  // Destroy defragmentators.
4656  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
4657  {
4658  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
4659  {
4660  vma_delete(this, pDefragmentators[memTypeIndex][blockVectorType]);
4661  }
4662  }
4663 
4664  return result;
4665 }
4666 
4667 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
4668 {
4669  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
4670  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
4671  pAllocationInfo->offset = hAllocation->GetOffset();
4672  pAllocationInfo->size = hAllocation->GetSize();
4673  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
4674  pAllocationInfo->pUserData = hAllocation->GetUserData();
4675 }
4676 
4677 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
4678 {
4679  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
4680 
4681  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
4682  {
4683  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4684  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
4685  VMA_ASSERT(pOwnAllocations);
4686  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
4687  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
4688  VmaAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess(
4689  pOwnAllocationsBeg,
4690  pOwnAllocationsEnd,
4691  allocation,
4692  VmaPointerLess());
4693  if(pOwnAllocationIt != pOwnAllocationsEnd)
4694  {
4695  const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg;
4696  VectorRemove(*pOwnAllocations, ownAllocationIndex);
4697  }
4698  else
4699  {
4700  VMA_ASSERT(0);
4701  }
4702  }
4703 
4704  VkDeviceMemory hMemory = allocation->GetMemory();
4705 
4706  // Callback.
4707  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
4708  {
4709  (*m_DeviceMemoryCallbacks.pfnFree)(this, memTypeIndex, hMemory, allocation->GetSize());
4710  }
4711 
4712  if(allocation->GetMappedData() != VMA_NULL)
4713  {
4714  vkUnmapMemory(m_hDevice, hMemory);
4715  }
4716 
4717  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
4718 
4719  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
4720 
4721  vma_delete(this, allocation);
4722 }
4723 
4724 #if VMA_STATS_STRING_ENABLED
4725 
4726 void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb)
4727 {
4728  bool ownAllocationsStarted = false;
4729  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4730  {
4731  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4732  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4733  {
4734  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
4735  VMA_ASSERT(pOwnAllocVector);
4736  if(pOwnAllocVector->empty() == false)
4737  {
4738  if(ownAllocationsStarted)
4739  {
4740  sb.Add(",\n\t\"Type ");
4741  }
4742  else
4743  {
4744  sb.Add(",\n\"OwnAllocations\": {\n\t\"Type ");
4745  ownAllocationsStarted = true;
4746  }
4747  sb.AddNumber(memTypeIndex);
4748  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
4749  {
4750  sb.Add(" Mapped");
4751  }
4752  sb.Add("\": [");
4753 
4754  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
4755  {
4756  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
4757  if(i > 0)
4758  {
4759  sb.Add(",\n\t\t{ \"Size\": ");
4760  }
4761  else
4762  {
4763  sb.Add("\n\t\t{ \"Size\": ");
4764  }
4765  sb.AddNumber(hAlloc->GetSize());
4766  sb.Add(", \"Type\": ");
4767  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
4768  sb.Add(" }");
4769  }
4770 
4771  sb.Add("\n\t]");
4772  }
4773  }
4774  }
4775  if(ownAllocationsStarted)
4776  {
4777  sb.Add("\n}");
4778  }
4779 
4780  {
4781  bool allocationsStarted = false;
4782  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4783  {
4784  VmaMutexLock globalAllocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4785  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4786  {
4787  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
4788  {
4789  if(allocationsStarted)
4790  {
4791  sb.Add(",\n\t\"Type ");
4792  }
4793  else
4794  {
4795  sb.Add(",\n\"Allocations\": {\n\t\"Type ");
4796  allocationsStarted = true;
4797  }
4798  sb.AddNumber(memTypeIndex);
4799  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
4800  {
4801  sb.Add(" Mapped");
4802  }
4803  sb.Add("\": [");
4804 
4805  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(sb);
4806 
4807  sb.Add("\n\t]");
4808  }
4809  }
4810  }
4811  if(allocationsStarted)
4812  {
4813  sb.Add("\n}");
4814  }
4815  }
4816 }
4817 
4818 #endif // #if VMA_STATS_STRING_ENABLED
4819 
4820 static VkResult AllocateMemoryForImage(
4821  VmaAllocator allocator,
4822  VkImage image,
4823  const VmaMemoryRequirements* pMemoryRequirements,
4824  VmaSuballocationType suballocType,
4825  VmaAllocation* pAllocation)
4826 {
4827  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pMemoryRequirements && pAllocation);
4828 
4829  VkMemoryRequirements vkMemReq = {};
4830  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
4831 
4832  return allocator->AllocateMemory(
4833  vkMemReq,
4834  *pMemoryRequirements,
4835  suballocType,
4836  pAllocation);
4837 }
4838 
4840 // Public interface
4841 
4842 VkResult vmaCreateAllocator(
4843  const VmaAllocatorCreateInfo* pCreateInfo,
4844  VmaAllocator* pAllocator)
4845 {
4846  VMA_ASSERT(pCreateInfo && pAllocator);
4847  VMA_DEBUG_LOG("vmaCreateAllocator");
4848  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
4849  return VK_SUCCESS;
4850 }
4851 
4852 void vmaDestroyAllocator(
4853  VmaAllocator allocator)
4854 {
4855  if(allocator != VK_NULL_HANDLE)
4856  {
4857  VMA_DEBUG_LOG("vmaDestroyAllocator");
4858  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
4859  vma_delete(&allocationCallbacks, allocator);
4860  }
4861 }
4862 
4863 void vmaGetPhysicalDeviceProperties(
4864  VmaAllocator allocator,
4865  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
4866 {
4867  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
4868  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
4869 }
4870 
4871 void vmaGetMemoryProperties(
4872  VmaAllocator allocator,
4873  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
4874 {
4875  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
4876  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
4877 }
4878 
4879 void vmaGetMemoryTypeProperties(
4880  VmaAllocator allocator,
4881  uint32_t memoryTypeIndex,
4882  VkMemoryPropertyFlags* pFlags)
4883 {
4884  VMA_ASSERT(allocator && pFlags);
4885  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
4886  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
4887 }
4888 
4889 void vmaCalculateStats(
4890  VmaAllocator allocator,
4891  VmaStats* pStats)
4892 {
4893  VMA_ASSERT(allocator && pStats);
4894  VMA_DEBUG_GLOBAL_MUTEX_LOCK
4895  allocator->CalculateStats(pStats);
4896 }
4897 
4898 #if VMA_STATS_STRING_ENABLED
4899 
4900 void vmaBuildStatsString(
4901  VmaAllocator allocator,
4902  char** ppStatsString,
4903  VkBool32 detailedMap)
4904 {
4905  VMA_ASSERT(allocator && ppStatsString);
4906  VMA_DEBUG_GLOBAL_MUTEX_LOCK
4907 
4908  VmaStringBuilder sb(allocator);
4909  {
4910  VmaStats stats;
4911  allocator->CalculateStats(&stats);
4912 
4913  sb.Add("{\n\"Total\": ");
4914  VmaPrintStatInfo(sb, stats.total);
4915 
4916  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
4917  {
4918  sb.Add(",\n\"Heap ");
4919  sb.AddNumber(heapIndex);
4920  sb.Add("\": {\n\t\"Size\": ");
4921  sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
4922  sb.Add(",\n\t\"Flags\": ");
4923  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
4924  {
4925  sb.AddString("DEVICE_LOCAL");
4926  }
4927  else
4928  {
4929  sb.AddString("");
4930  }
4931  if(stats.memoryHeap[heapIndex].AllocationCount > 0)
4932  {
4933  sb.Add(",\n\t\"Stats:\": ");
4934  VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]);
4935  }
4936 
4937  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
4938  {
4939  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
4940  {
4941  sb.Add(",\n\t\"Type ");
4942  sb.AddNumber(typeIndex);
4943  sb.Add("\": {\n\t\t\"Flags\": \"");
4944  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
4945  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4946  {
4947  sb.Add(" DEVICE_LOCAL");
4948  }
4949  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4950  {
4951  sb.Add(" HOST_VISIBLE");
4952  }
4953  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
4954  {
4955  sb.Add(" HOST_COHERENT");
4956  }
4957  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
4958  {
4959  sb.Add(" HOST_CACHED");
4960  }
4961  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
4962  {
4963  sb.Add(" LAZILY_ALLOCATED");
4964  }
4965  sb.Add("\"");
4966  if(stats.memoryType[typeIndex].AllocationCount > 0)
4967  {
4968  sb.Add(",\n\t\t\"Stats\": ");
4969  VmaPrintStatInfo(sb, stats.memoryType[typeIndex]);
4970  }
4971  sb.Add("\n\t}");
4972  }
4973  }
4974  sb.Add("\n}");
4975  }
4976  if(detailedMap == VK_TRUE)
4977  {
4978  allocator->PrintDetailedMap(sb);
4979  }
4980  sb.Add("\n}\n");
4981  }
4982 
4983  const size_t len = sb.GetLength();
4984  char* const pChars = vma_new_array(allocator, char, len + 1);
4985  if(len > 0)
4986  {
4987  memcpy(pChars, sb.GetData(), len);
4988  }
4989  pChars[len] = '\0';
4990  *ppStatsString = pChars;
4991 }
4992 
4993 void vmaFreeStatsString(
4994  VmaAllocator allocator,
4995  char* pStatsString)
4996 {
4997  if(pStatsString != VMA_NULL)
4998  {
4999  VMA_ASSERT(allocator);
5000  size_t len = strlen(pStatsString);
5001  vma_delete_array(allocator, pStatsString, len + 1);
5002  }
5003 }
5004 
5005 #endif // #if VMA_STATS_STRING_ENABLED
5006 
5009 VkResult vmaFindMemoryTypeIndex(
5010  VmaAllocator allocator,
5011  uint32_t memoryTypeBits,
5012  const VmaMemoryRequirements* pMemoryRequirements,
5013  uint32_t* pMemoryTypeIndex)
5014 {
5015  VMA_ASSERT(allocator != VK_NULL_HANDLE);
5016  VMA_ASSERT(pMemoryRequirements != VMA_NULL);
5017  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
5018 
5019  uint32_t requiredFlags = pMemoryRequirements->requiredFlags;
5020  uint32_t preferredFlags = pMemoryRequirements->preferredFlags;
5021  if(preferredFlags == 0)
5022  {
5023  preferredFlags = requiredFlags;
5024  }
5025  // preferredFlags, if not 0, must be a superset of requiredFlags.
5026  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
5027 
5028  // Convert usage to requiredFlags and preferredFlags.
5029  switch(pMemoryRequirements->usage)
5030  {
5031  case VMA_MEMORY_USAGE_UNKNOWN:
5032  break;
5033  case VMA_MEMORY_USAGE_GPU_ONLY:
5034  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
5035  break;
5036  case VMA_MEMORY_USAGE_CPU_ONLY:
5037  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
5038  break;
5039  case VMA_MEMORY_USAGE_CPU_TO_GPU:
5040  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5041  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
5042  break;
5043  case VMA_MEMORY_USAGE_GPU_TO_CPU:
5044  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5045  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
5046  break;
5047  default:
5048  break;
5049  }
5050 
5051  if((pMemoryRequirements->flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0)
5052  {
5053  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5054  }
5055 
5056  *pMemoryTypeIndex = UINT32_MAX;
5057  uint32_t minCost = UINT32_MAX;
5058  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
5059  memTypeIndex < allocator->GetMemoryTypeCount();
5060  ++memTypeIndex, memTypeBit <<= 1)
5061  {
5062  // This memory type is acceptable according to memoryTypeBits bitmask.
5063  if((memTypeBit & memoryTypeBits) != 0)
5064  {
5065  const VkMemoryPropertyFlags currFlags =
5066  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
5067  // This memory type contains requiredFlags.
5068  if((requiredFlags & ~currFlags) == 0)
5069  {
5070  // Calculate cost as number of bits from preferredFlags not present in this memory type.
5071  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
5072  // Remember memory type with lowest cost.
5073  if(currCost < minCost)
5074  {
5075  *pMemoryTypeIndex = memTypeIndex;
5076  if(currCost == 0)
5077  {
5078  return VK_SUCCESS;
5079  }
5080  minCost = currCost;
5081  }
5082  }
5083  }
5084  }
5085  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
5086 }
5087 
5088 VkResult vmaAllocateMemory(
5089  VmaAllocator allocator,
5090  const VkMemoryRequirements* pVkMemoryRequirements,
5091  const VmaMemoryRequirements* pVmaMemoryRequirements,
5092  VmaAllocation* pAllocation,
5093  VmaAllocationInfo* pAllocationInfo)
5094 {
5095  VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pAllocation);
5096 
5097  VMA_DEBUG_LOG("vmaAllocateMemory");
5098 
5099  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5100 
5101  return allocator->AllocateMemory(
5102  *pVkMemoryRequirements,
5103  *pVmaMemoryRequirements,
5104  VMA_SUBALLOCATION_TYPE_UNKNOWN,
5105  pAllocation);
5106 
5107  if(pAllocationInfo)
5108  {
5109  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5110  }
5111 }
5112 
5113 VkResult vmaAllocateMemoryForBuffer(
5114  VmaAllocator allocator,
5115  VkBuffer buffer,
5116  const VmaMemoryRequirements* pMemoryRequirements,
5117  VmaAllocation* pAllocation,
5118  VmaAllocationInfo* pAllocationInfo)
5119 {
5120  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
5121 
5122  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
5123 
5124  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5125 
5126  VkMemoryRequirements vkMemReq = {};
5127  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
5128 
5129  return allocator->AllocateMemory(
5130  vkMemReq,
5131  *pMemoryRequirements,
5132  VMA_SUBALLOCATION_TYPE_BUFFER,
5133  pAllocation);
5134 
5135  if(pAllocationInfo)
5136  {
5137  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5138  }
5139 }
5140 
5141 VkResult vmaAllocateMemoryForImage(
5142  VmaAllocator allocator,
5143  VkImage image,
5144  const VmaMemoryRequirements* pMemoryRequirements,
5145  VmaAllocation* pAllocation,
5146  VmaAllocationInfo* pAllocationInfo)
5147 {
5148  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
5149 
5150  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
5151 
5152  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5153 
5154  return AllocateMemoryForImage(
5155  allocator,
5156  image,
5157  pMemoryRequirements,
5158  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
5159  pAllocation);
5160 
5161  if(pAllocationInfo)
5162  {
5163  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5164  }
5165 }
5166 
5167 void vmaFreeMemory(
5168  VmaAllocator allocator,
5169  VmaAllocation allocation)
5170 {
5171  VMA_ASSERT(allocator && allocation);
5172 
5173  VMA_DEBUG_LOG("vmaFreeMemory");
5174 
5175  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5176 
5177  allocator->FreeMemory(allocation);
5178 }
5179 
5180 void vmaGetAllocationInfo(
5181  VmaAllocator allocator,
5182  VmaAllocation allocation,
5183  VmaAllocationInfo* pAllocationInfo)
5184 {
5185  VMA_ASSERT(allocator && allocation && pAllocationInfo);
5186 
5187  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5188 
5189  allocator->GetAllocationInfo(allocation, pAllocationInfo);
5190 }
5191 
5192 void vmaSetAllocationUserData(
5193  VmaAllocator allocator,
5194  VmaAllocation allocation,
5195  void* pUserData)
5196 {
5197  VMA_ASSERT(allocator && allocation);
5198 
5199  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5200 
5201  allocation->SetUserData(pUserData);
5202 }
5203 
5204 VkResult vmaMapMemory(
5205  VmaAllocator allocator,
5206  VmaAllocation allocation,
5207  void** ppData)
5208 {
5209  VMA_ASSERT(allocator && allocation && ppData);
5210 
5211  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5212 
5213  return vkMapMemory(allocator->m_hDevice, allocation->GetMemory(),
5214  allocation->GetOffset(), allocation->GetSize(), 0, ppData);
5215 }
5216 
5217 void vmaUnmapMemory(
5218  VmaAllocator allocator,
5219  VmaAllocation allocation)
5220 {
5221  VMA_ASSERT(allocator && allocation);
5222 
5223  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5224 
5225  vkUnmapMemory(allocator->m_hDevice, allocation->GetMemory());
5226 }
5227 
5228 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
5229 {
5230  VMA_ASSERT(allocator);
5231 
5232  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5233 
5234  allocator->UnmapPersistentlyMappedMemory();
5235 }
5236 
5237 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
5238 {
5239  VMA_ASSERT(allocator);
5240 
5241  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5242 
5243  return allocator->MapPersistentlyMappedMemory();
5244 }
5245 
5246 VkResult vmaDefragment(
5247  VmaAllocator allocator,
5248  VmaAllocation* pAllocations,
5249  size_t allocationCount,
5250  VkBool32* pAllocationsChanged,
5251  const VmaDefragmentationInfo *pDefragmentationInfo,
5252  VmaDefragmentationStats* pDefragmentationStats)
5253 {
5254  VMA_ASSERT(allocator && pAllocations);
5255 
5256  VMA_DEBUG_LOG("vmaDefragment");
5257 
5258  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5259 
5260  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
5261 }
5262 
5263 VkResult vmaCreateBuffer(
5264  VmaAllocator allocator,
5265  const VkBufferCreateInfo* pCreateInfo,
5266  const VmaMemoryRequirements* pMemoryRequirements,
5267  VkBuffer* pBuffer,
5268  VmaAllocation* pAllocation,
5269  VmaAllocationInfo* pAllocationInfo)
5270 {
5271  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pBuffer && pAllocation);
5272 
5273  VMA_DEBUG_LOG("vmaCreateBuffer");
5274 
5275  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5276 
5277  // 1. Create VkBuffer.
5278  VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
5279  if(res >= 0)
5280  {
5281  // 2. vkGetBufferMemoryRequirements.
5282  VkMemoryRequirements vkMemReq = {};
5283  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
5284 
5285  // 3. Allocate memory using allocator.
5286  res = allocator->AllocateMemory(
5287  vkMemReq,
5288  *pMemoryRequirements,
5289  VMA_SUBALLOCATION_TYPE_BUFFER,
5290  pAllocation);
5291  if(res >= 0)
5292  {
5293  // 3. Bind buffer with memory.
5294  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
5295  if(res >= 0)
5296  {
5297  // All steps succeeded.
5298  if(pAllocationInfo != VMA_NULL)
5299  {
5300  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5301  }
5302  return VK_SUCCESS;
5303  }
5304  allocator->FreeMemory(*pAllocation);
5305  return res;
5306  }
5307  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
5308  return res;
5309  }
5310  return res;
5311 }
5312 
5313 void vmaDestroyBuffer(
5314  VmaAllocator allocator,
5315  VkBuffer buffer,
5316  VmaAllocation allocation)
5317 {
5318  if(buffer != VK_NULL_HANDLE)
5319  {
5320  VMA_ASSERT(allocator);
5321 
5322  VMA_DEBUG_LOG("vmaDestroyBuffer");
5323 
5324  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5325 
5326  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
5327 
5328  allocator->FreeMemory(allocation);
5329  }
5330 }
5331 
5332 VkResult vmaCreateImage(
5333  VmaAllocator allocator,
5334  const VkImageCreateInfo* pCreateInfo,
5335  const VmaMemoryRequirements* pMemoryRequirements,
5336  VkImage* pImage,
5337  VmaAllocation* pAllocation,
5338  VmaAllocationInfo* pAllocationInfo)
5339 {
5340  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pImage && pAllocation);
5341 
5342  VMA_DEBUG_LOG("vmaCreateImage");
5343 
5344  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5345 
5346  // 1. Create VkImage.
5347  VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage);
5348  if(res >= 0)
5349  {
5350  VkMappedMemoryRange mem = {};
5351  VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
5352  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
5353  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
5354 
5355  // 2. Allocate memory using allocator.
5356  res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, pAllocation);
5357  if(res >= 0)
5358  {
5359  // 3. Bind image with memory.
5360  res = vkBindImageMemory(allocator->m_hDevice, *pImage, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
5361  if(res >= 0)
5362  {
5363  // All steps succeeded.
5364  if(pAllocationInfo != VMA_NULL)
5365  {
5366  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5367  }
5368  return VK_SUCCESS;
5369  }
5370  allocator->FreeMemory(*pAllocation);
5371  return res;
5372  }
5373  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
5374  return res;
5375  }
5376  return res;
5377 }
5378 
5379 void vmaDestroyImage(
5380  VmaAllocator allocator,
5381  VkImage image,
5382  VmaAllocation allocation)
5383 {
5384  if(image != VK_NULL_HANDLE)
5385  {
5386  VMA_ASSERT(allocator);
5387 
5388  VMA_DEBUG_LOG("vmaDestroyImage");
5389 
5390  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5391 
5392  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
5393 
5394  allocator->FreeMemory(allocation);
5395  }
5396 }
5397 
5398 #endif // #ifdef VMA_IMPLEMENTATION
VmaMemoryRequirementFlagBits
VmaMemoryRequirementFlagBits
Flags to be passed as VmaMemoryRequirements::flags.
Definition: vk_mem_alloc.h:336
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
161 #include <vulkan/vulkan.h>
162 
164 
168 VK_DEFINE_HANDLE(VmaAllocator)
169 
170 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
172  VmaAllocator allocator,
173  uint32_t memoryType,
174  VkDeviceMemory memory,
175  VkDeviceSize size);
177 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
178  VmaAllocator allocator,
179  uint32_t memoryType,
180  VkDeviceMemory memory,
181  VkDeviceSize size);
182 
188 typedef struct VmaDeviceMemoryCallbacks {
190  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
192  PFN_vmaFreeDeviceMemoryFunction pfnFree;
193 } VmaDeviceMemoryCallbacks;
194 
196 typedef enum VmaAllocatorFlagBits {
201  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
202 
203  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
204 } VmaAllocatorFlagBits;
205 typedef VkFlags VmaAllocatorFlags;
206 
208 typedef struct VmaAllocatorCreateInfo
209 {
211  VmaAllocatorFlags flags;
213 
214  VkPhysicalDevice physicalDevice;
216 
217  VkDevice device;
219 
220  VkDeviceSize preferredLargeHeapBlockSize;
222 
223  VkDeviceSize preferredSmallHeapBlockSize;
225 
226  const VkAllocationCallbacks* pAllocationCallbacks;
228 
229  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
230 } VmaAllocatorCreateInfo;
231 
233 VkResult vmaCreateAllocator(
234  const VmaAllocatorCreateInfo* pCreateInfo,
235  VmaAllocator* pAllocator);
236 
238 void vmaDestroyAllocator(
239  VmaAllocator allocator);
240 
245 void vmaGetPhysicalDeviceProperties(
246  VmaAllocator allocator,
247  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
248 
253 void vmaGetMemoryProperties(
254  VmaAllocator allocator,
255  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
256 
263 void vmaGetMemoryTypeProperties(
264  VmaAllocator allocator,
265  uint32_t memoryTypeIndex,
266  VkMemoryPropertyFlags* pFlags);
267 
268 typedef struct VmaStatInfo
269 {
270  uint32_t AllocationCount;
271  uint32_t SuballocationCount;
272  uint32_t UnusedRangeCount;
273  VkDeviceSize UsedBytes;
274  VkDeviceSize UnusedBytes;
275  VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax;
276  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
277 } VmaStatInfo;
278 
280 struct VmaStats
281 {
282  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
283  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
284  VmaStatInfo total;
285 };
286 
288 void vmaCalculateStats(
289  VmaAllocator allocator,
290  VmaStats* pStats);
291 
292 #define VMA_STATS_STRING_ENABLED 1
293 
294 #if VMA_STATS_STRING_ENABLED
295 
297 
299 void vmaBuildStatsString(
300  VmaAllocator allocator,
301  char** ppStatsString,
302  VkBool32 detailedMap);
303 
304 void vmaFreeStatsString(
305  VmaAllocator allocator,
306  char* pStatsString);
307 
308 #endif // #if VMA_STATS_STRING_ENABLED
309 
312 
317 typedef enum VmaMemoryUsage
318 {
320  VMA_MEMORY_USAGE_UNKNOWN = 0,
322  VMA_MEMORY_USAGE_GPU_ONLY = 1,
324 
325  VMA_MEMORY_USAGE_CPU_ONLY = 2,
327 
328  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
330 
331  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
332  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
333 } VmaMemoryUsage;
334 
336 typedef enum VmaMemoryRequirementFlagBits {
345  VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001,
346 
354  VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002,
361  VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004,
362 
363  VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
364 } VmaMemoryRequirementFlagBits;
365 typedef VkFlags VmaMemoryRequirementFlags;
366 
367 typedef struct VmaMemoryRequirements
368 {
369  VmaMemoryRequirementFlags flags;
374  VmaMemoryUsage usage;
378  VkMemoryPropertyFlags requiredFlags;
383  VkMemoryPropertyFlags preferredFlags;
385  void* pUserData;
386 } VmaMemoryRequirements;
387 
402 VkResult vmaFindMemoryTypeIndex(
403  VmaAllocator allocator,
404  uint32_t memoryTypeBits,
405  const VmaMemoryRequirements* pMemoryRequirements,
406  uint32_t* pMemoryTypeIndex);
407 
410 
415 VK_DEFINE_HANDLE(VmaAllocation)
416 
417 
419 typedef struct VmaAllocationInfo {
424  uint32_t memoryType;
431  VkDeviceMemory deviceMemory;
436  VkDeviceSize offset;
441  VkDeviceSize size;
447  void* pMappedData;
452  void* pUserData;
453 } VmaAllocationInfo;
454 
465 VkResult vmaAllocateMemory(
466  VmaAllocator allocator,
467  const VkMemoryRequirements* pVkMemoryRequirements,
468  const VmaMemoryRequirements* pVmaMemoryRequirements,
469  VmaAllocation* pAllocation,
470  VmaAllocationInfo* pAllocationInfo);
471 
478 VkResult vmaAllocateMemoryForBuffer(
479  VmaAllocator allocator,
480  VkBuffer buffer,
481  const VmaMemoryRequirements* pMemoryRequirements,
482  VmaAllocation* pAllocation,
483  VmaAllocationInfo* pAllocationInfo);
484 
486 VkResult vmaAllocateMemoryForImage(
487  VmaAllocator allocator,
488  VkImage image,
489  const VmaMemoryRequirements* pMemoryRequirements,
490  VmaAllocation* pAllocation,
491  VmaAllocationInfo* pAllocationInfo);
492 
494 void vmaFreeMemory(
495  VmaAllocator allocator,
496  VmaAllocation allocation);
497 
499 void vmaGetAllocationInfo(
500  VmaAllocator allocator,
501  VmaAllocation allocation,
502  VmaAllocationInfo* pAllocationInfo);
503 
505 void vmaSetAllocationUserData(
506  VmaAllocator allocator,
507  VmaAllocation allocation,
508  void* pUserData);
509 
518 VkResult vmaMapMemory(
519  VmaAllocator allocator,
520  VmaAllocation allocation,
521  void** ppData);
522 
523 void vmaUnmapMemory(
524  VmaAllocator allocator,
525  VmaAllocation allocation);
526 
545 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
546 
554 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
555 
557 typedef struct VmaDefragmentationInfo {
562  VkDeviceSize maxBytesToMove;
567  uint32_t maxAllocationsToMove;
568 } VmaDefragmentationInfo;
569 
571 typedef struct VmaDefragmentationStats {
573  VkDeviceSize bytesMoved;
575  VkDeviceSize bytesFreed;
577  uint32_t allocationsMoved;
579  uint32_t deviceMemoryBlocksFreed;
580 } VmaDefragmentationStats;
581 
652 VkResult vmaDefragment(
653  VmaAllocator allocator,
654  VmaAllocation* pAllocations,
655  size_t allocationCount,
656  VkBool32* pAllocationsChanged,
657  const VmaDefragmentationInfo *pDefragmentationInfo,
658  VmaDefragmentationStats* pDefragmentationStats);
659 
662 
682 VkResult vmaCreateBuffer(
683  VmaAllocator allocator,
684  const VkBufferCreateInfo* pCreateInfo,
685  const VmaMemoryRequirements* pMemoryRequirements,
686  VkBuffer* pBuffer,
687  VmaAllocation* pAllocation,
688  VmaAllocationInfo* pAllocationInfo);
689 
690 void vmaDestroyBuffer(
691  VmaAllocator allocator,
692  VkBuffer buffer,
693  VmaAllocation allocation);
694 
696 VkResult vmaCreateImage(
697  VmaAllocator allocator,
698  const VkImageCreateInfo* pCreateInfo,
699  const VmaMemoryRequirements* pMemoryRequirements,
700  VkImage* pImage,
701  VmaAllocation* pAllocation,
702  VmaAllocationInfo* pAllocationInfo);
703 
704 void vmaDestroyImage(
705  VmaAllocator allocator,
706  VkImage image,
707  VmaAllocation allocation);
708 
711 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
712 
713 #ifdef VMA_IMPLEMENTATION
714 #undef VMA_IMPLEMENTATION
715 
716 #include <cstdint>
717 #include <cstdlib>
718 
719 /*******************************************************************************
720 CONFIGURATION SECTION
721 
722 Define some of these macros before each #include of this header or change them
723 here if you need other then default behavior depending on your environment.
724 */
725 
726 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
727 //#define VMA_USE_STL_CONTAINERS 1
728 
729 /* Set this macro to 1 to make the library including and using STL containers:
730 std::pair, std::vector, std::list, std::unordered_map.
731 
732 Set it to 0 or undefined to make the library using its own implementation of
733 the containers.
734 */
735 #if VMA_USE_STL_CONTAINERS
736  #define VMA_USE_STL_VECTOR 1
737  #define VMA_USE_STL_UNORDERED_MAP 1
738  #define VMA_USE_STL_LIST 1
739 #endif
740 
741 #if VMA_USE_STL_VECTOR
742  #include <vector>
743 #endif
744 
745 #if VMA_USE_STL_UNORDERED_MAP
746  #include <unordered_map>
747 #endif
748 
749 #if VMA_USE_STL_LIST
750  #include <list>
751 #endif
752 
753 /*
754 Following headers are used in this CONFIGURATION section only, so feel free to
755 remove them if not needed.
756 */
757 #include <cassert> // for assert
758 #include <algorithm> // for min, max
759 #include <mutex> // for std::mutex
760 
761 #if !defined(_WIN32)
762  #include <malloc.h> // for aligned_alloc()
763 #endif
764 
765 // Normal assert to check for programmer's errors, especially in Debug configuration.
766 #ifndef VMA_ASSERT
767  #ifdef _DEBUG
768  #define VMA_ASSERT(expr) assert(expr)
769  #else
770  #define VMA_ASSERT(expr)
771  #endif
772 #endif
773 
774 // Assert that will be called very often, like inside data structures e.g. operator[].
775 // Making it non-empty can make program slow.
776 #ifndef VMA_HEAVY_ASSERT
777  #ifdef _DEBUG
778  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
779  #else
780  #define VMA_HEAVY_ASSERT(expr)
781  #endif
782 #endif
783 
784 #ifndef VMA_NULL
785  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
786  #define VMA_NULL nullptr
787 #endif
788 
789 #ifndef VMA_ALIGN_OF
790  #define VMA_ALIGN_OF(type) (__alignof(type))
791 #endif
792 
793 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
794  #if defined(_WIN32)
795  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
796  #else
797  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
798  #endif
799 #endif
800 
801 #ifndef VMA_SYSTEM_FREE
802  #if defined(_WIN32)
803  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
804  #else
805  #define VMA_SYSTEM_FREE(ptr) free(ptr)
806  #endif
807 #endif
808 
809 #ifndef VMA_MIN
810  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
811 #endif
812 
813 #ifndef VMA_MAX
814  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
815 #endif
816 
817 #ifndef VMA_SWAP
818  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
819 #endif
820 
821 #ifndef VMA_SORT
822  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
823 #endif
824 
825 #ifndef VMA_DEBUG_LOG
826  #define VMA_DEBUG_LOG(format, ...)
827  /*
828  #define VMA_DEBUG_LOG(format, ...) do { \
829  printf(format, __VA_ARGS__); \
830  printf("\n"); \
831  } while(false)
832  */
833 #endif
834 
835 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
836 #if VMA_STATS_STRING_ENABLED
837  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
838  {
839  _ultoa_s(num, outStr, strLen, 10);
840  }
841  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
842  {
843  _ui64toa_s(num, outStr, strLen, 10);
844  }
845 #endif
846 
847 #ifndef VMA_MUTEX
848  class VmaMutex
849  {
850  public:
851  VmaMutex() { }
852  ~VmaMutex() { }
853  void Lock() { m_Mutex.lock(); }
854  void Unlock() { m_Mutex.unlock(); }
855  private:
856  std::mutex m_Mutex;
857  };
858  #define VMA_MUTEX VmaMutex
859 #endif
860 
861 #ifndef VMA_BEST_FIT
862 
874  #define VMA_BEST_FIT (1)
875 #endif
876 
877 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
878 
882  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
883 #endif
884 
885 #ifndef VMA_DEBUG_ALIGNMENT
886 
890  #define VMA_DEBUG_ALIGNMENT (1)
891 #endif
892 
893 #ifndef VMA_DEBUG_MARGIN
894 
898  #define VMA_DEBUG_MARGIN (0)
899 #endif
900 
901 #ifndef VMA_DEBUG_GLOBAL_MUTEX
902 
906  #define VMA_DEBUG_GLOBAL_MUTEX (0)
907 #endif
908 
909 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
910 
914  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
915 #endif
916 
917 #ifndef VMA_SMALL_HEAP_MAX_SIZE
918  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
920 #endif
921 
922 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
923  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
925 #endif
926 
927 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
928  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
930 #endif
931 
932 /*******************************************************************************
933 END OF CONFIGURATION
934 */
935 
936 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
937  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
938 
939 // Returns number of bits set to 1 in (v).
940 static inline uint32_t CountBitsSet(uint32_t v)
941 {
942  uint32_t c = v - ((v >> 1) & 0x55555555);
943  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
944  c = ((c >> 4) + c) & 0x0F0F0F0F;
945  c = ((c >> 8) + c) & 0x00FF00FF;
946  c = ((c >> 16) + c) & 0x0000FFFF;
947  return c;
948 }
949 
950 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
951 // Use types like uint32_t, uint64_t as T.
952 template <typename T>
953 static inline T VmaAlignUp(T val, T align)
954 {
955  return (val + align - 1) / align * align;
956 }
957 
958 // Division with mathematical rounding to nearest number.
959 template <typename T>
960 inline T VmaRoundDiv(T x, T y)
961 {
962  return (x + (y / (T)2)) / y;
963 }
964 
965 #ifndef VMA_SORT
966 
967 template<typename Iterator, typename Compare>
968 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
969 {
970  Iterator centerValue = end; --centerValue;
971  Iterator insertIndex = beg;
972  for(Iterator i = beg; i < centerValue; ++i)
973  {
974  if(cmp(*i, *centerValue))
975  {
976  if(insertIndex != i)
977  {
978  VMA_SWAP(*i, *insertIndex);
979  }
980  ++insertIndex;
981  }
982  }
983  if(insertIndex != centerValue)
984  {
985  VMA_SWAP(*insertIndex, *centerValue);
986  }
987  return insertIndex;
988 }
989 
990 template<typename Iterator, typename Compare>
991 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
992 {
993  if(beg < end)
994  {
995  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
996  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
997  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
998  }
999 }
1000 
1001 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
1002 
1003 #endif // #ifndef VMA_SORT
1004 
1005 /*
1006 Returns true if two memory blocks occupy overlapping pages.
1007 ResourceA must be in less memory offset than ResourceB.
1008 
1009 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
1010 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
1011 */
1012 static inline bool VmaBlocksOnSamePage(
1013  VkDeviceSize resourceAOffset,
1014  VkDeviceSize resourceASize,
1015  VkDeviceSize resourceBOffset,
1016  VkDeviceSize pageSize)
1017 {
1018  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
1019  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
1020  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
1021  VkDeviceSize resourceBStart = resourceBOffset;
1022  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
1023  return resourceAEndPage == resourceBStartPage;
1024 }
1025 
1026 enum VmaSuballocationType
1027 {
1028  VMA_SUBALLOCATION_TYPE_FREE = 0,
1029  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
1030  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
1031  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
1032  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
1033  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
1034  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
1035 };
1036 
1037 /*
1038 Returns true if given suballocation types could conflict and must respect
1039 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
1040 or linear image and another one is optimal image. If type is unknown, behave
1041 conservatively.
1042 */
1043 static inline bool VmaIsBufferImageGranularityConflict(
1044  VmaSuballocationType suballocType1,
1045  VmaSuballocationType suballocType2)
1046 {
1047  if(suballocType1 > suballocType2)
1048  {
1049  VMA_SWAP(suballocType1, suballocType2);
1050  }
1051 
1052  switch(suballocType1)
1053  {
1054  case VMA_SUBALLOCATION_TYPE_FREE:
1055  return false;
1056  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
1057  return true;
1058  case VMA_SUBALLOCATION_TYPE_BUFFER:
1059  return
1060  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1061  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1062  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
1063  return
1064  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
1065  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
1066  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1067  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
1068  return
1069  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
1070  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
1071  return false;
1072  default:
1073  VMA_ASSERT(0);
1074  return true;
1075  }
1076 }
1077 
1078 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
1079 struct VmaMutexLock
1080 {
1081 public:
1082  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
1083  m_pMutex(useMutex ? &mutex : VMA_NULL)
1084  {
1085  if(m_pMutex)
1086  {
1087  m_pMutex->Lock();
1088  }
1089  }
1090 
1091  ~VmaMutexLock()
1092  {
1093  if(m_pMutex)
1094  {
1095  m_pMutex->Unlock();
1096  }
1097  }
1098 
1099 private:
1100  VMA_MUTEX* m_pMutex;
1101 };
1102 
1103 #if VMA_DEBUG_GLOBAL_MUTEX
1104  static VMA_MUTEX gDebugGlobalMutex;
1105  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex);
1106 #else
1107  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
1108 #endif
1109 
1110 // Minimum size of a free suballocation to register it in the free suballocation collection.
1111 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
1112 
1113 /*
1114 Performs binary search and returns iterator to first element that is greater or
1115 equal to (key), according to comparison (cmp).
1116 
1117 Cmp should return true if first argument is less than second argument.
1118 
1119 Returned value is the found element, if present in the collection or place where
1120 new element with value (key) should be inserted.
1121 */
1122 template <typename IterT, typename KeyT, typename CmpT>
1123 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
1124 {
1125  size_t down = 0, up = (end - beg);
1126  while(down < up)
1127  {
1128  const size_t mid = (down + up) / 2;
1129  if(cmp(*(beg+mid), key))
1130  {
1131  down = mid + 1;
1132  }
1133  else
1134  {
1135  up = mid;
1136  }
1137  }
1138  return beg + down;
1139 }
1140 
1142 // Memory allocation
1143 
1144 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
1145 {
1146  if((pAllocationCallbacks != VMA_NULL) &&
1147  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
1148  {
1149  return (*pAllocationCallbacks->pfnAllocation)(
1150  pAllocationCallbacks->pUserData,
1151  size,
1152  alignment,
1153  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1154  }
1155  else
1156  {
1157  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
1158  }
1159 }
1160 
1161 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
1162 {
1163  if((pAllocationCallbacks != VMA_NULL) &&
1164  (pAllocationCallbacks->pfnFree != VMA_NULL))
1165  {
1166  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
1167  }
1168  else
1169  {
1170  VMA_SYSTEM_FREE(ptr);
1171  }
1172 }
1173 
1174 template<typename T>
1175 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
1176 {
1177  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
1178 }
1179 
1180 template<typename T>
1181 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
1182 {
1183  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
1184 }
1185 
1186 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
1187 
1188 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
1189 
1190 template<typename T>
1191 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
1192 {
1193  ptr->~T();
1194  VmaFree(pAllocationCallbacks, ptr);
1195 }
1196 
1197 template<typename T>
1198 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
1199 {
1200  if(ptr != VMA_NULL)
1201  {
1202  for(size_t i = count; i--; )
1203  {
1204  ptr[i].~T();
1205  }
1206  VmaFree(pAllocationCallbacks, ptr);
1207  }
1208 }
1209 
1210 // STL-compatible allocator.
1211 template<typename T>
1212 class VmaStlAllocator
1213 {
1214 public:
1215  const VkAllocationCallbacks* const m_pCallbacks;
1216  typedef T value_type;
1217 
1218  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
1219  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
1220 
1221  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
1222  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
1223 
1224  template<typename U>
1225  bool operator==(const VmaStlAllocator<U>& rhs) const
1226  {
1227  return m_pCallbacks == rhs.m_pCallbacks;
1228  }
1229  template<typename U>
1230  bool operator!=(const VmaStlAllocator<U>& rhs) const
1231  {
1232  return m_pCallbacks != rhs.m_pCallbacks;
1233  }
1234 
1235  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
1236 };
1237 
1238 #if VMA_USE_STL_VECTOR
1239 
1240 #define VmaVector std::vector
1241 
1242 template<typename T, typename allocatorT>
1243 static void VectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
1244 {
1245  vec.insert(vec.begin() + index, item);
1246 }
1247 
1248 template<typename T, typename allocatorT>
1249 static void VectorRemove(std::vector<T, allocatorT>& vec, size_t index)
1250 {
1251  vec.erase(vec.begin() + index);
1252 }
1253 
1254 #else // #if VMA_USE_STL_VECTOR
1255 
1256 /* Class with interface compatible with subset of std::vector.
1257 T must be POD because constructors and destructors are not called and memcpy is
1258 used for these objects. */
1259 template<typename T, typename AllocatorT>
1260 class VmaVector
1261 {
1262 public:
1263  VmaVector(const AllocatorT& allocator) :
1264  m_Allocator(allocator),
1265  m_pArray(VMA_NULL),
1266  m_Count(0),
1267  m_Capacity(0)
1268  {
1269  }
1270 
1271  VmaVector(size_t count, const AllocatorT& allocator) :
1272  m_Allocator(allocator),
1273  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, count) : VMA_NULL),
1274  m_Count(count),
1275  m_Capacity(count)
1276  {
1277  }
1278 
1279  VmaVector(const VmaVector<T, AllocatorT>& src) :
1280  m_Allocator(src.m_Allocator),
1281  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src->m_pCallbacks, src.m_Count) : VMA_NULL),
1282  m_Count(src.m_Count),
1283  m_Capacity(src.m_Count)
1284  {
1285  if(m_Count != 0)
1286  {
1287  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
1288  }
1289  }
1290 
1291  ~VmaVector()
1292  {
1293  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1294  }
1295 
1296  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
1297  {
1298  if(&rhs != this)
1299  {
1300  Resize(rhs.m_Count);
1301  if(m_Count != 0)
1302  {
1303  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
1304  }
1305  }
1306  return *this;
1307  }
1308 
1309  bool empty() const { return m_Count == 0; }
1310  size_t size() const { return m_Count; }
1311  T* data() { return m_pArray; }
1312  const T* data() const { return m_pArray; }
1313 
1314  T& operator[](size_t index)
1315  {
1316  VMA_HEAVY_ASSERT(index < m_Count);
1317  return m_pArray[index];
1318  }
1319  const T& operator[](size_t index) const
1320  {
1321  VMA_HEAVY_ASSERT(index < m_Count);
1322  return m_pArray[index];
1323  }
1324 
1325  T& front()
1326  {
1327  VMA_HEAVY_ASSERT(m_Count > 0);
1328  return m_pArray[0];
1329  }
1330  const T& front() const
1331  {
1332  VMA_HEAVY_ASSERT(m_Count > 0);
1333  return m_pArray[0];
1334  }
1335  T& back()
1336  {
1337  VMA_HEAVY_ASSERT(m_Count > 0);
1338  return m_pArray[m_Count - 1];
1339  }
1340  const T& back() const
1341  {
1342  VMA_HEAVY_ASSERT(m_Count > 0);
1343  return m_pArray[m_Count - 1];
1344  }
1345 
1346  void reserve(size_t newCapacity, bool freeMemory = false)
1347  {
1348  newCapacity = VMA_MAX(newCapacity, m_Count);
1349 
1350  if((newCapacity < m_Capacity) && !freeMemory)
1351  {
1352  newCapacity = m_Capacity;
1353  }
1354 
1355  if(newCapacity != m_Capacity)
1356  {
1357  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1358  if(m_Count != 0)
1359  {
1360  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1361  }
1362  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1363  m_Capacity = newCapacity;
1364  m_pArray = newArray;
1365  }
1366  }
1367 
1368  void resize(size_t newCount, bool freeMemory = false)
1369  {
1370  size_t newCapacity = m_Capacity;
1371  if(newCount > m_Capacity)
1372  {
1373  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1374  }
1375  else if(freeMemory)
1376  {
1377  newCapacity = newCount;
1378  }
1379 
1380  if(newCapacity != m_Capacity)
1381  {
1382  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1383  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1384  if(elementsToCopy != 0)
1385  {
1386  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1387  }
1388  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1389  m_Capacity = newCapacity;
1390  m_pArray = newArray;
1391  }
1392 
1393  m_Count = newCount;
1394  }
1395 
1396  void clear(bool freeMemory = false)
1397  {
1398  resize(0, freeMemory);
1399  }
1400 
1401  void insert(size_t index, const T& src)
1402  {
1403  VMA_HEAVY_ASSERT(index <= m_Count);
1404  const size_t oldCount = size();
1405  resize(oldCount + 1);
1406  if(index < oldCount)
1407  {
1408  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1409  }
1410  m_pArray[index] = src;
1411  }
1412 
1413  void remove(size_t index)
1414  {
1415  VMA_HEAVY_ASSERT(index < m_Count);
1416  const size_t oldCount = size();
1417  if(index < oldCount - 1)
1418  {
1419  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1420  }
1421  resize(oldCount - 1);
1422  }
1423 
1424  void push_back(const T& src)
1425  {
1426  const size_t newIndex = size();
1427  resize(newIndex + 1);
1428  m_pArray[newIndex] = src;
1429  }
1430 
1431  void pop_back()
1432  {
1433  VMA_HEAVY_ASSERT(m_Count > 0);
1434  resize(size() - 1);
1435  }
1436 
1437  void push_front(const T& src)
1438  {
1439  insert(0, src);
1440  }
1441 
1442  void pop_front()
1443  {
1444  VMA_HEAVY_ASSERT(m_Count > 0);
1445  remove(0);
1446  }
1447 
1448  typedef T* iterator;
1449 
1450  iterator begin() { return m_pArray; }
1451  iterator end() { return m_pArray + m_Count; }
1452 
1453 private:
1454  AllocatorT m_Allocator;
1455  T* m_pArray;
1456  size_t m_Count;
1457  size_t m_Capacity;
1458 };
1459 
1460 template<typename T, typename allocatorT>
1461 static void VectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
1462 {
1463  vec.insert(index, item);
1464 }
1465 
1466 template<typename T, typename allocatorT>
1467 static void VectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
1468 {
1469  vec.remove(index);
1470 }
1471 
1472 #endif // #if VMA_USE_STL_VECTOR
1473 
1475 // class VmaPoolAllocator
1476 
1477 /*
1478 Allocator for objects of type T using a list of arrays (pools) to speed up
1479 allocation. Number of elements that can be allocated is not bounded because
1480 allocator can create multiple blocks.
1481 */
1482 template<typename T>
1483 class VmaPoolAllocator
1484 {
1485 public:
1486  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
1487  ~VmaPoolAllocator();
1488  void Clear();
1489  T* Alloc();
1490  void Free(T* ptr);
1491 
1492 private:
1493  union Item
1494  {
1495  uint32_t NextFreeIndex;
1496  T Value;
1497  };
1498 
1499  struct ItemBlock
1500  {
1501  Item* pItems;
1502  uint32_t FirstFreeIndex;
1503  };
1504 
1505  const VkAllocationCallbacks* m_pAllocationCallbacks;
1506  size_t m_ItemsPerBlock;
1507  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
1508 
1509  ItemBlock& CreateNewBlock();
1510 };
1511 
1512 template<typename T>
1513 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
1514  m_pAllocationCallbacks(pAllocationCallbacks),
1515  m_ItemsPerBlock(itemsPerBlock),
1516  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
1517 {
1518  VMA_ASSERT(itemsPerBlock > 0);
1519 }
1520 
1521 template<typename T>
1522 VmaPoolAllocator<T>::~VmaPoolAllocator()
1523 {
1524  Clear();
1525 }
1526 
1527 template<typename T>
1528 void VmaPoolAllocator<T>::Clear()
1529 {
1530  for(size_t i = m_ItemBlocks.size(); i--; )
1531  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
1532  m_ItemBlocks.clear();
1533 }
1534 
1535 template<typename T>
1536 T* VmaPoolAllocator<T>::Alloc()
1537 {
1538  for(size_t i = m_ItemBlocks.size(); i--; )
1539  {
1540  ItemBlock& block = m_ItemBlocks[i];
1541  // This block has some free items: Use first one.
1542  if(block.FirstFreeIndex != UINT32_MAX)
1543  {
1544  Item* const pItem = &block.pItems[block.FirstFreeIndex];
1545  block.FirstFreeIndex = pItem->NextFreeIndex;
1546  return &pItem->Value;
1547  }
1548  }
1549 
1550  // No block has free item: Create new one and use it.
1551  ItemBlock& newBlock = CreateNewBlock();
1552  Item* const pItem = &newBlock.pItems[0];
1553  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
1554  return &pItem->Value;
1555 }
1556 
1557 template<typename T>
1558 void VmaPoolAllocator<T>::Free(T* ptr)
1559 {
1560  // Search all memory blocks to find ptr.
1561  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
1562  {
1563  ItemBlock& block = m_ItemBlocks[i];
1564 
1565  // Casting to union.
1566  Item* pItemPtr;
1567  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
1568 
1569  // Check if pItemPtr is in address range of this block.
1570  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
1571  {
1572  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
1573  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
1574  block.FirstFreeIndex = index;
1575  return;
1576  }
1577  }
1578  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
1579 }
1580 
1581 template<typename T>
1582 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
1583 {
1584  ItemBlock newBlock = {
1585  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
1586 
1587  m_ItemBlocks.push_back(newBlock);
1588 
1589  // Setup singly-linked list of all free items in this block.
1590  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
1591  newBlock.pItems[i].NextFreeIndex = i + 1;
1592  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
1593  return m_ItemBlocks.back();
1594 }
1595 
1597 // class VmaRawList, VmaList
1598 
1599 #if VMA_USE_STL_LIST
1600 
1601 #define VmaList std::list
1602 
1603 #else // #if VMA_USE_STL_LIST
1604 
1605 template<typename T>
1606 struct VmaListItem
1607 {
1608  VmaListItem* pPrev;
1609  VmaListItem* pNext;
1610  T Value;
1611 };
1612 
1613 // Doubly linked list.
1614 template<typename T>
1615 class VmaRawList
1616 {
1617 public:
1618  typedef VmaListItem<T> ItemType;
1619 
1620  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
1621  ~VmaRawList();
1622  void Clear();
1623 
1624  size_t GetCount() const { return m_Count; }
1625  bool IsEmpty() const { return m_Count == 0; }
1626 
1627  ItemType* Front() { return m_pFront; }
1628  const ItemType* Front() const { return m_pFront; }
1629  ItemType* Back() { return m_pBack; }
1630  const ItemType* Back() const { return m_pBack; }
1631 
1632  ItemType* PushBack();
1633  ItemType* PushFront();
1634  ItemType* PushBack(const T& value);
1635  ItemType* PushFront(const T& value);
1636  void PopBack();
1637  void PopFront();
1638 
1639  // Item can be null - it means PushBack.
1640  ItemType* InsertBefore(ItemType* pItem);
1641  // Item can be null - it means PushFront.
1642  ItemType* InsertAfter(ItemType* pItem);
1643 
1644  ItemType* InsertBefore(ItemType* pItem, const T& value);
1645  ItemType* InsertAfter(ItemType* pItem, const T& value);
1646 
1647  void Remove(ItemType* pItem);
1648 
1649 private:
1650  const VkAllocationCallbacks* const m_pAllocationCallbacks;
1651  VmaPoolAllocator<ItemType> m_ItemAllocator;
1652  ItemType* m_pFront;
1653  ItemType* m_pBack;
1654  size_t m_Count;
1655 
1656  // Declared not defined, to block copy constructor and assignment operator.
1657  VmaRawList(const VmaRawList<T>& src);
1658  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
1659 };
1660 
1661 template<typename T>
1662 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
1663  m_pAllocationCallbacks(pAllocationCallbacks),
1664  m_ItemAllocator(pAllocationCallbacks, 128),
1665  m_pFront(VMA_NULL),
1666  m_pBack(VMA_NULL),
1667  m_Count(0)
1668 {
1669 }
1670 
1671 template<typename T>
1672 VmaRawList<T>::~VmaRawList()
1673 {
1674  // Intentionally not calling Clear, because that would be unnecessary
1675  // computations to return all items to m_ItemAllocator as free.
1676 }
1677 
1678 template<typename T>
1679 void VmaRawList<T>::Clear()
1680 {
1681  if(IsEmpty() == false)
1682  {
1683  ItemType* pItem = m_pBack;
1684  while(pItem != VMA_NULL)
1685  {
1686  ItemType* const pPrevItem = pItem->pPrev;
1687  m_ItemAllocator.Free(pItem);
1688  pItem = pPrevItem;
1689  }
1690  m_pFront = VMA_NULL;
1691  m_pBack = VMA_NULL;
1692  m_Count = 0;
1693  }
1694 }
1695 
1696 template<typename T>
1697 VmaListItem<T>* VmaRawList<T>::PushBack()
1698 {
1699  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1700  pNewItem->pNext = VMA_NULL;
1701  if(IsEmpty())
1702  {
1703  pNewItem->pPrev = VMA_NULL;
1704  m_pFront = pNewItem;
1705  m_pBack = pNewItem;
1706  m_Count = 1;
1707  }
1708  else
1709  {
1710  pNewItem->pPrev = m_pBack;
1711  m_pBack->pNext = pNewItem;
1712  m_pBack = pNewItem;
1713  ++m_Count;
1714  }
1715  return pNewItem;
1716 }
1717 
1718 template<typename T>
1719 VmaListItem<T>* VmaRawList<T>::PushFront()
1720 {
1721  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1722  pNewItem->pPrev = VMA_NULL;
1723  if(IsEmpty())
1724  {
1725  pNewItem->pNext = VMA_NULL;
1726  m_pFront = pNewItem;
1727  m_pBack = pNewItem;
1728  m_Count = 1;
1729  }
1730  else
1731  {
1732  pNewItem->pNext = m_pFront;
1733  m_pFront->pPrev = pNewItem;
1734  m_pFront = pNewItem;
1735  ++m_Count;
1736  }
1737  return pNewItem;
1738 }
1739 
1740 template<typename T>
1741 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
1742 {
1743  ItemType* const pNewItem = PushBack();
1744  pNewItem->Value = value;
1745  return pNewItem;
1746 }
1747 
1748 template<typename T>
1749 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
1750 {
1751  ItemType* const pNewItem = PushFront();
1752  pNewItem->Value = value;
1753  return pNewItem;
1754 }
1755 
1756 template<typename T>
1757 void VmaRawList<T>::PopBack()
1758 {
1759  VMA_HEAVY_ASSERT(m_Count > 0);
1760  ItemType* const pBackItem = m_pBack;
1761  ItemType* const pPrevItem = pBackItem->pPrev;
1762  if(pPrevItem != VMA_NULL)
1763  {
1764  pPrevItem->pNext = VMA_NULL;
1765  }
1766  m_pBack = pPrevItem;
1767  m_ItemAllocator.Free(pBackItem);
1768  --m_Count;
1769 }
1770 
1771 template<typename T>
1772 void VmaRawList<T>::PopFront()
1773 {
1774  VMA_HEAVY_ASSERT(m_Count > 0);
1775  ItemType* const pFrontItem = m_pFront;
1776  ItemType* const pNextItem = pFrontItem->pNext;
1777  if(pNextItem != VMA_NULL)
1778  {
1779  pNextItem->pPrev = VMA_NULL;
1780  }
1781  m_pFront = pNextItem;
1782  m_ItemAllocator.Free(pFrontItem);
1783  --m_Count;
1784 }
1785 
1786 template<typename T>
1787 void VmaRawList<T>::Remove(ItemType* pItem)
1788 {
1789  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
1790  VMA_HEAVY_ASSERT(m_Count > 0);
1791 
1792  if(pItem->pPrev != VMA_NULL)
1793  {
1794  pItem->pPrev->pNext = pItem->pNext;
1795  }
1796  else
1797  {
1798  VMA_HEAVY_ASSERT(m_pFront == pItem);
1799  m_pFront = pItem->pNext;
1800  }
1801 
1802  if(pItem->pNext != VMA_NULL)
1803  {
1804  pItem->pNext->pPrev = pItem->pPrev;
1805  }
1806  else
1807  {
1808  VMA_HEAVY_ASSERT(m_pBack == pItem);
1809  m_pBack = pItem->pPrev;
1810  }
1811 
1812  m_ItemAllocator.Free(pItem);
1813  --m_Count;
1814 }
1815 
1816 template<typename T>
1817 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
1818 {
1819  if(pItem != VMA_NULL)
1820  {
1821  ItemType* const prevItem = pItem->pPrev;
1822  ItemType* const newItem = m_ItemAllocator.Alloc();
1823  newItem->pPrev = prevItem;
1824  newItem->pNext = pItem;
1825  pItem->pPrev = newItem;
1826  if(prevItem != VMA_NULL)
1827  {
1828  prevItem->pNext = newItem;
1829  }
1830  else
1831  {
1832  VMA_HEAVY_ASSERT(m_pFront == pItem);
1833  m_pFront = newItem;
1834  }
1835  ++m_Count;
1836  return newItem;
1837  }
1838  else
1839  return PushBack();
1840 }
1841 
1842 template<typename T>
1843 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
1844 {
1845  if(pItem != VMA_NULL)
1846  {
1847  ItemType* const nextItem = pItem->pNext;
1848  ItemType* const newItem = m_ItemAllocator.Alloc();
1849  newItem->pNext = nextItem;
1850  newItem->pPrev = pItem;
1851  pItem->pNext = newItem;
1852  if(nextItem != VMA_NULL)
1853  {
1854  nextItem->pPrev = newItem;
1855  }
1856  else
1857  {
1858  VMA_HEAVY_ASSERT(m_pBack == pItem);
1859  m_pBack = newItem;
1860  }
1861  ++m_Count;
1862  return newItem;
1863  }
1864  else
1865  return PushFront();
1866 }
1867 
1868 template<typename T>
1869 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
1870 {
1871  ItemType* const newItem = InsertBefore(pItem);
1872  newItem->Value = value;
1873  return newItem;
1874 }
1875 
1876 template<typename T>
1877 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
1878 {
1879  ItemType* const newItem = InsertAfter(pItem);
1880  newItem->Value = value;
1881  return newItem;
1882 }
1883 
1884 template<typename T, typename AllocatorT>
1885 class VmaList
1886 {
1887 public:
1888  class iterator
1889  {
1890  public:
1891  iterator() :
1892  m_pList(VMA_NULL),
1893  m_pItem(VMA_NULL)
1894  {
1895  }
1896 
1897  T& operator*() const
1898  {
1899  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1900  return m_pItem->Value;
1901  }
1902  T* operator->() const
1903  {
1904  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1905  return &m_pItem->Value;
1906  }
1907 
1908  iterator& operator++()
1909  {
1910  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1911  m_pItem = m_pItem->pNext;
1912  return *this;
1913  }
1914  iterator& operator--()
1915  {
1916  if(m_pItem != VMA_NULL)
1917  {
1918  m_pItem = m_pItem->pPrev;
1919  }
1920  else
1921  {
1922  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
1923  m_pItem = m_pList->Back();
1924  }
1925  return *this;
1926  }
1927 
1928  iterator operator++(int)
1929  {
1930  iterator result = *this;
1931  ++*this;
1932  return result;
1933  }
1934  iterator operator--(int)
1935  {
1936  iterator result = *this;
1937  --*this;
1938  return result;
1939  }
1940 
1941  bool operator==(const iterator& rhs) const
1942  {
1943  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1944  return m_pItem == rhs.m_pItem;
1945  }
1946  bool operator!=(const iterator& rhs) const
1947  {
1948  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1949  return m_pItem != rhs.m_pItem;
1950  }
1951 
1952  private:
1953  VmaRawList<T>* m_pList;
1954  VmaListItem<T>* m_pItem;
1955 
1956  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
1957  m_pList(pList),
1958  m_pItem(pItem)
1959  {
1960  }
1961 
1962  friend class VmaList<T, AllocatorT>;
1963  friend class VmaList<T, AllocatorT>:: const_iterator;
1964  };
1965 
1966  class const_iterator
1967  {
1968  public:
1969  const_iterator() :
1970  m_pList(VMA_NULL),
1971  m_pItem(VMA_NULL)
1972  {
1973  }
1974 
1975  const_iterator(const iterator& src) :
1976  m_pList(src.m_pList),
1977  m_pItem(src.m_pItem)
1978  {
1979  }
1980 
1981  const T& operator*() const
1982  {
1983  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1984  return m_pItem->Value;
1985  }
1986  const T* operator->() const
1987  {
1988  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1989  return &m_pItem->Value;
1990  }
1991 
1992  const_iterator& operator++()
1993  {
1994  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1995  m_pItem = m_pItem->pNext;
1996  return *this;
1997  }
1998  const_iterator& operator--()
1999  {
2000  if(m_pItem != VMA_NULL)
2001  {
2002  m_pItem = m_pItem->pPrev;
2003  }
2004  else
2005  {
2006  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
2007  m_pItem = m_pList->Back();
2008  }
2009  return *this;
2010  }
2011 
2012  const_iterator operator++(int)
2013  {
2014  const_iterator result = *this;
2015  ++*this;
2016  return result;
2017  }
2018  const_iterator operator--(int)
2019  {
2020  const_iterator result = *this;
2021  --*this;
2022  return result;
2023  }
2024 
2025  bool operator==(const const_iterator& rhs) const
2026  {
2027  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2028  return m_pItem == rhs.m_pItem;
2029  }
2030  bool operator!=(const const_iterator& rhs) const
2031  {
2032  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
2033  return m_pItem != rhs.m_pItem;
2034  }
2035 
2036  private:
2037  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
2038  m_pList(pList),
2039  m_pItem(pItem)
2040  {
2041  }
2042 
2043  const VmaRawList<T>* m_pList;
2044  const VmaListItem<T>* m_pItem;
2045 
2046  friend class VmaList<T, AllocatorT>;
2047  };
2048 
2049  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
2050 
2051  bool empty() const { return m_RawList.IsEmpty(); }
2052  size_t size() const { return m_RawList.GetCount(); }
2053 
2054  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
2055  iterator end() { return iterator(&m_RawList, VMA_NULL); }
2056 
2057  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
2058  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
2059 
2060  void clear() { m_RawList.Clear(); }
2061  void push_back(const T& value) { m_RawList.PushBack(value); }
2062  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
2063  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
2064 
2065 private:
2066  VmaRawList<T> m_RawList;
2067 };
2068 
2069 #endif // #if VMA_USE_STL_LIST
2070 
2072 // class VmaMap
2073 
2074 #if VMA_USE_STL_UNORDERED_MAP
2075 
2076 #define VmaPair std::pair
2077 
2078 #define VMA_MAP_TYPE(KeyT, ValueT) \
2079  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
2080 
2081 #else // #if VMA_USE_STL_UNORDERED_MAP
2082 
2083 template<typename T1, typename T2>
2084 struct VmaPair
2085 {
2086  T1 first;
2087  T2 second;
2088 
2089  VmaPair() : first(), second() { }
2090  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
2091 };
2092 
2093 /* Class compatible with subset of interface of std::unordered_map.
2094 KeyT, ValueT must be POD because they will be stored in VmaVector.
2095 */
2096 template<typename KeyT, typename ValueT>
2097 class VmaMap
2098 {
2099 public:
2100  typedef VmaPair<KeyT, ValueT> PairType;
2101  typedef PairType* iterator;
2102 
2103  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
2104 
2105  iterator begin() { return m_Vector.begin(); }
2106  iterator end() { return m_Vector.end(); }
2107 
2108  void insert(const PairType& pair);
2109  iterator find(const KeyT& key);
2110  void erase(iterator it);
2111 
2112 private:
2113  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
2114 };
2115 
2116 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
2117 
2118 template<typename FirstT, typename SecondT>
2119 struct VmaPairFirstLess
2120 {
2121  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
2122  {
2123  return lhs.first < rhs.first;
2124  }
2125  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
2126  {
2127  return lhs.first < rhsFirst;
2128  }
2129 };
2130 
2131 template<typename KeyT, typename ValueT>
2132 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
2133 {
2134  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2135  m_Vector.data(),
2136  m_Vector.data() + m_Vector.size(),
2137  pair,
2138  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
2139  VectorInsert(m_Vector, indexToInsert, pair);
2140 }
2141 
2142 template<typename KeyT, typename ValueT>
2143 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
2144 {
2145  PairType* it = VmaBinaryFindFirstNotLess(
2146  m_Vector.data(),
2147  m_Vector.data() + m_Vector.size(),
2148  key,
2149  VmaPairFirstLess<KeyT, ValueT>());
2150  if((it != m_Vector.end()) && (it->first == key))
2151  {
2152  return it;
2153  }
2154  else
2155  {
2156  return m_Vector.end();
2157  }
2158 }
2159 
2160 template<typename KeyT, typename ValueT>
2161 void VmaMap<KeyT, ValueT>::erase(iterator it)
2162 {
2163  VectorRemove(m_Vector, it - m_Vector.begin());
2164 }
2165 
2166 #endif // #if VMA_USE_STL_UNORDERED_MAP
2167 
2169 
2170 class VmaBlock;
2171 
2172 enum VMA_BLOCK_VECTOR_TYPE
2173 {
2174  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
2175  VMA_BLOCK_VECTOR_TYPE_MAPPED,
2176  VMA_BLOCK_VECTOR_TYPE_COUNT
2177 };
2178 
2179 static VMA_BLOCK_VECTOR_TYPE VmaMemoryRequirementFlagsToBlockVectorType(VmaMemoryRequirementFlags flags)
2180 {
2181  return (flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0 ?
2182  VMA_BLOCK_VECTOR_TYPE_MAPPED :
2183  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
2184 }
2185 
2186 struct VmaAllocation_T
2187 {
2188 public:
2189  enum ALLOCATION_TYPE
2190  {
2191  ALLOCATION_TYPE_NONE,
2192  ALLOCATION_TYPE_BLOCK,
2193  ALLOCATION_TYPE_OWN,
2194  };
2195 
2196  VmaAllocation_T()
2197  {
2198  memset(this, 0, sizeof(VmaAllocation_T));
2199  }
2200 
2201  void InitBlockAllocation(
2202  VmaBlock* block,
2203  VkDeviceSize offset,
2204  VkDeviceSize alignment,
2205  VkDeviceSize size,
2206  VmaSuballocationType suballocationType,
2207  void* pUserData)
2208  {
2209  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2210  VMA_ASSERT(block != VMA_NULL);
2211  m_Type = ALLOCATION_TYPE_BLOCK;
2212  m_Alignment = alignment;
2213  m_Size = size;
2214  m_pUserData = pUserData;
2215  m_SuballocationType = suballocationType;
2216  m_BlockAllocation.m_Block = block;
2217  m_BlockAllocation.m_Offset = offset;
2218  }
2219 
2220  void ChangeBlockAllocation(
2221  VmaBlock* block,
2222  VkDeviceSize offset)
2223  {
2224  VMA_ASSERT(block != VMA_NULL);
2225  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2226  m_BlockAllocation.m_Block = block;
2227  m_BlockAllocation.m_Offset = offset;
2228  }
2229 
2230  void InitOwnAllocation(
2231  uint32_t memoryTypeIndex,
2232  VkDeviceMemory hMemory,
2233  VmaSuballocationType suballocationType,
2234  bool persistentMap,
2235  void* pMappedData,
2236  VkDeviceSize size,
2237  void* pUserData)
2238  {
2239  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
2240  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
2241  m_Type = ALLOCATION_TYPE_OWN;
2242  m_Alignment = 0;
2243  m_Size = size;
2244  m_pUserData = pUserData;
2245  m_SuballocationType = suballocationType;
2246  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
2247  m_OwnAllocation.m_hMemory = hMemory;
2248  m_OwnAllocation.m_PersistentMap = persistentMap;
2249  m_OwnAllocation.m_pMappedData = pMappedData;
2250  }
2251 
2252  ALLOCATION_TYPE GetType() const { return m_Type; }
2253  VkDeviceSize GetAlignment() const { return m_Alignment; }
2254  VkDeviceSize GetSize() const { return m_Size; }
2255  void* GetUserData() const { return m_pUserData; }
2256  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
2257  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
2258 
2259  VmaBlock* GetBlock() const
2260  {
2261  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
2262  return m_BlockAllocation.m_Block;
2263  }
2264  VkDeviceSize GetOffset() const
2265  {
2266  return (m_Type == ALLOCATION_TYPE_BLOCK) ? m_BlockAllocation.m_Offset : 0;
2267  }
2268  VkDeviceMemory GetMemory() const;
2269  uint32_t GetMemoryTypeIndex() const;
2270  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
2271  void* GetMappedData() const;
2272 
2273  VkResult OwnAllocMapPersistentlyMappedMemory(VkDevice hDevice)
2274  {
2275  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2276  if(m_OwnAllocation.m_PersistentMap)
2277  {
2278  return vkMapMemory(hDevice, m_OwnAllocation.m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_OwnAllocation.m_pMappedData);
2279  }
2280  return VK_SUCCESS;
2281  }
2282  void OwnAllocUnmapPersistentlyMappedMemory(VkDevice hDevice)
2283  {
2284  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
2285  if(m_OwnAllocation.m_pMappedData)
2286  {
2287  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
2288  vkUnmapMemory(hDevice, m_OwnAllocation.m_hMemory);
2289  m_OwnAllocation.m_pMappedData = VMA_NULL;
2290  }
2291  }
2292 
2293 private:
2294  VkDeviceSize m_Alignment;
2295  VkDeviceSize m_Size;
2296  void* m_pUserData;
2297  ALLOCATION_TYPE m_Type;
2298  VmaSuballocationType m_SuballocationType;
2299 
2300  union
2301  {
2302  // Allocation out of VmaBlock.
2303  struct BlockAllocation
2304  {
2305  VmaBlock* m_Block;
2306  VkDeviceSize m_Offset;
2307  } m_BlockAllocation;
2308 
2309  // Allocation for an object that has its own private VkDeviceMemory.
2310  struct OwnAllocation
2311  {
2312  uint32_t m_MemoryTypeIndex;
2313  VkDeviceMemory m_hMemory;
2314  bool m_PersistentMap;
2315  void* m_pMappedData;
2316  } m_OwnAllocation;
2317  };
2318 };
2319 
2320 /*
2321 Represents a region of VmaBlock that is either assigned and returned as
2322 allocated memory block or free.
2323 */
2324 struct VmaSuballocation
2325 {
2326  VkDeviceSize offset;
2327  VkDeviceSize size;
2328  VmaSuballocationType type;
2329 };
2330 
2331 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
2332 
2333 // Parameters of an allocation.
2334 struct VmaAllocationRequest
2335 {
2336  VmaSuballocationList::iterator freeSuballocationItem;
2337  VkDeviceSize offset;
2338 };
2339 
2340 /* Single block of memory - VkDeviceMemory with all the data about its regions
2341 assigned or free. */
2342 class VmaBlock
2343 {
2344 public:
2345  uint32_t m_MemoryTypeIndex;
2346  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
2347  VkDeviceMemory m_hMemory;
2348  VkDeviceSize m_Size;
2349  bool m_PersistentMap;
2350  void* m_pMappedData;
2351  uint32_t m_FreeCount;
2352  VkDeviceSize m_SumFreeSize;
2353  VmaSuballocationList m_Suballocations;
2354  // Suballocations that are free and have size greater than certain threshold.
2355  // Sorted by size, ascending.
2356  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
2357 
2358  VmaBlock(VmaAllocator hAllocator);
2359 
2360  ~VmaBlock()
2361  {
2362  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2363  }
2364 
2365  // Always call after construction.
2366  void Init(
2367  uint32_t newMemoryTypeIndex,
2368  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
2369  VkDeviceMemory newMemory,
2370  VkDeviceSize newSize,
2371  bool persistentMap,
2372  void* pMappedData);
2373  // Always call before destruction.
2374  void Destroy(VmaAllocator allocator);
2375 
2376  // Validates all data structures inside this object. If not valid, returns false.
2377  bool Validate() const;
2378 
2379  // Tries to find a place for suballocation with given parameters inside this allocation.
2380  // If succeeded, fills pAllocationRequest and returns true.
2381  // If failed, returns false.
2382  bool CreateAllocationRequest(
2383  VkDeviceSize bufferImageGranularity,
2384  VkDeviceSize allocSize,
2385  VkDeviceSize allocAlignment,
2386  VmaSuballocationType allocType,
2387  VmaAllocationRequest* pAllocationRequest);
2388 
2389  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
2390  // If yes, fills pOffset and returns true. If no, returns false.
2391  bool CheckAllocation(
2392  VkDeviceSize bufferImageGranularity,
2393  VkDeviceSize allocSize,
2394  VkDeviceSize allocAlignment,
2395  VmaSuballocationType allocType,
2396  VmaSuballocationList::const_iterator freeSuballocItem,
2397  VkDeviceSize* pOffset) const;
2398 
2399  // Returns true if this allocation is empty - contains only single free suballocation.
2400  bool IsEmpty() const;
2401 
2402  // Makes actual allocation based on request. Request must already be checked
2403  // and valid.
2404  void Alloc(
2405  const VmaAllocationRequest& request,
2406  VmaSuballocationType type,
2407  VkDeviceSize allocSize);
2408 
2409  // Frees suballocation assigned to given memory region.
2410  void Free(const VmaAllocation allocation);
2411 
2412 #if VMA_STATS_STRING_ENABLED
2413  void PrintDetailedMap(class VmaStringBuilder& sb) const;
2414 #endif
2415 
2416 private:
2417  // Given free suballocation, it merges it with following one, which must also be free.
2418  void MergeFreeWithNext(VmaSuballocationList::iterator item);
2419  // Releases given suballocation, making it free. Merges it with adjacent free
2420  // suballocations if applicable.
2421  void FreeSuballocation(VmaSuballocationList::iterator suballocItem);
2422  // Given free suballocation, it inserts it into sorted list of
2423  // m_FreeSuballocationsBySize if it's suitable.
2424  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
2425  // Given free suballocation, it removes it from sorted list of
2426  // m_FreeSuballocationsBySize if it's suitable.
2427  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
2428 };
2429 
2430 struct VmaPointerLess
2431 {
2432  bool operator()(const void* lhs, const void* rhs) const
2433  {
2434  return lhs < rhs;
2435  }
2436 };
2437 
2438 /* Sequence of VmaBlock. Represents memory blocks allocated for a specific
2439 Vulkan memory type. */
2440 struct VmaBlockVector
2441 {
2442  // Incrementally sorted by sumFreeSize, ascending.
2443  VmaVector< VmaBlock*, VmaStlAllocator<VmaBlock*> > m_Blocks;
2444 
2445  VmaBlockVector(VmaAllocator hAllocator);
2446  ~VmaBlockVector();
2447 
2448  bool IsEmpty() const { return m_Blocks.empty(); }
2449 
2450  // Finds and removes given block from vector.
2451  void Remove(VmaBlock* pBlock);
2452 
2453  // Performs single step in sorting m_Blocks. They may not be fully sorted
2454  // after this call.
2455  void IncrementallySortBlocks();
2456 
2457  // Adds statistics of this BlockVector to pStats.
2458  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const;
2459 
2460 #if VMA_STATS_STRING_ENABLED
2461  void PrintDetailedMap(class VmaStringBuilder& sb) const;
2462 #endif
2463 
2464  void UnmapPersistentlyMappedMemory();
2465  VkResult MapPersistentlyMappedMemory();
2466 
2467 private:
2468  VmaAllocator m_hAllocator;
2469 };
2470 
2471 // Main allocator object.
2472 struct VmaAllocator_T
2473 {
2474  bool m_UseMutex;
2475  VkDevice m_hDevice;
2476  bool m_AllocationCallbacksSpecified;
2477  VkAllocationCallbacks m_AllocationCallbacks;
2478  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
2479  VkDeviceSize m_PreferredLargeHeapBlockSize;
2480  VkDeviceSize m_PreferredSmallHeapBlockSize;
2481  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
2482  // Counter to allow nested calls to these functions.
2483  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
2484 
2485  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
2486  VkPhysicalDeviceMemoryProperties m_MemProps;
2487 
2488  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
2489  /* There can be at most one allocation that is completely empty - a
2490  hysteresis to avoid pessimistic case of alternating creation and destruction
2491  of a VkDeviceMemory. */
2492  bool m_HasEmptyBlock[VK_MAX_MEMORY_TYPES];
2493  VMA_MUTEX m_BlocksMutex[VK_MAX_MEMORY_TYPES];
2494 
2495  // Each vector is sorted by memory (handle value).
2496  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
2497  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
2498  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
2499 
2500  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
2501  ~VmaAllocator_T();
2502 
2503  const VkAllocationCallbacks* GetAllocationCallbacks() const
2504  {
2505  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
2506  }
2507 
2508  VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const;
2509 
2510  VkDeviceSize GetBufferImageGranularity() const
2511  {
2512  return VMA_MAX(
2513  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
2514  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
2515  }
2516 
2517  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
2518  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
2519 
2520  // Main allocation function.
2521  VkResult AllocateMemory(
2522  const VkMemoryRequirements& vkMemReq,
2523  const VmaMemoryRequirements& vmaMemReq,
2524  VmaSuballocationType suballocType,
2525  VmaAllocation* pAllocation);
2526 
2527  // Main deallocation function.
2528  void FreeMemory(const VmaAllocation allocation);
2529 
2530  void CalculateStats(VmaStats* pStats);
2531 
2532 #if VMA_STATS_STRING_ENABLED
2533  void PrintDetailedMap(class VmaStringBuilder& sb);
2534 #endif
2535 
2536  void UnmapPersistentlyMappedMemory();
2537  VkResult MapPersistentlyMappedMemory();
2538 
2539  VkResult Defragment(
2540  VmaAllocation* pAllocations,
2541  size_t allocationCount,
2542  VkBool32* pAllocationsChanged,
2543  const VmaDefragmentationInfo* pDefragmentationInfo,
2544  VmaDefragmentationStats* pDefragmentationStats);
2545 
2546  static void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
2547 
2548 private:
2549  VkPhysicalDevice m_PhysicalDevice;
2550 
2551  VkResult AllocateMemoryOfType(
2552  const VkMemoryRequirements& vkMemReq,
2553  const VmaMemoryRequirements& vmaMemReq,
2554  uint32_t memTypeIndex,
2555  VmaSuballocationType suballocType,
2556  VmaAllocation* pAllocation);
2557 
2558  // Allocates and registers new VkDeviceMemory specifically for single allocation.
2559  VkResult AllocateOwnMemory(
2560  VkDeviceSize size,
2561  VmaSuballocationType suballocType,
2562  uint32_t memTypeIndex,
2563  bool map,
2564  void* pUserData,
2565  VmaAllocation* pAllocation);
2566 
2567  // Tries to free pMemory as Own Memory. Returns true if found and freed.
2568  void FreeOwnMemory(VmaAllocation allocation);
2569 };
2570 
2572 // Memory allocation #2 after VmaAllocator_T definition
2573 
2574 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
2575 {
2576  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
2577 }
2578 
2579 static void VmaFree(VmaAllocator hAllocator, void* ptr)
2580 {
2581  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
2582 }
2583 
2584 template<typename T>
2585 static T* VmaAllocate(VmaAllocator hAllocator)
2586 {
2587  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
2588 }
2589 
2590 template<typename T>
2591 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
2592 {
2593  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
2594 }
2595 
2596 template<typename T>
2597 static void vma_delete(VmaAllocator hAllocator, T* ptr)
2598 {
2599  if(ptr != VMA_NULL)
2600  {
2601  ptr->~T();
2602  VmaFree(hAllocator, ptr);
2603  }
2604 }
2605 
2606 template<typename T>
2607 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
2608 {
2609  if(ptr != VMA_NULL)
2610  {
2611  for(size_t i = count; i--; )
2612  ptr[i].~T();
2613  VmaFree(hAllocator, ptr);
2614  }
2615 }
2616 
2618 // VmaStringBuilder
2619 
2620 #if VMA_STATS_STRING_ENABLED
2621 
2622 class VmaStringBuilder
2623 {
2624 public:
2625  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
2626  size_t GetLength() const { return m_Data.size(); }
2627  const char* GetData() const { return m_Data.data(); }
2628 
2629  void Add(char ch) { m_Data.push_back(ch); }
2630  void Add(const char* pStr);
2631  void AddNewLine() { Add('\n'); }
2632  void AddNumber(uint32_t num);
2633  void AddNumber(uint64_t num);
2634  void AddBool(bool b) { Add(b ? "true" : "false"); }
2635  void AddNull() { Add("null"); }
2636  void AddString(const char* pStr);
2637 
2638 private:
2639  VmaVector< char, VmaStlAllocator<char> > m_Data;
2640 };
2641 
2642 void VmaStringBuilder::Add(const char* pStr)
2643 {
2644  const size_t strLen = strlen(pStr);
2645  if(strLen > 0)
2646  {
2647  const size_t oldCount = m_Data.size();
2648  m_Data.resize(oldCount + strLen);
2649  memcpy(m_Data.data() + oldCount, pStr, strLen);
2650  }
2651 }
2652 
2653 void VmaStringBuilder::AddNumber(uint32_t num)
2654 {
2655  char buf[11];
2656  VmaUint32ToStr(buf, sizeof(buf), num);
2657  Add(buf);
2658 }
2659 
2660 void VmaStringBuilder::AddNumber(uint64_t num)
2661 {
2662  char buf[21];
2663  VmaUint64ToStr(buf, sizeof(buf), num);
2664  Add(buf);
2665 }
2666 
2667 void VmaStringBuilder::AddString(const char* pStr)
2668 {
2669  Add('"');
2670  const size_t strLen = strlen(pStr);
2671  for(size_t i = 0; i < strLen; ++i)
2672  {
2673  char ch = pStr[i];
2674  if(ch == '\'')
2675  {
2676  Add("\\\\");
2677  }
2678  else if(ch == '"')
2679  {
2680  Add("\\\"");
2681  }
2682  else if(ch >= 32)
2683  {
2684  Add(ch);
2685  }
2686  else switch(ch)
2687  {
2688  case '\n':
2689  Add("\\n");
2690  break;
2691  case '\r':
2692  Add("\\r");
2693  break;
2694  case '\t':
2695  Add("\\t");
2696  break;
2697  default:
2698  VMA_ASSERT(0 && "Character not currently supported.");
2699  break;
2700  }
2701  }
2702  Add('"');
2703 }
2704 
2706 
2707 VkDeviceMemory VmaAllocation_T::GetMemory() const
2708 {
2709  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2710  m_BlockAllocation.m_Block->m_hMemory : m_OwnAllocation.m_hMemory;
2711 }
2712 
2713 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
2714 {
2715  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2716  m_BlockAllocation.m_Block->m_MemoryTypeIndex : m_OwnAllocation.m_MemoryTypeIndex;
2717 }
2718 
2719 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
2720 {
2721  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
2722  m_BlockAllocation.m_Block->m_BlockVectorType :
2723  (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
2724 }
2725 
2726 void* VmaAllocation_T::GetMappedData() const
2727 {
2728  switch(m_Type)
2729  {
2730  case ALLOCATION_TYPE_BLOCK:
2731  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
2732  {
2733  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
2734  }
2735  else
2736  {
2737  return VMA_NULL;
2738  }
2739  break;
2740  case ALLOCATION_TYPE_OWN:
2741  return m_OwnAllocation.m_pMappedData;
2742  default:
2743  VMA_ASSERT(0);
2744  return VMA_NULL;
2745  }
2746 }
2747 
2748 // Correspond to values of enum VmaSuballocationType.
2749 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
2750  "FREE",
2751  "UNKNOWN",
2752  "BUFFER",
2753  "IMAGE_UNKNOWN",
2754  "IMAGE_LINEAR",
2755  "IMAGE_OPTIMAL",
2756 };
2757 
2758 static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat)
2759 {
2760  sb.Add("{ \"Allocations\": ");
2761  sb.AddNumber(stat.AllocationCount);
2762  sb.Add(", \"Suballocations\": ");
2763  sb.AddNumber(stat.SuballocationCount);
2764  sb.Add(", \"UnusedRanges\": ");
2765  sb.AddNumber(stat.UnusedRangeCount);
2766  sb.Add(", \"UsedBytes\": ");
2767  sb.AddNumber(stat.UsedBytes);
2768  sb.Add(", \"UnusedBytes\": ");
2769  sb.AddNumber(stat.UnusedBytes);
2770  sb.Add(", \"SuballocationSize\": { \"Min\": ");
2771  sb.AddNumber(stat.SuballocationSizeMin);
2772  sb.Add(", \"Avg\": ");
2773  sb.AddNumber(stat.SuballocationSizeAvg);
2774  sb.Add(", \"Max\": ");
2775  sb.AddNumber(stat.SuballocationSizeMax);
2776  sb.Add(" }, \"UnusedRangeSize\": { \"Min\": ");
2777  sb.AddNumber(stat.UnusedRangeSizeMin);
2778  sb.Add(", \"Avg\": ");
2779  sb.AddNumber(stat.UnusedRangeSizeAvg);
2780  sb.Add(", \"Max\": ");
2781  sb.AddNumber(stat.UnusedRangeSizeMax);
2782  sb.Add(" } }");
2783 }
2784 
2785 #endif // #if VMA_STATS_STRING_ENABLED
2786 
2787 struct VmaSuballocationItemSizeLess
2788 {
2789  bool operator()(
2790  const VmaSuballocationList::iterator lhs,
2791  const VmaSuballocationList::iterator rhs) const
2792  {
2793  return lhs->size < rhs->size;
2794  }
2795  bool operator()(
2796  const VmaSuballocationList::iterator lhs,
2797  VkDeviceSize rhsSize) const
2798  {
2799  return lhs->size < rhsSize;
2800  }
2801 };
2802 
2803 VmaBlock::VmaBlock(VmaAllocator hAllocator) :
2804  m_MemoryTypeIndex(UINT32_MAX),
2805  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
2806  m_hMemory(VK_NULL_HANDLE),
2807  m_Size(0),
2808  m_PersistentMap(false),
2809  m_pMappedData(VMA_NULL),
2810  m_FreeCount(0),
2811  m_SumFreeSize(0),
2812  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
2813  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
2814 {
2815 }
2816 
2817 void VmaBlock::Init(
2818  uint32_t newMemoryTypeIndex,
2819  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
2820  VkDeviceMemory newMemory,
2821  VkDeviceSize newSize,
2822  bool persistentMap,
2823  void* pMappedData)
2824 {
2825  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2826 
2827  m_MemoryTypeIndex = newMemoryTypeIndex;
2828  m_BlockVectorType = newBlockVectorType;
2829  m_hMemory = newMemory;
2830  m_Size = newSize;
2831  m_PersistentMap = persistentMap;
2832  m_pMappedData = pMappedData;
2833  m_FreeCount = 1;
2834  m_SumFreeSize = newSize;
2835 
2836  m_Suballocations.clear();
2837  m_FreeSuballocationsBySize.clear();
2838 
2839  VmaSuballocation suballoc = {};
2840  suballoc.offset = 0;
2841  suballoc.size = newSize;
2842  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2843 
2844  m_Suballocations.push_back(suballoc);
2845  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
2846  --suballocItem;
2847  m_FreeSuballocationsBySize.push_back(suballocItem);
2848 }
2849 
2850 void VmaBlock::Destroy(VmaAllocator allocator)
2851 {
2852  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
2853  if(m_pMappedData != VMA_NULL)
2854  {
2855  vkUnmapMemory(allocator->m_hDevice, m_hMemory);
2856  m_pMappedData = VMA_NULL;
2857  }
2858 
2859  // Callback.
2860  if(allocator->m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
2861  {
2862  (*allocator->m_DeviceMemoryCallbacks.pfnFree)(allocator, m_MemoryTypeIndex, m_hMemory, m_Size);
2863  }
2864 
2865  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
2866  m_hMemory = VK_NULL_HANDLE;
2867 }
2868 
2869 bool VmaBlock::Validate() const
2870 {
2871  if((m_hMemory == VK_NULL_HANDLE) ||
2872  (m_Size == 0) ||
2873  m_Suballocations.empty())
2874  {
2875  return false;
2876  }
2877 
2878  // Expected offset of new suballocation as calculates from previous ones.
2879  VkDeviceSize calculatedOffset = 0;
2880  // Expected number of free suballocations as calculated from traversing their list.
2881  uint32_t calculatedFreeCount = 0;
2882  // Expected sum size of free suballocations as calculated from traversing their list.
2883  VkDeviceSize calculatedSumFreeSize = 0;
2884  // Expected number of free suballocations that should be registered in
2885  // m_FreeSuballocationsBySize calculated from traversing their list.
2886  size_t freeSuballocationsToRegister = 0;
2887  // True if previous visisted suballocation was free.
2888  bool prevFree = false;
2889 
2890  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2891  suballocItem != m_Suballocations.cend();
2892  ++suballocItem)
2893  {
2894  const VmaSuballocation& subAlloc = *suballocItem;
2895 
2896  // Actual offset of this suballocation doesn't match expected one.
2897  if(subAlloc.offset != calculatedOffset)
2898  {
2899  return false;
2900  }
2901 
2902  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
2903  // Two adjacent free suballocations are invalid. They should be merged.
2904  if(prevFree && currFree)
2905  {
2906  return false;
2907  }
2908  prevFree = currFree;
2909 
2910  if(currFree)
2911  {
2912  calculatedSumFreeSize += subAlloc.size;
2913  ++calculatedFreeCount;
2914  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2915  {
2916  ++freeSuballocationsToRegister;
2917  }
2918  }
2919 
2920  calculatedOffset += subAlloc.size;
2921  }
2922 
2923  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
2924  // match expected one.
2925  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
2926  {
2927  return false;
2928  }
2929 
2930  VkDeviceSize lastSize = 0;
2931  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
2932  {
2933  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
2934 
2935  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
2936  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
2937  {
2938  return false;
2939  }
2940  // They must be sorted by size ascending.
2941  if(suballocItem->size < lastSize)
2942  {
2943  return false;
2944  }
2945 
2946  lastSize = suballocItem->size;
2947  }
2948 
2949  // Check if totals match calculacted values.
2950  return
2951  (calculatedOffset == m_Size) &&
2952  (calculatedSumFreeSize == m_SumFreeSize) &&
2953  (calculatedFreeCount == m_FreeCount);
2954 }
2955 
2956 /*
2957 How many suitable free suballocations to analyze before choosing best one.
2958 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
2959  be chosen.
2960 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
2961  suballocations will be analized and best one will be chosen.
2962 - Any other value is also acceptable.
2963 */
2964 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
2965 
2966 bool VmaBlock::CreateAllocationRequest(
2967  VkDeviceSize bufferImageGranularity,
2968  VkDeviceSize allocSize,
2969  VkDeviceSize allocAlignment,
2970  VmaSuballocationType allocType,
2971  VmaAllocationRequest* pAllocationRequest)
2972 {
2973  VMA_ASSERT(allocSize > 0);
2974  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2975  VMA_ASSERT(pAllocationRequest != VMA_NULL);
2976  VMA_HEAVY_ASSERT(Validate());
2977 
2978  // There is not enough total free space in this allocation to fullfill the request: Early return.
2979  if(m_SumFreeSize < allocSize)
2980  {
2981  return false;
2982  }
2983 
2984  // Old brute-force algorithm, linearly searching suballocations.
2985  /*
2986  uint32_t suitableSuballocationsFound = 0;
2987  for(VmaSuballocationList::iterator suballocItem = suballocations.Front();
2988  suballocItem != VMA_NULL &&
2989  suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK;
2990  suballocItem = suballocItem->Next)
2991  {
2992  if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE)
2993  {
2994  VkDeviceSize offset = 0, cost = 0;
2995  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost))
2996  {
2997  ++suitableSuballocationsFound;
2998  if(cost < costLimit)
2999  {
3000  pAllocationRequest->freeSuballocationItem = suballocItem;
3001  pAllocationRequest->offset = offset;
3002  pAllocationRequest->cost = cost;
3003  if(cost == 0)
3004  return true;
3005  costLimit = cost;
3006  betterSuballocationFound = true;
3007  }
3008  }
3009  }
3010  }
3011  */
3012 
3013  // New algorithm, efficiently searching freeSuballocationsBySize.
3014  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
3015  if(freeSuballocCount > 0)
3016  {
3017  if(VMA_BEST_FIT)
3018  {
3019  // Find first free suballocation with size not less than allocSize.
3020  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3021  m_FreeSuballocationsBySize.data(),
3022  m_FreeSuballocationsBySize.data() + freeSuballocCount,
3023  allocSize,
3024  VmaSuballocationItemSizeLess());
3025  size_t index = it - m_FreeSuballocationsBySize.data();
3026  for(; index < freeSuballocCount; ++index)
3027  {
3028  VkDeviceSize offset = 0;
3029  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
3030  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
3031  {
3032  pAllocationRequest->freeSuballocationItem = suballocItem;
3033  pAllocationRequest->offset = offset;
3034  return true;
3035  }
3036  }
3037  }
3038  else
3039  {
3040  // Search staring from biggest suballocations.
3041  for(size_t index = freeSuballocCount; index--; )
3042  {
3043  VkDeviceSize offset = 0;
3044  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
3045  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
3046  {
3047  pAllocationRequest->freeSuballocationItem = suballocItem;
3048  pAllocationRequest->offset = offset;
3049  return true;
3050  }
3051  }
3052  }
3053  }
3054 
3055  return false;
3056 }
3057 
3058 bool VmaBlock::CheckAllocation(
3059  VkDeviceSize bufferImageGranularity,
3060  VkDeviceSize allocSize,
3061  VkDeviceSize allocAlignment,
3062  VmaSuballocationType allocType,
3063  VmaSuballocationList::const_iterator freeSuballocItem,
3064  VkDeviceSize* pOffset) const
3065 {
3066  VMA_ASSERT(allocSize > 0);
3067  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
3068  VMA_ASSERT(freeSuballocItem != m_Suballocations.cend());
3069  VMA_ASSERT(pOffset != VMA_NULL);
3070 
3071  const VmaSuballocation& suballoc = *freeSuballocItem;
3072  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
3073 
3074  // Size of this suballocation is too small for this request: Early return.
3075  if(suballoc.size < allocSize)
3076  {
3077  return false;
3078  }
3079 
3080  // Start from offset equal to beginning of this suballocation.
3081  *pOffset = suballoc.offset;
3082 
3083  // Apply VMA_DEBUG_MARGIN at the beginning.
3084  if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin())
3085  {
3086  *pOffset += VMA_DEBUG_MARGIN;
3087  }
3088 
3089  // Apply alignment.
3090  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
3091  *pOffset = VmaAlignUp(*pOffset, alignment);
3092 
3093  // Check previous suballocations for BufferImageGranularity conflicts.
3094  // Make bigger alignment if necessary.
3095  if(bufferImageGranularity > 1)
3096  {
3097  bool bufferImageGranularityConflict = false;
3098  VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem;
3099  while(prevSuballocItem != m_Suballocations.cbegin())
3100  {
3101  --prevSuballocItem;
3102  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
3103  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
3104  {
3105  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
3106  {
3107  bufferImageGranularityConflict = true;
3108  break;
3109  }
3110  }
3111  else
3112  // Already on previous page.
3113  break;
3114  }
3115  if(bufferImageGranularityConflict)
3116  {
3117  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
3118  }
3119  }
3120 
3121  // Calculate padding at the beginning based on current offset.
3122  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
3123 
3124  // Calculate required margin at the end if this is not last suballocation.
3125  VmaSuballocationList::const_iterator next = freeSuballocItem;
3126  ++next;
3127  const VkDeviceSize requiredEndMargin =
3128  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
3129 
3130  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
3131  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
3132  {
3133  return false;
3134  }
3135 
3136  // Check next suballocations for BufferImageGranularity conflicts.
3137  // If conflict exists, allocation cannot be made here.
3138  if(bufferImageGranularity > 1)
3139  {
3140  VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem;
3141  ++nextSuballocItem;
3142  while(nextSuballocItem != m_Suballocations.cend())
3143  {
3144  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
3145  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
3146  {
3147  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
3148  {
3149  return false;
3150  }
3151  }
3152  else
3153  {
3154  // Already on next page.
3155  break;
3156  }
3157  ++nextSuballocItem;
3158  }
3159  }
3160 
3161  // All tests passed: Success. pOffset is already filled.
3162  return true;
3163 }
3164 
3165 bool VmaBlock::IsEmpty() const
3166 {
3167  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
3168 }
3169 
3170 void VmaBlock::Alloc(
3171  const VmaAllocationRequest& request,
3172  VmaSuballocationType type,
3173  VkDeviceSize allocSize)
3174 {
3175  VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end());
3176  VmaSuballocation& suballoc = *request.freeSuballocationItem;
3177  // Given suballocation is a free block.
3178  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
3179  // Given offset is inside this suballocation.
3180  VMA_ASSERT(request.offset >= suballoc.offset);
3181  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
3182  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
3183  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
3184 
3185  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
3186  // it to become used.
3187  UnregisterFreeSuballocation(request.freeSuballocationItem);
3188 
3189  suballoc.offset = request.offset;
3190  suballoc.size = allocSize;
3191  suballoc.type = type;
3192 
3193  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
3194  if(paddingEnd)
3195  {
3196  VmaSuballocation paddingSuballoc = {};
3197  paddingSuballoc.offset = request.offset + allocSize;
3198  paddingSuballoc.size = paddingEnd;
3199  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3200  VmaSuballocationList::iterator next = request.freeSuballocationItem;
3201  ++next;
3202  const VmaSuballocationList::iterator paddingEndItem =
3203  m_Suballocations.insert(next, paddingSuballoc);
3204  RegisterFreeSuballocation(paddingEndItem);
3205  }
3206 
3207  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
3208  if(paddingBegin)
3209  {
3210  VmaSuballocation paddingSuballoc = {};
3211  paddingSuballoc.offset = request.offset - paddingBegin;
3212  paddingSuballoc.size = paddingBegin;
3213  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3214  const VmaSuballocationList::iterator paddingBeginItem =
3215  m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc);
3216  RegisterFreeSuballocation(paddingBeginItem);
3217  }
3218 
3219  // Update totals.
3220  m_FreeCount = m_FreeCount - 1;
3221  if(paddingBegin > 0)
3222  {
3223  ++m_FreeCount;
3224  }
3225  if(paddingEnd > 0)
3226  {
3227  ++m_FreeCount;
3228  }
3229  m_SumFreeSize -= allocSize;
3230 }
3231 
3232 void VmaBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
3233 {
3234  // Change this suballocation to be marked as free.
3235  VmaSuballocation& suballoc = *suballocItem;
3236  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
3237 
3238  // Update totals.
3239  ++m_FreeCount;
3240  m_SumFreeSize += suballoc.size;
3241 
3242  // Merge with previous and/or next suballocation if it's also free.
3243  bool mergeWithNext = false;
3244  bool mergeWithPrev = false;
3245 
3246  VmaSuballocationList::iterator nextItem = suballocItem;
3247  ++nextItem;
3248  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
3249  {
3250  mergeWithNext = true;
3251  }
3252 
3253  VmaSuballocationList::iterator prevItem = suballocItem;
3254  if(suballocItem != m_Suballocations.begin())
3255  {
3256  --prevItem;
3257  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
3258  {
3259  mergeWithPrev = true;
3260  }
3261  }
3262 
3263  if(mergeWithNext)
3264  {
3265  UnregisterFreeSuballocation(nextItem);
3266  MergeFreeWithNext(suballocItem);
3267  }
3268 
3269  if(mergeWithPrev)
3270  {
3271  UnregisterFreeSuballocation(prevItem);
3272  MergeFreeWithNext(prevItem);
3273  RegisterFreeSuballocation(prevItem);
3274  }
3275  else
3276  RegisterFreeSuballocation(suballocItem);
3277 }
3278 
3279 void VmaBlock::Free(const VmaAllocation allocation)
3280 {
3281  const VkDeviceSize allocationOffset = allocation->GetOffset();
3282  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
3283  suballocItem != m_Suballocations.end();
3284  ++suballocItem)
3285  {
3286  VmaSuballocation& suballoc = *suballocItem;
3287  if(suballoc.offset == allocationOffset)
3288  {
3289  FreeSuballocation(suballocItem);
3290  VMA_HEAVY_ASSERT(Validate());
3291  return;
3292  }
3293  }
3294  VMA_ASSERT(0 && "Not found!");
3295 }
3296 
3297 #if VMA_STATS_STRING_ENABLED
3298 
3299 void VmaBlock::PrintDetailedMap(class VmaStringBuilder& sb) const
3300 {
3301  sb.Add("{\n\t\t\t\"Bytes\": ");
3302  sb.AddNumber(m_Size);
3303  sb.Add(",\n\t\t\t\"FreeBytes\": ");
3304  sb.AddNumber(m_SumFreeSize);
3305  sb.Add(",\n\t\t\t\"Suballocations\": ");
3306  sb.AddNumber(m_Suballocations.size());
3307  sb.Add(",\n\t\t\t\"FreeSuballocations\": ");
3308  sb.AddNumber(m_FreeCount);
3309  sb.Add(",\n\t\t\t\"SuballocationList\": [");
3310 
3311  size_t i = 0;
3312  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
3313  suballocItem != m_Suballocations.cend();
3314  ++suballocItem, ++i)
3315  {
3316  if(i > 0)
3317  {
3318  sb.Add(",\n\t\t\t\t{ \"Type\": ");
3319  }
3320  else
3321  {
3322  sb.Add("\n\t\t\t\t{ \"Type\": ");
3323  }
3324  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
3325  sb.Add(", \"Size\": ");
3326  sb.AddNumber(suballocItem->size);
3327  sb.Add(", \"Offset\": ");
3328  sb.AddNumber(suballocItem->offset);
3329  sb.Add(" }");
3330  }
3331 
3332  sb.Add("\n\t\t\t]\n\t\t}");
3333 }
3334 
3335 #endif // #if VMA_STATS_STRING_ENABLED
3336 
3337 void VmaBlock::MergeFreeWithNext(VmaSuballocationList::iterator item)
3338 {
3339  VMA_ASSERT(item != m_Suballocations.end());
3340  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3341 
3342  VmaSuballocationList::iterator nextItem = item;
3343  ++nextItem;
3344  VMA_ASSERT(nextItem != m_Suballocations.end());
3345  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
3346 
3347  item->size += nextItem->size;
3348  --m_FreeCount;
3349  m_Suballocations.erase(nextItem);
3350 }
3351 
3352 void VmaBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
3353 {
3354  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3355  VMA_ASSERT(item->size > 0);
3356 
3357  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
3358  {
3359  if(m_FreeSuballocationsBySize.empty())
3360  {
3361  m_FreeSuballocationsBySize.push_back(item);
3362  }
3363  else
3364  {
3365  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3366  m_FreeSuballocationsBySize.data(),
3367  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
3368  item,
3369  VmaSuballocationItemSizeLess());
3370  size_t index = it - m_FreeSuballocationsBySize.data();
3371  VectorInsert(m_FreeSuballocationsBySize, index, item);
3372  }
3373  }
3374 }
3375 
3376 void VmaBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
3377 {
3378  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
3379  VMA_ASSERT(item->size > 0);
3380 
3381  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
3382  {
3383  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
3384  m_FreeSuballocationsBySize.data(),
3385  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
3386  item,
3387  VmaSuballocationItemSizeLess());
3388  for(size_t index = it - m_FreeSuballocationsBySize.data();
3389  index < m_FreeSuballocationsBySize.size();
3390  ++index)
3391  {
3392  if(m_FreeSuballocationsBySize[index] == item)
3393  {
3394  VectorRemove(m_FreeSuballocationsBySize, index);
3395  return;
3396  }
3397  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
3398  }
3399  VMA_ASSERT(0 && "Not found.");
3400  }
3401 }
3402 
3403 static void InitStatInfo(VmaStatInfo& outInfo)
3404 {
3405  memset(&outInfo, 0, sizeof(outInfo));
3406  outInfo.SuballocationSizeMin = UINT64_MAX;
3407  outInfo.UnusedRangeSizeMin = UINT64_MAX;
3408 }
3409 
3410 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc)
3411 {
3412  outInfo.AllocationCount = 1;
3413 
3414  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
3415  outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount;
3416  outInfo.UnusedRangeCount = alloc.m_FreeCount;
3417 
3418  outInfo.UnusedBytes = alloc.m_SumFreeSize;
3419  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
3420 
3421  outInfo.SuballocationSizeMin = UINT64_MAX;
3422  outInfo.SuballocationSizeMax = 0;
3423  outInfo.UnusedRangeSizeMin = UINT64_MAX;
3424  outInfo.UnusedRangeSizeMax = 0;
3425 
3426  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
3427  suballocItem != alloc.m_Suballocations.cend();
3428  ++suballocItem)
3429  {
3430  const VmaSuballocation& suballoc = *suballocItem;
3431  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
3432  {
3433  outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size);
3434  outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size);
3435  }
3436  else
3437  {
3438  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
3439  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
3440  }
3441  }
3442 }
3443 
3444 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
3445 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
3446 {
3447  inoutInfo.AllocationCount += srcInfo.AllocationCount;
3448  inoutInfo.SuballocationCount += srcInfo.SuballocationCount;
3449  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
3450  inoutInfo.UsedBytes += srcInfo.UsedBytes;
3451  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
3452  inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin);
3453  inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax);
3454  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
3455  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
3456 }
3457 
3458 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
3459 {
3460  inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ?
3461  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0;
3462  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
3463  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
3464 }
3465 
3466 VmaBlockVector::VmaBlockVector(VmaAllocator hAllocator) :
3467  m_hAllocator(hAllocator),
3468  m_Blocks(VmaStlAllocator<VmaBlock*>(hAllocator->GetAllocationCallbacks()))
3469 {
3470 }
3471 
3472 VmaBlockVector::~VmaBlockVector()
3473 {
3474  for(size_t i = m_Blocks.size(); i--; )
3475  {
3476  m_Blocks[i]->Destroy(m_hAllocator);
3477  vma_delete(m_hAllocator, m_Blocks[i]);
3478  }
3479 }
3480 
3481 void VmaBlockVector::Remove(VmaBlock* pBlock)
3482 {
3483  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
3484  {
3485  if(m_Blocks[blockIndex] == pBlock)
3486  {
3487  VectorRemove(m_Blocks, blockIndex);
3488  return;
3489  }
3490  }
3491  VMA_ASSERT(0);
3492 }
3493 
3494 void VmaBlockVector::IncrementallySortBlocks()
3495 {
3496  // Bubble sort only until first swap.
3497  for(size_t i = 1; i < m_Blocks.size(); ++i)
3498  {
3499  if(m_Blocks[i - 1]->m_SumFreeSize > m_Blocks[i]->m_SumFreeSize)
3500  {
3501  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
3502  return;
3503  }
3504  }
3505 }
3506 
3507 #if VMA_STATS_STRING_ENABLED
3508 
3509 void VmaBlockVector::PrintDetailedMap(class VmaStringBuilder& sb) const
3510 {
3511  for(size_t i = 0; i < m_Blocks.size(); ++i)
3512  {
3513  if(i > 0)
3514  {
3515  sb.Add(",\n\t\t");
3516  }
3517  else
3518  {
3519  sb.Add("\n\t\t");
3520  }
3521  m_Blocks[i]->PrintDetailedMap(sb);
3522  }
3523 }
3524 
3525 #endif // #if VMA_STATS_STRING_ENABLED
3526 
3527 void VmaBlockVector::UnmapPersistentlyMappedMemory()
3528 {
3529  for(size_t i = m_Blocks.size(); i--; )
3530  {
3531  VmaBlock* pBlock = m_Blocks[i];
3532  if(pBlock->m_pMappedData != VMA_NULL)
3533  {
3534  VMA_ASSERT(pBlock->m_PersistentMap != false);
3535  vkUnmapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory);
3536  pBlock->m_pMappedData = VMA_NULL;
3537  }
3538  }
3539 }
3540 
3541 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
3542 {
3543  VkResult finalResult = VK_SUCCESS;
3544  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
3545  {
3546  VmaBlock* pBlock = m_Blocks[i];
3547  if(pBlock->m_PersistentMap)
3548  {
3549  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
3550  VkResult localResult = vkMapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &pBlock->m_pMappedData);
3551  if(localResult != VK_SUCCESS)
3552  {
3553  finalResult = localResult;
3554  }
3555  }
3556  }
3557  return finalResult;
3558 }
3559 
3560 void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const
3561 {
3562  for(uint32_t allocIndex = 0; allocIndex < m_Blocks.size(); ++allocIndex)
3563  {
3564  const VmaBlock* const pBlock = m_Blocks[allocIndex];
3565  VMA_ASSERT(pBlock);
3566  VMA_HEAVY_ASSERT(pBlock->Validate());
3567  VmaStatInfo allocationStatInfo;
3568  CalcAllocationStatInfo(allocationStatInfo, *pBlock);
3569  VmaAddStatInfo(pStats->total, allocationStatInfo);
3570  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
3571  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
3572  }
3573 }
3574 
3576 // VmaDefragmentator
3577 
3578 class VmaDefragmentator
3579 {
3580  VkDevice m_hDevice;
3581  const VkAllocationCallbacks* m_pAllocationCallbacks;
3582  VkDeviceSize m_BufferImageGranularity;
3583  uint32_t m_MemTypeIndex;
3584  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
3585  VkDeviceSize m_BytesMoved;
3586  uint32_t m_AllocationsMoved;
3587 
3588  struct AllocationInfo
3589  {
3590  VmaAllocation m_hAllocation;
3591  VkBool32* m_pChanged;
3592 
3593  AllocationInfo() :
3594  m_hAllocation(VK_NULL_HANDLE),
3595  m_pChanged(VMA_NULL)
3596  {
3597  }
3598  };
3599 
3600  struct AllocationInfoSizeGreater
3601  {
3602  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
3603  {
3604  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
3605  }
3606  };
3607 
3608  // Used between AddAllocation and Defragment.
3609  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3610 
3611  struct BlockInfo
3612  {
3613  VmaBlock* m_pBlock;
3614  bool m_HasNonMovableAllocations;
3615  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
3616 
3617  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
3618  m_pBlock(VMA_NULL),
3619  m_HasNonMovableAllocations(true),
3620  m_Allocations(pAllocationCallbacks),
3621  m_pMappedDataForDefragmentation(VMA_NULL)
3622  {
3623  }
3624 
3625  void CalcHasNonMovableAllocations()
3626  {
3627  const size_t blockAllocCount =
3628  m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount;
3629  const size_t defragmentAllocCount = m_Allocations.size();
3630  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
3631  }
3632 
3633  void SortAllocationsBySizeDescecnding()
3634  {
3635  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
3636  }
3637 
3638  VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData)
3639  {
3640  // It has already been mapped for defragmentation.
3641  if(m_pMappedDataForDefragmentation)
3642  {
3643  *ppMappedData = m_pMappedDataForDefragmentation;
3644  return VK_SUCCESS;
3645  }
3646 
3647  // It is persistently mapped.
3648  if(m_pBlock->m_PersistentMap)
3649  {
3650  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
3651  *ppMappedData = m_pBlock->m_pMappedData;
3652  return VK_SUCCESS;
3653  }
3654 
3655  // Map on first usage.
3656  VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation);
3657  *ppMappedData = m_pMappedDataForDefragmentation;
3658  return res;
3659  }
3660 
3661  void Unmap(VkDevice hDevice)
3662  {
3663  if(m_pMappedDataForDefragmentation != VMA_NULL)
3664  {
3665  vkUnmapMemory(hDevice, m_pBlock->m_hMemory);
3666  }
3667  }
3668 
3669  private:
3670  // Not null if mapped for defragmentation only, not persistently mapped.
3671  void* m_pMappedDataForDefragmentation;
3672  };
3673 
3674  struct BlockPointerLess
3675  {
3676  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaBlock* pRhsBlock) const
3677  {
3678  return pLhsBlockInfo->m_pBlock < pRhsBlock;
3679  }
3680  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3681  {
3682  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
3683  }
3684  };
3685 
3686  // 1. Blocks with some non-movable allocations go first.
3687  // 2. Blocks with smaller sumFreeSize go first.
3688  struct BlockInfoCompareMoveDestination
3689  {
3690  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
3691  {
3692  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
3693  {
3694  return true;
3695  }
3696  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
3697  {
3698  return false;
3699  }
3700  if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize)
3701  {
3702  return true;
3703  }
3704  return false;
3705  }
3706  };
3707 
3708  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
3709  BlockInfoVector m_Blocks;
3710 
3711  VkResult DefragmentRound(
3712  VkDeviceSize maxBytesToMove,
3713  uint32_t maxAllocationsToMove);
3714 
3715  static bool MoveMakesSense(
3716  size_t dstBlockIndex, VkDeviceSize dstOffset,
3717  size_t srcBlockIndex, VkDeviceSize srcOffset);
3718 
3719 public:
3720  VmaDefragmentator(
3721  VkDevice hDevice,
3722  const VkAllocationCallbacks* pAllocationCallbacks,
3723  VkDeviceSize bufferImageGranularity,
3724  uint32_t memTypeIndex,
3725  VMA_BLOCK_VECTOR_TYPE blockVectorType);
3726 
3727  ~VmaDefragmentator();
3728 
3729  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
3730  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
3731 
3732  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
3733 
3734  VkResult Defragment(
3735  VmaBlockVector* pBlockVector,
3736  VkDeviceSize maxBytesToMove,
3737  uint32_t maxAllocationsToMove);
3738 };
3739 
3740 VmaDefragmentator::VmaDefragmentator(
3741  VkDevice hDevice,
3742  const VkAllocationCallbacks* pAllocationCallbacks,
3743  VkDeviceSize bufferImageGranularity,
3744  uint32_t memTypeIndex,
3745  VMA_BLOCK_VECTOR_TYPE blockVectorType) :
3746  m_hDevice(hDevice),
3747  m_pAllocationCallbacks(pAllocationCallbacks),
3748  m_BufferImageGranularity(bufferImageGranularity),
3749  m_MemTypeIndex(memTypeIndex),
3750  m_BlockVectorType(blockVectorType),
3751  m_BytesMoved(0),
3752  m_AllocationsMoved(0),
3753  m_Allocations(VmaStlAllocator<AllocationInfo>(pAllocationCallbacks)),
3754  m_Blocks(VmaStlAllocator<BlockInfo*>(pAllocationCallbacks))
3755 {
3756 }
3757 
3758 VmaDefragmentator::~VmaDefragmentator()
3759 {
3760  for(size_t i = m_Blocks.size(); i--; )
3761  {
3762  vma_delete(m_pAllocationCallbacks, m_Blocks[i]);
3763  }
3764 }
3765 
3766 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
3767 {
3768  AllocationInfo allocInfo;
3769  allocInfo.m_hAllocation = hAlloc;
3770  allocInfo.m_pChanged = pChanged;
3771  m_Allocations.push_back(allocInfo);
3772 }
3773 
3774 VkResult VmaDefragmentator::DefragmentRound(
3775  VkDeviceSize maxBytesToMove,
3776  uint32_t maxAllocationsToMove)
3777 {
3778  if(m_Blocks.empty())
3779  {
3780  return VK_SUCCESS;
3781  }
3782 
3783  size_t srcBlockIndex = m_Blocks.size() - 1;
3784  size_t srcAllocIndex = SIZE_MAX;
3785  for(;;)
3786  {
3787  // 1. Find next allocation to move.
3788  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
3789  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
3790  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
3791  {
3792  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
3793  {
3794  // Finished: no more allocations to process.
3795  if(srcBlockIndex == 0)
3796  {
3797  return VK_SUCCESS;
3798  }
3799  else
3800  {
3801  --srcBlockIndex;
3802  srcAllocIndex = SIZE_MAX;
3803  }
3804  }
3805  else
3806  {
3807  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
3808  }
3809  }
3810 
3811  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
3812  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
3813 
3814  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
3815  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
3816  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
3817  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
3818 
3819  // 2. Try to find new place for this allocation in preceding or current block.
3820  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
3821  {
3822  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
3823  VmaAllocationRequest dstAllocRequest;
3824  if(pDstBlockInfo->m_pBlock->CreateAllocationRequest(
3825  m_BufferImageGranularity,
3826  size,
3827  alignment,
3828  suballocType,
3829  &dstAllocRequest) &&
3830  MoveMakesSense(
3831  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
3832  {
3833  // Reached limit on number of allocations or bytes to move.
3834  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
3835  (m_BytesMoved + size > maxBytesToMove))
3836  {
3837  return VK_INCOMPLETE;
3838  }
3839 
3840  void* pDstMappedData = VMA_NULL;
3841  VkResult res = pDstBlockInfo->EnsureMapping(m_hDevice, &pDstMappedData);
3842  if(res != VK_SUCCESS)
3843  {
3844  return res;
3845  }
3846 
3847  void* pSrcMappedData = VMA_NULL;
3848  res = pSrcBlockInfo->EnsureMapping(m_hDevice, &pSrcMappedData);
3849  if(res != VK_SUCCESS)
3850  {
3851  return res;
3852  }
3853 
3854  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
3855  memcpy(
3856  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
3857  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
3858  size);
3859 
3860  pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size);
3861  pSrcBlockInfo->m_pBlock->Free(allocInfo.m_hAllocation);
3862 
3863  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
3864 
3865  if(allocInfo.m_pChanged != VMA_NULL)
3866  {
3867  *allocInfo.m_pChanged = VK_TRUE;
3868  }
3869 
3870  ++m_AllocationsMoved;
3871  m_BytesMoved += size;
3872 
3873  VectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
3874 
3875  break;
3876  }
3877  }
3878 
3879  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
3880 
3881  if(srcAllocIndex > 0)
3882  {
3883  --srcAllocIndex;
3884  }
3885  else
3886  {
3887  if(srcBlockIndex > 0)
3888  {
3889  --srcBlockIndex;
3890  srcAllocIndex = SIZE_MAX;
3891  }
3892  else
3893  {
3894  return VK_SUCCESS;
3895  }
3896  }
3897  }
3898 }
3899 
3900 VkResult VmaDefragmentator::Defragment(
3901  VmaBlockVector* pBlockVector,
3902  VkDeviceSize maxBytesToMove,
3903  uint32_t maxAllocationsToMove)
3904 {
3905  if(m_Allocations.empty())
3906  {
3907  return VK_SUCCESS;
3908  }
3909 
3910  // Create block info for each block.
3911  const size_t blockCount = pBlockVector->m_Blocks.size();
3912  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3913  {
3914  BlockInfo* pBlockInfo = vma_new(m_pAllocationCallbacks, BlockInfo)(m_pAllocationCallbacks);
3915  pBlockInfo->m_pBlock = pBlockVector->m_Blocks[blockIndex];
3916  m_Blocks.push_back(pBlockInfo);
3917  }
3918 
3919  // Sort them by m_pBlock pointer value.
3920  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
3921 
3922  // Move allocation infos from m_Allocations to appropriate m_Blocks[i].m_Allocations.
3923  for(size_t allocIndex = 0, allocCount = m_Allocations.size(); allocIndex < allocCount; ++allocIndex)
3924  {
3925  AllocationInfo& allocInfo = m_Allocations[allocIndex];
3926  VmaBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
3927  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
3928  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
3929  {
3930  (*it)->m_Allocations.push_back(allocInfo);
3931  }
3932  else
3933  {
3934  VMA_ASSERT(0);
3935  }
3936  }
3937  m_Allocations.clear();
3938 
3939  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3940  {
3941  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
3942  pBlockInfo->CalcHasNonMovableAllocations();
3943  pBlockInfo->SortAllocationsBySizeDescecnding();
3944  }
3945 
3946  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
3947  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
3948 
3949  // Execute defragmentation round (the main part).
3950  VkResult result = VK_SUCCESS;
3951  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
3952  {
3953  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
3954  }
3955 
3956  // Unmap blocks that were mapped for defragmentation.
3957  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
3958  {
3959  m_Blocks[blockIndex]->Unmap(m_hDevice);
3960  }
3961 
3962  return result;
3963 }
3964 
3965 bool VmaDefragmentator::MoveMakesSense(
3966  size_t dstBlockIndex, VkDeviceSize dstOffset,
3967  size_t srcBlockIndex, VkDeviceSize srcOffset)
3968 {
3969  if(dstBlockIndex < srcBlockIndex)
3970  {
3971  return true;
3972  }
3973  if(dstBlockIndex > srcBlockIndex)
3974  {
3975  return false;
3976  }
3977  if(dstOffset < srcOffset)
3978  {
3979  return true;
3980  }
3981  return false;
3982 }
3983 
3985 // VmaAllocator_T
3986 
3987 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
3988  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
3989  m_PhysicalDevice(pCreateInfo->physicalDevice),
3990  m_hDevice(pCreateInfo->device),
3991  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
3992  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
3993  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
3994  m_PreferredLargeHeapBlockSize(0),
3995  m_PreferredSmallHeapBlockSize(0),
3996  m_UnmapPersistentlyMappedMemoryCounter(0)
3997 {
3998  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
3999 
4000  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
4001  memset(&m_MemProps, 0, sizeof(m_MemProps));
4002  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
4003 
4004  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
4005  memset(&m_HasEmptyBlock, 0, sizeof(m_HasEmptyBlock));
4006  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
4007 
4008  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
4009  {
4010  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
4011  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
4012  }
4013 
4014  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
4015  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
4016  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
4017  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
4018 
4019  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
4020  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
4021 
4022  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
4023  {
4024  for(size_t j = 0; j < VMA_BLOCK_VECTOR_TYPE_COUNT; ++j)
4025  {
4026  m_pBlockVectors[i][j] = vma_new(this, VmaBlockVector)(this);
4027  m_pOwnAllocations[i][j] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
4028  }
4029  }
4030 }
4031 
4032 VmaAllocator_T::~VmaAllocator_T()
4033 {
4034  for(size_t i = GetMemoryTypeCount(); i--; )
4035  {
4036  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
4037  {
4038  vma_delete(this, m_pOwnAllocations[i][j]);
4039  vma_delete(this, m_pBlockVectors[i][j]);
4040  }
4041  }
4042 }
4043 
4044 VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const
4045 {
4046  VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
4047  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
4048  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
4049 }
4050 
4051 VkResult VmaAllocator_T::AllocateMemoryOfType(
4052  const VkMemoryRequirements& vkMemReq,
4053  const VmaMemoryRequirements& vmaMemReq,
4054  uint32_t memTypeIndex,
4055  VmaSuballocationType suballocType,
4056  VmaAllocation* pAllocation)
4057 {
4058  VMA_ASSERT(pAllocation != VMA_NULL);
4059  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
4060 
4061  const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex);
4062  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
4063  const bool ownMemory =
4064  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 ||
4065  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
4066  ((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) == 0 &&
4067  vkMemReq.size > preferredBlockSize / 2);
4068 
4069  if(ownMemory)
4070  {
4071  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4072  {
4073  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4074  }
4075  else
4076  {
4077  return AllocateOwnMemory(
4078  vkMemReq.size,
4079  suballocType,
4080  memTypeIndex,
4081  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
4082  vmaMemReq.pUserData,
4083  pAllocation);
4084  }
4085  }
4086  else
4087  {
4088  uint32_t blockVectorType = VmaMemoryRequirementFlagsToBlockVectorType(vmaMemReq.flags);
4089 
4090  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4091  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4092  VMA_ASSERT(blockVector);
4093 
4094  // 1. Search existing allocations.
4095  // Forward order - prefer blocks with smallest amount of free space.
4096  for(size_t allocIndex = 0; allocIndex < blockVector->m_Blocks.size(); ++allocIndex )
4097  {
4098  VmaBlock* const pBlock = blockVector->m_Blocks[allocIndex];
4099  VMA_ASSERT(pBlock);
4100  VmaAllocationRequest allocRequest = {};
4101  // Check if can allocate from pBlock.
4102  if(pBlock->CreateAllocationRequest(
4103  GetBufferImageGranularity(),
4104  vkMemReq.size,
4105  vkMemReq.alignment,
4106  suballocType,
4107  &allocRequest))
4108  {
4109  // We no longer have an empty Allocation.
4110  if(pBlock->IsEmpty())
4111  {
4112  m_HasEmptyBlock[memTypeIndex] = false;
4113  }
4114  // Allocate from this pBlock.
4115  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
4116  *pAllocation = vma_new(this, VmaAllocation_T)();
4117  (*pAllocation)->InitBlockAllocation(
4118  pBlock,
4119  allocRequest.offset,
4120  vkMemReq.alignment,
4121  vkMemReq.size,
4122  suballocType,
4123  vmaMemReq.pUserData);
4124  VMA_HEAVY_ASSERT(pBlock->Validate());
4125  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex);
4126  return VK_SUCCESS;
4127  }
4128  }
4129 
4130  // 2. Create new Allocation.
4131  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4132  {
4133  VMA_DEBUG_LOG(" FAILED due to VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT");
4134  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4135  }
4136  else
4137  {
4138  // Start with full preferredBlockSize.
4139  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
4140  allocInfo.memoryTypeIndex = memTypeIndex;
4141  allocInfo.allocationSize = preferredBlockSize;
4142  VkDeviceMemory mem = VK_NULL_HANDLE;
4143  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4144  if(res < 0)
4145  {
4146  // 3. Try half the size.
4147  allocInfo.allocationSize /= 2;
4148  if(allocInfo.allocationSize >= vkMemReq.size)
4149  {
4150  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4151  if(res < 0)
4152  {
4153  // 4. Try quarter the size.
4154  allocInfo.allocationSize /= 2;
4155  if(allocInfo.allocationSize >= vkMemReq.size)
4156  {
4157  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
4158  }
4159  }
4160  }
4161  }
4162  if(res < 0)
4163  {
4164  // 5. Try OwnAlloc.
4165  res = AllocateOwnMemory(
4166  vkMemReq.size,
4167  suballocType,
4168  memTypeIndex,
4169  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
4170  vmaMemReq.pUserData,
4171  pAllocation);
4172  if(res == VK_SUCCESS)
4173  {
4174  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
4175  VMA_DEBUG_LOG(" Allocated as OwnMemory");
4176  return VK_SUCCESS;
4177  }
4178  else
4179  {
4180  // Everything failed: Return error code.
4181  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
4182  return res;
4183  }
4184  }
4185 
4186  // New VkDeviceMemory successfully created.
4187 
4188  // Map memory if needed.
4189  void* pMappedData = VMA_NULL;
4190  const bool persistentMap = (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0;
4191  if(persistentMap && m_UnmapPersistentlyMappedMemoryCounter == 0)
4192  {
4193  res = vkMapMemory(m_hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData);
4194  if(res < 0)
4195  {
4196  VMA_DEBUG_LOG(" vkMapMemory FAILED");
4197  vkFreeMemory(m_hDevice, mem, GetAllocationCallbacks());
4198  return res;
4199  }
4200  }
4201 
4202  // Callback.
4203  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
4204  {
4205  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, mem, allocInfo.allocationSize);
4206  }
4207 
4208  // Create new Allocation for it.
4209  VmaBlock* const pBlock = vma_new(this, VmaBlock)(this);
4210  pBlock->Init(
4211  memTypeIndex,
4212  (VMA_BLOCK_VECTOR_TYPE)blockVectorType,
4213  mem,
4214  allocInfo.allocationSize,
4215  persistentMap,
4216  pMappedData);
4217 
4218  blockVector->m_Blocks.push_back(pBlock);
4219 
4220  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
4221  VmaAllocationRequest allocRequest = {};
4222  allocRequest.freeSuballocationItem = pBlock->m_Suballocations.begin();
4223  allocRequest.offset = 0;
4224  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
4225  *pAllocation = vma_new(this, VmaAllocation_T)();
4226  (*pAllocation)->InitBlockAllocation(
4227  pBlock,
4228  allocRequest.offset,
4229  vkMemReq.alignment,
4230  vkMemReq.size,
4231  suballocType,
4232  vmaMemReq.pUserData);
4233  VMA_HEAVY_ASSERT(pBlock->Validate());
4234  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
4235  return VK_SUCCESS;
4236  }
4237  }
4238 }
4239 
4240 VkResult VmaAllocator_T::AllocateOwnMemory(
4241  VkDeviceSize size,
4242  VmaSuballocationType suballocType,
4243  uint32_t memTypeIndex,
4244  bool map,
4245  void* pUserData,
4246  VmaAllocation* pAllocation)
4247 {
4248  VMA_ASSERT(pAllocation);
4249 
4250  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
4251  allocInfo.memoryTypeIndex = memTypeIndex;
4252  allocInfo.allocationSize = size;
4253 
4254  // Allocate VkDeviceMemory.
4255  VkDeviceMemory hMemory = VK_NULL_HANDLE;
4256  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &hMemory);
4257  if(res < 0)
4258  {
4259  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
4260  return res;
4261  }
4262 
4263  void* pMappedData = nullptr;
4264  if(map)
4265  {
4266  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
4267  {
4268  res = vkMapMemory(m_hDevice, hMemory, 0, VK_WHOLE_SIZE, 0, &pMappedData);
4269  if(res < 0)
4270  {
4271  VMA_DEBUG_LOG(" vkMapMemory FAILED");
4272  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
4273  return res;
4274  }
4275  }
4276  }
4277 
4278  // Callback.
4279  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
4280  {
4281  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, hMemory, size);
4282  }
4283 
4284  *pAllocation = vma_new(this, VmaAllocation_T)();
4285  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
4286 
4287  // Register it in m_pOwnAllocations.
4288  {
4289  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4290  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
4291  VMA_ASSERT(pOwnAllocations);
4292  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
4293  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
4294  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4295  pOwnAllocationsBeg,
4296  pOwnAllocationsEnd,
4297  *pAllocation,
4298  VmaPointerLess()) - pOwnAllocationsBeg;
4299  VectorInsert(*pOwnAllocations, indexToInsert, *pAllocation);
4300  }
4301 
4302  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
4303 
4304  return VK_SUCCESS;
4305 }
4306 
4307 VkResult VmaAllocator_T::AllocateMemory(
4308  const VkMemoryRequirements& vkMemReq,
4309  const VmaMemoryRequirements& vmaMemReq,
4310  VmaSuballocationType suballocType,
4311  VmaAllocation* pAllocation)
4312 {
4313  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 &&
4314  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
4315  {
4316  VMA_ASSERT(0 && "Specifying VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT together with VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT makes no sense.");
4317  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4318  }
4319 
4320  // Bit mask of memory Vulkan types acceptable for this allocation.
4321  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
4322  uint32_t memTypeIndex = UINT32_MAX;
4323  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
4324  if(res == VK_SUCCESS)
4325  {
4326  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
4327  // Succeeded on first try.
4328  if(res == VK_SUCCESS)
4329  {
4330  return res;
4331  }
4332  // Allocation from this memory type failed. Try other compatible memory types.
4333  else
4334  {
4335  for(;;)
4336  {
4337  // Remove old memTypeIndex from list of possibilities.
4338  memoryTypeBits &= ~(1u << memTypeIndex);
4339  // Find alternative memTypeIndex.
4340  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
4341  if(res == VK_SUCCESS)
4342  {
4343  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
4344  // Allocation from this alternative memory type succeeded.
4345  if(res == VK_SUCCESS)
4346  {
4347  return res;
4348  }
4349  // else: Allocation from this memory type failed. Try next one - next loop iteration.
4350  }
4351  // No other matching memory type index could be found.
4352  else
4353  {
4354  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
4355  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
4356  }
4357  }
4358  }
4359  }
4360  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
4361  else
4362  return res;
4363 }
4364 
4365 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
4366 {
4367  VMA_ASSERT(allocation);
4368 
4369  if(allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
4370  {
4371  VmaBlock* pBlockToDelete = VMA_NULL;
4372 
4373  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
4374  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
4375  {
4376  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4377 
4378  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4379  VmaBlock* pBlock = allocation->GetBlock();
4380 
4381  pBlock->Free(allocation);
4382  VMA_HEAVY_ASSERT(pBlock->Validate());
4383 
4384  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
4385 
4386  // pBlock became empty after this deallocation.
4387  if(pBlock->IsEmpty())
4388  {
4389  // Already has empty Allocation. We don't want to have two, so delete this one.
4390  if(m_HasEmptyBlock[memTypeIndex])
4391  {
4392  pBlockToDelete = pBlock;
4393  pBlockVector->Remove(pBlock);
4394  }
4395  // We now have first empty Allocation.
4396  else
4397  {
4398  m_HasEmptyBlock[memTypeIndex] = true;
4399  }
4400  }
4401  // Must be called after srcBlockIndex is used, because later it may become invalid!
4402  pBlockVector->IncrementallySortBlocks();
4403  }
4404  // Destruction of a free Allocation. Deferred until this point, outside of mutex
4405  // lock, for performance reason.
4406  if(pBlockToDelete != VMA_NULL)
4407  {
4408  VMA_DEBUG_LOG(" Deleted empty allocation");
4409  pBlockToDelete->Destroy(this);
4410  vma_delete(this, pBlockToDelete);
4411  }
4412 
4413  vma_delete(this, allocation);
4414  }
4415  else // VmaAllocation_T::ALLOCATION_TYPE_OWN
4416  {
4417  FreeOwnMemory(allocation);
4418  }
4419 }
4420 
4421 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
4422 {
4423  InitStatInfo(pStats->total);
4424  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
4425  InitStatInfo(pStats->memoryType[i]);
4426  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
4427  InitStatInfo(pStats->memoryHeap[i]);
4428 
4429  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4430  {
4431  VmaMutexLock allocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4432  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4433  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4434  {
4435  const VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4436  VMA_ASSERT(pBlockVector);
4437  pBlockVector->AddStats(pStats, memTypeIndex, heapIndex);
4438  }
4439  }
4440 
4441  VmaPostprocessCalcStatInfo(pStats->total);
4442  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
4443  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
4444  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
4445  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
4446 }
4447 
4448 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
4449 
4450 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
4451 {
4452  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
4453  {
4454  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
4455  {
4456  for(size_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
4457  {
4458  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
4459  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
4460  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4461  {
4462  // Process OwnAllocations.
4463  {
4464  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4465  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4466  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
4467  {
4468  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
4469  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(m_hDevice);
4470  }
4471  }
4472 
4473  // Process normal Allocations.
4474  {
4475  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4476  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4477  pBlockVector->UnmapPersistentlyMappedMemory();
4478  }
4479  }
4480  }
4481  }
4482  }
4483 }
4484 
4485 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
4486 {
4487  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
4488  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
4489  {
4490  VkResult finalResult = VK_SUCCESS;
4491  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
4492  {
4493  for(size_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
4494  {
4495  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
4496  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
4497  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4498  {
4499  // Process OwnAllocations.
4500  {
4501  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4502  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4503  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
4504  {
4505  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
4506  hAlloc->OwnAllocMapPersistentlyMappedMemory(m_hDevice);
4507  }
4508  }
4509 
4510  // Process normal Allocations.
4511  {
4512  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4513  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
4514  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
4515  if(localResult != VK_SUCCESS)
4516  {
4517  finalResult = localResult;
4518  }
4519  }
4520  }
4521  }
4522  }
4523  return finalResult;
4524  }
4525  else
4526  return VK_SUCCESS;
4527 }
4528 
4529 VkResult VmaAllocator_T::Defragment(
4530  VmaAllocation* pAllocations,
4531  size_t allocationCount,
4532  VkBool32* pAllocationsChanged,
4533  const VmaDefragmentationInfo* pDefragmentationInfo,
4534  VmaDefragmentationStats* pDefragmentationStats)
4535 {
4536  if(pAllocationsChanged != VMA_NULL)
4537  {
4538  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
4539  }
4540  if(pDefragmentationStats != VMA_NULL)
4541  {
4542  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
4543  }
4544 
4545  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
4546  {
4547  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
4548  return VK_ERROR_MEMORY_MAP_FAILED;
4549  }
4550 
4551  // Initialize defragmentators per memory type.
4552  const VkDeviceSize bufferImageGranularity = GetBufferImageGranularity();
4553  VmaDefragmentator* pDefragmentators[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
4554  memset(pDefragmentators, 0, sizeof(pDefragmentators));
4555  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4556  {
4557  // Only HOST_VISIBLE memory types can be defragmented.
4558  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4559  {
4560  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4561  {
4562  pDefragmentators[memTypeIndex][blockVectorType] = vma_new(this, VmaDefragmentator)(
4563  m_hDevice,
4564  GetAllocationCallbacks(),
4565  bufferImageGranularity,
4566  memTypeIndex,
4567  (VMA_BLOCK_VECTOR_TYPE)blockVectorType);
4568  }
4569  }
4570  }
4571 
4572  // Dispatch pAllocations among defragmentators.
4573  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
4574  {
4575  VmaAllocation hAlloc = pAllocations[allocIndex];
4576  VMA_ASSERT(hAlloc);
4577  if(hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
4578  {
4579  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
4580  // Only HOST_VISIBLE memory types can be defragmented.
4581  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4582  {
4583  const VMA_BLOCK_VECTOR_TYPE blockVectorType = hAlloc->GetBlockVectorType();
4584  VkBool32* pChanged = (pAllocationsChanged != VMA_NULL) ?
4585  &pAllocationsChanged[allocIndex] : VMA_NULL;
4586  pDefragmentators[memTypeIndex][blockVectorType]->AddAllocation(hAlloc, pChanged);
4587  }
4588  // else: skip this allocation, cannot move it.
4589  }
4590  // else ALLOCATION_TYPE_OWN: skip this allocation, nothing to defragment.
4591  }
4592 
4593  VkResult result = VK_SUCCESS;
4594 
4595  // Main processing.
4596  VkDeviceSize maxBytesToMove = SIZE_MAX;
4597  uint32_t maxAllocationsToMove = UINT32_MAX;
4598  if(pDefragmentationInfo != VMA_NULL)
4599  {
4600  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
4601  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
4602  }
4603  for(uint32_t memTypeIndex = 0;
4604  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
4605  ++memTypeIndex)
4606  {
4607  // Only HOST_VISIBLE memory types can be defragmented.
4608  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4609  {
4610  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4611 
4612  for(uint32_t blockVectorType = 0;
4613  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
4614  ++blockVectorType)
4615  {
4616  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
4617 
4618  // Defragment.
4619  result = pDefragmentators[memTypeIndex][blockVectorType]->Defragment(pBlockVector, maxBytesToMove, maxAllocationsToMove);
4620 
4621  // Accumulate statistics.
4622  if(pDefragmentationStats != VMA_NULL)
4623  {
4624  const VkDeviceSize bytesMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetBytesMoved();
4625  const uint32_t allocationsMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetAllocationsMoved();
4626  pDefragmentationStats->bytesMoved += bytesMoved;
4627  pDefragmentationStats->allocationsMoved += allocationsMoved;
4628  VMA_ASSERT(bytesMoved <= maxBytesToMove);
4629  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
4630  maxBytesToMove -= bytesMoved;
4631  maxAllocationsToMove -= allocationsMoved;
4632  }
4633 
4634  // Free empty blocks.
4635  for(size_t blockIndex = pBlockVector->m_Blocks.size(); blockIndex--; )
4636  {
4637  VmaBlock* pBlock = pBlockVector->m_Blocks[blockIndex];
4638  if(pBlock->IsEmpty())
4639  {
4640  if(pDefragmentationStats != VMA_NULL)
4641  {
4642  ++pDefragmentationStats->deviceMemoryBlocksFreed;
4643  pDefragmentationStats->bytesFreed += pBlock->m_Size;
4644  }
4645 
4646  VectorRemove(pBlockVector->m_Blocks, blockIndex);
4647  pBlock->Destroy(this);
4648  vma_delete(this, pBlock);
4649  }
4650  }
4651 
4652  // All block vector types processed: we can be sure that all empty allocations have been freed.
4653  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_COUNT - 1)
4654  {
4655  m_HasEmptyBlock[memTypeIndex] = false;
4656  }
4657  }
4658  }
4659  }
4660 
4661  // Destroy defragmentators.
4662  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
4663  {
4664  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
4665  {
4666  vma_delete(this, pDefragmentators[memTypeIndex][blockVectorType]);
4667  }
4668  }
4669 
4670  return result;
4671 }
4672 
4673 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
4674 {
4675  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
4676  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
4677  pAllocationInfo->offset = hAllocation->GetOffset();
4678  pAllocationInfo->size = hAllocation->GetSize();
4679  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
4680  pAllocationInfo->pUserData = hAllocation->GetUserData();
4681 }
4682 
4683 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
4684 {
4685  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
4686 
4687  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
4688  {
4689  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4690  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
4691  VMA_ASSERT(pOwnAllocations);
4692  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
4693  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
4694  VmaAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess(
4695  pOwnAllocationsBeg,
4696  pOwnAllocationsEnd,
4697  allocation,
4698  VmaPointerLess());
4699  if(pOwnAllocationIt != pOwnAllocationsEnd)
4700  {
4701  const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg;
4702  VectorRemove(*pOwnAllocations, ownAllocationIndex);
4703  }
4704  else
4705  {
4706  VMA_ASSERT(0);
4707  }
4708  }
4709 
4710  VkDeviceMemory hMemory = allocation->GetMemory();
4711 
4712  // Callback.
4713  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
4714  {
4715  (*m_DeviceMemoryCallbacks.pfnFree)(this, memTypeIndex, hMemory, allocation->GetSize());
4716  }
4717 
4718  if(allocation->GetMappedData() != VMA_NULL)
4719  {
4720  vkUnmapMemory(m_hDevice, hMemory);
4721  }
4722 
4723  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
4724 
4725  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
4726 
4727  vma_delete(this, allocation);
4728 }
4729 
4730 #if VMA_STATS_STRING_ENABLED
4731 
4732 void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb)
4733 {
4734  bool ownAllocationsStarted = false;
4735  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4736  {
4737  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
4738  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4739  {
4740  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
4741  VMA_ASSERT(pOwnAllocVector);
4742  if(pOwnAllocVector->empty() == false)
4743  {
4744  if(ownAllocationsStarted)
4745  {
4746  sb.Add(",\n\t\"Type ");
4747  }
4748  else
4749  {
4750  sb.Add(",\n\"OwnAllocations\": {\n\t\"Type ");
4751  ownAllocationsStarted = true;
4752  }
4753  sb.AddNumber(memTypeIndex);
4754  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
4755  {
4756  sb.Add(" Mapped");
4757  }
4758  sb.Add("\": [");
4759 
4760  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
4761  {
4762  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
4763  if(i > 0)
4764  {
4765  sb.Add(",\n\t\t{ \"Size\": ");
4766  }
4767  else
4768  {
4769  sb.Add("\n\t\t{ \"Size\": ");
4770  }
4771  sb.AddNumber(hAlloc->GetSize());
4772  sb.Add(", \"Type\": ");
4773  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
4774  sb.Add(" }");
4775  }
4776 
4777  sb.Add("\n\t]");
4778  }
4779  }
4780  }
4781  if(ownAllocationsStarted)
4782  {
4783  sb.Add("\n}");
4784  }
4785 
4786  {
4787  bool allocationsStarted = false;
4788  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
4789  {
4790  VmaMutexLock globalAllocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
4791  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
4792  {
4793  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
4794  {
4795  if(allocationsStarted)
4796  {
4797  sb.Add(",\n\t\"Type ");
4798  }
4799  else
4800  {
4801  sb.Add(",\n\"Allocations\": {\n\t\"Type ");
4802  allocationsStarted = true;
4803  }
4804  sb.AddNumber(memTypeIndex);
4805  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
4806  {
4807  sb.Add(" Mapped");
4808  }
4809  sb.Add("\": [");
4810 
4811  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(sb);
4812 
4813  sb.Add("\n\t]");
4814  }
4815  }
4816  }
4817  if(allocationsStarted)
4818  {
4819  sb.Add("\n}");
4820  }
4821  }
4822 }
4823 
4824 #endif // #if VMA_STATS_STRING_ENABLED
4825 
4826 static VkResult AllocateMemoryForImage(
4827  VmaAllocator allocator,
4828  VkImage image,
4829  const VmaMemoryRequirements* pMemoryRequirements,
4830  VmaSuballocationType suballocType,
4831  VmaAllocation* pAllocation)
4832 {
4833  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pMemoryRequirements && pAllocation);
4834 
4835  VkMemoryRequirements vkMemReq = {};
4836  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
4837 
4838  return allocator->AllocateMemory(
4839  vkMemReq,
4840  *pMemoryRequirements,
4841  suballocType,
4842  pAllocation);
4843 }
4844 
4846 // Public interface
4847 
4848 VkResult vmaCreateAllocator(
4849  const VmaAllocatorCreateInfo* pCreateInfo,
4850  VmaAllocator* pAllocator)
4851 {
4852  VMA_ASSERT(pCreateInfo && pAllocator);
4853  VMA_DEBUG_LOG("vmaCreateAllocator");
4854  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
4855  return VK_SUCCESS;
4856 }
4857 
4858 void vmaDestroyAllocator(
4859  VmaAllocator allocator)
4860 {
4861  if(allocator != VK_NULL_HANDLE)
4862  {
4863  VMA_DEBUG_LOG("vmaDestroyAllocator");
4864  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
4865  vma_delete(&allocationCallbacks, allocator);
4866  }
4867 }
4868 
4869 void vmaGetPhysicalDeviceProperties(
4870  VmaAllocator allocator,
4871  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
4872 {
4873  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
4874  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
4875 }
4876 
4877 void vmaGetMemoryProperties(
4878  VmaAllocator allocator,
4879  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
4880 {
4881  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
4882  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
4883 }
4884 
4885 void vmaGetMemoryTypeProperties(
4886  VmaAllocator allocator,
4887  uint32_t memoryTypeIndex,
4888  VkMemoryPropertyFlags* pFlags)
4889 {
4890  VMA_ASSERT(allocator && pFlags);
4891  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
4892  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
4893 }
4894 
4895 void vmaCalculateStats(
4896  VmaAllocator allocator,
4897  VmaStats* pStats)
4898 {
4899  VMA_ASSERT(allocator && pStats);
4900  VMA_DEBUG_GLOBAL_MUTEX_LOCK
4901  allocator->CalculateStats(pStats);
4902 }
4903 
4904 #if VMA_STATS_STRING_ENABLED
4905 
4906 void vmaBuildStatsString(
4907  VmaAllocator allocator,
4908  char** ppStatsString,
4909  VkBool32 detailedMap)
4910 {
4911  VMA_ASSERT(allocator && ppStatsString);
4912  VMA_DEBUG_GLOBAL_MUTEX_LOCK
4913 
4914  VmaStringBuilder sb(allocator);
4915  {
4916  VmaStats stats;
4917  allocator->CalculateStats(&stats);
4918 
4919  sb.Add("{\n\"Total\": ");
4920  VmaPrintStatInfo(sb, stats.total);
4921 
4922  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
4923  {
4924  sb.Add(",\n\"Heap ");
4925  sb.AddNumber(heapIndex);
4926  sb.Add("\": {\n\t\"Size\": ");
4927  sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
4928  sb.Add(",\n\t\"Flags\": ");
4929  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
4930  {
4931  sb.AddString("DEVICE_LOCAL");
4932  }
4933  else
4934  {
4935  sb.AddString("");
4936  }
4937  if(stats.memoryHeap[heapIndex].AllocationCount > 0)
4938  {
4939  sb.Add(",\n\t\"Stats:\": ");
4940  VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]);
4941  }
4942 
4943  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
4944  {
4945  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
4946  {
4947  sb.Add(",\n\t\"Type ");
4948  sb.AddNumber(typeIndex);
4949  sb.Add("\": {\n\t\t\"Flags\": \"");
4950  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
4951  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
4952  {
4953  sb.Add(" DEVICE_LOCAL");
4954  }
4955  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
4956  {
4957  sb.Add(" HOST_VISIBLE");
4958  }
4959  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
4960  {
4961  sb.Add(" HOST_COHERENT");
4962  }
4963  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
4964  {
4965  sb.Add(" HOST_CACHED");
4966  }
4967  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
4968  {
4969  sb.Add(" LAZILY_ALLOCATED");
4970  }
4971  sb.Add("\"");
4972  if(stats.memoryType[typeIndex].AllocationCount > 0)
4973  {
4974  sb.Add(",\n\t\t\"Stats\": ");
4975  VmaPrintStatInfo(sb, stats.memoryType[typeIndex]);
4976  }
4977  sb.Add("\n\t}");
4978  }
4979  }
4980  sb.Add("\n}");
4981  }
4982  if(detailedMap == VK_TRUE)
4983  {
4984  allocator->PrintDetailedMap(sb);
4985  }
4986  sb.Add("\n}\n");
4987  }
4988 
4989  const size_t len = sb.GetLength();
4990  char* const pChars = vma_new_array(allocator, char, len + 1);
4991  if(len > 0)
4992  {
4993  memcpy(pChars, sb.GetData(), len);
4994  }
4995  pChars[len] = '\0';
4996  *ppStatsString = pChars;
4997 }
4998 
4999 void vmaFreeStatsString(
5000  VmaAllocator allocator,
5001  char* pStatsString)
5002 {
5003  if(pStatsString != VMA_NULL)
5004  {
5005  VMA_ASSERT(allocator);
5006  size_t len = strlen(pStatsString);
5007  vma_delete_array(allocator, pStatsString, len + 1);
5008  }
5009 }
5010 
5011 #endif // #if VMA_STATS_STRING_ENABLED
5012 
5015 VkResult vmaFindMemoryTypeIndex(
5016  VmaAllocator allocator,
5017  uint32_t memoryTypeBits,
5018  const VmaMemoryRequirements* pMemoryRequirements,
5019  uint32_t* pMemoryTypeIndex)
5020 {
5021  VMA_ASSERT(allocator != VK_NULL_HANDLE);
5022  VMA_ASSERT(pMemoryRequirements != VMA_NULL);
5023  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
5024 
5025  uint32_t requiredFlags = pMemoryRequirements->requiredFlags;
5026  uint32_t preferredFlags = pMemoryRequirements->preferredFlags;
5027  if(preferredFlags == 0)
5028  {
5029  preferredFlags = requiredFlags;
5030  }
5031  // preferredFlags, if not 0, must be a superset of requiredFlags.
5032  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
5033 
5034  // Convert usage to requiredFlags and preferredFlags.
5035  switch(pMemoryRequirements->usage)
5036  {
5037  case VMA_MEMORY_USAGE_UNKNOWN:
5038  break;
5039  case VMA_MEMORY_USAGE_GPU_ONLY:
5040  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
5041  break;
5042  case VMA_MEMORY_USAGE_CPU_ONLY:
5043  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
5044  break;
5045  case VMA_MEMORY_USAGE_CPU_TO_GPU:
5046  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5047  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
5048  break;
5049  case VMA_MEMORY_USAGE_GPU_TO_CPU:
5050  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5051  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
5052  break;
5053  default:
5054  break;
5055  }
5056 
5057  if((pMemoryRequirements->flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0)
5058  {
5059  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5060  }
5061 
5062  *pMemoryTypeIndex = UINT32_MAX;
5063  uint32_t minCost = UINT32_MAX;
5064  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
5065  memTypeIndex < allocator->GetMemoryTypeCount();
5066  ++memTypeIndex, memTypeBit <<= 1)
5067  {
5068  // This memory type is acceptable according to memoryTypeBits bitmask.
5069  if((memTypeBit & memoryTypeBits) != 0)
5070  {
5071  const VkMemoryPropertyFlags currFlags =
5072  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
5073  // This memory type contains requiredFlags.
5074  if((requiredFlags & ~currFlags) == 0)
5075  {
5076  // Calculate cost as number of bits from preferredFlags not present in this memory type.
5077  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
5078  // Remember memory type with lowest cost.
5079  if(currCost < minCost)
5080  {
5081  *pMemoryTypeIndex = memTypeIndex;
5082  if(currCost == 0)
5083  {
5084  return VK_SUCCESS;
5085  }
5086  minCost = currCost;
5087  }
5088  }
5089  }
5090  }
5091  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
5092 }
5093 
5094 VkResult vmaAllocateMemory(
5095  VmaAllocator allocator,
5096  const VkMemoryRequirements* pVkMemoryRequirements,
5097  const VmaMemoryRequirements* pVmaMemoryRequirements,
5098  VmaAllocation* pAllocation,
5099  VmaAllocationInfo* pAllocationInfo)
5100 {
5101  VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pAllocation);
5102 
5103  VMA_DEBUG_LOG("vmaAllocateMemory");
5104 
5105  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5106 
5107  return allocator->AllocateMemory(
5108  *pVkMemoryRequirements,
5109  *pVmaMemoryRequirements,
5110  VMA_SUBALLOCATION_TYPE_UNKNOWN,
5111  pAllocation);
5112 
5113  if(pAllocationInfo)
5114  {
5115  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5116  }
5117 }
5118 
5119 VkResult vmaAllocateMemoryForBuffer(
5120  VmaAllocator allocator,
5121  VkBuffer buffer,
5122  const VmaMemoryRequirements* pMemoryRequirements,
5123  VmaAllocation* pAllocation,
5124  VmaAllocationInfo* pAllocationInfo)
5125 {
5126  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
5127 
5128  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
5129 
5130  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5131 
5132  VkMemoryRequirements vkMemReq = {};
5133  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
5134 
5135  return allocator->AllocateMemory(
5136  vkMemReq,
5137  *pMemoryRequirements,
5138  VMA_SUBALLOCATION_TYPE_BUFFER,
5139  pAllocation);
5140 
5141  if(pAllocationInfo)
5142  {
5143  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5144  }
5145 }
5146 
5147 VkResult vmaAllocateMemoryForImage(
5148  VmaAllocator allocator,
5149  VkImage image,
5150  const VmaMemoryRequirements* pMemoryRequirements,
5151  VmaAllocation* pAllocation,
5152  VmaAllocationInfo* pAllocationInfo)
5153 {
5154  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
5155 
5156  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
5157 
5158  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5159 
5160  return AllocateMemoryForImage(
5161  allocator,
5162  image,
5163  pMemoryRequirements,
5164  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
5165  pAllocation);
5166 
5167  if(pAllocationInfo)
5168  {
5169  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5170  }
5171 }
5172 
5173 void vmaFreeMemory(
5174  VmaAllocator allocator,
5175  VmaAllocation allocation)
5176 {
5177  VMA_ASSERT(allocator && allocation);
5178 
5179  VMA_DEBUG_LOG("vmaFreeMemory");
5180 
5181  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5182 
5183  allocator->FreeMemory(allocation);
5184 }
5185 
5186 void vmaGetAllocationInfo(
5187  VmaAllocator allocator,
5188  VmaAllocation allocation,
5189  VmaAllocationInfo* pAllocationInfo)
5190 {
5191  VMA_ASSERT(allocator && allocation && pAllocationInfo);
5192 
5193  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5194 
5195  allocator->GetAllocationInfo(allocation, pAllocationInfo);
5196 }
5197 
5198 void vmaSetAllocationUserData(
5199  VmaAllocator allocator,
5200  VmaAllocation allocation,
5201  void* pUserData)
5202 {
5203  VMA_ASSERT(allocator && allocation);
5204 
5205  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5206 
5207  allocation->SetUserData(pUserData);
5208 }
5209 
5210 VkResult vmaMapMemory(
5211  VmaAllocator allocator,
5212  VmaAllocation allocation,
5213  void** ppData)
5214 {
5215  VMA_ASSERT(allocator && allocation && ppData);
5216 
5217  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5218 
5219  return vkMapMemory(allocator->m_hDevice, allocation->GetMemory(),
5220  allocation->GetOffset(), allocation->GetSize(), 0, ppData);
5221 }
5222 
5223 void vmaUnmapMemory(
5224  VmaAllocator allocator,
5225  VmaAllocation allocation)
5226 {
5227  VMA_ASSERT(allocator && allocation);
5228 
5229  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5230 
5231  vkUnmapMemory(allocator->m_hDevice, allocation->GetMemory());
5232 }
5233 
5234 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
5235 {
5236  VMA_ASSERT(allocator);
5237 
5238  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5239 
5240  allocator->UnmapPersistentlyMappedMemory();
5241 }
5242 
5243 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
5244 {
5245  VMA_ASSERT(allocator);
5246 
5247  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5248 
5249  return allocator->MapPersistentlyMappedMemory();
5250 }
5251 
5252 VkResult vmaDefragment(
5253  VmaAllocator allocator,
5254  VmaAllocation* pAllocations,
5255  size_t allocationCount,
5256  VkBool32* pAllocationsChanged,
5257  const VmaDefragmentationInfo *pDefragmentationInfo,
5258  VmaDefragmentationStats* pDefragmentationStats)
5259 {
5260  VMA_ASSERT(allocator && pAllocations);
5261 
5262  VMA_DEBUG_LOG("vmaDefragment");
5263 
5264  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5265 
5266  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
5267 }
5268 
5269 VkResult vmaCreateBuffer(
5270  VmaAllocator allocator,
5271  const VkBufferCreateInfo* pCreateInfo,
5272  const VmaMemoryRequirements* pMemoryRequirements,
5273  VkBuffer* pBuffer,
5274  VmaAllocation* pAllocation,
5275  VmaAllocationInfo* pAllocationInfo)
5276 {
5277  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pBuffer && pAllocation);
5278 
5279  VMA_DEBUG_LOG("vmaCreateBuffer");
5280 
5281  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5282 
5283  // 1. Create VkBuffer.
5284  VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
5285  if(res >= 0)
5286  {
5287  // 2. vkGetBufferMemoryRequirements.
5288  VkMemoryRequirements vkMemReq = {};
5289  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
5290 
5291  // 3. Allocate memory using allocator.
5292  res = allocator->AllocateMemory(
5293  vkMemReq,
5294  *pMemoryRequirements,
5295  VMA_SUBALLOCATION_TYPE_BUFFER,
5296  pAllocation);
5297  if(res >= 0)
5298  {
5299  // 3. Bind buffer with memory.
5300  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
5301  if(res >= 0)
5302  {
5303  // All steps succeeded.
5304  if(pAllocationInfo != VMA_NULL)
5305  {
5306  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5307  }
5308  return VK_SUCCESS;
5309  }
5310  allocator->FreeMemory(*pAllocation);
5311  return res;
5312  }
5313  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
5314  return res;
5315  }
5316  return res;
5317 }
5318 
5319 void vmaDestroyBuffer(
5320  VmaAllocator allocator,
5321  VkBuffer buffer,
5322  VmaAllocation allocation)
5323 {
5324  if(buffer != VK_NULL_HANDLE)
5325  {
5326  VMA_ASSERT(allocator);
5327 
5328  VMA_DEBUG_LOG("vmaDestroyBuffer");
5329 
5330  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5331 
5332  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
5333 
5334  allocator->FreeMemory(allocation);
5335  }
5336 }
5337 
5338 VkResult vmaCreateImage(
5339  VmaAllocator allocator,
5340  const VkImageCreateInfo* pCreateInfo,
5341  const VmaMemoryRequirements* pMemoryRequirements,
5342  VkImage* pImage,
5343  VmaAllocation* pAllocation,
5344  VmaAllocationInfo* pAllocationInfo)
5345 {
5346  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pImage && pAllocation);
5347 
5348  VMA_DEBUG_LOG("vmaCreateImage");
5349 
5350  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5351 
5352  // 1. Create VkImage.
5353  VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage);
5354  if(res >= 0)
5355  {
5356  VkMappedMemoryRange mem = {};
5357  VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
5358  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
5359  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
5360 
5361  // 2. Allocate memory using allocator.
5362  res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, pAllocation);
5363  if(res >= 0)
5364  {
5365  // 3. Bind image with memory.
5366  res = vkBindImageMemory(allocator->m_hDevice, *pImage, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
5367  if(res >= 0)
5368  {
5369  // All steps succeeded.
5370  if(pAllocationInfo != VMA_NULL)
5371  {
5372  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
5373  }
5374  return VK_SUCCESS;
5375  }
5376  allocator->FreeMemory(*pAllocation);
5377  return res;
5378  }
5379  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
5380  return res;
5381  }
5382  return res;
5383 }
5384 
5385 void vmaDestroyImage(
5386  VmaAllocator allocator,
5387  VkImage image,
5388  VmaAllocation allocation)
5389 {
5390  if(image != VK_NULL_HANDLE)
5391  {
5392  VMA_ASSERT(allocator);
5393 
5394  VMA_DEBUG_LOG("vmaDestroyImage");
5395 
5396  VMA_DEBUG_GLOBAL_MUTEX_LOCK
5397 
5398  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
5399 
5400  allocator->FreeMemory(allocation);
5401  }
5402 }
5403 
5404 #endif // #ifdef VMA_IMPLEMENTATION
VmaMemoryRequirementFlagBits
VmaMemoryRequirementFlagBits
Flags to be passed as VmaMemoryRequirements::flags.
Definition: vk_mem_alloc.h:336
VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:345
VmaMemoryRequirements
struct VmaMemoryRequirements VmaMemoryRequirements
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:214
@@ -70,7 +70,7 @@ $(function() {
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for writing on device and readback on host.
Definition: vk_mem_alloc.h:331
VmaMemoryRequirements::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:374
VmaAllocationInfo::deviceMemory
VkDeviceMemory deviceMemory
Handle to Vulkan memory object.
Definition: vk_mem_alloc.h:431
-
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:561
+
VmaDefragmentationInfo::maxAllocationsToMove
uint32_t maxAllocationsToMove
Maximum number of allocations that can be moved to different place.
Definition: vk_mem_alloc.h:567
vmaGetAllocationInfo
void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
Returns current information about specified allocation.
vmaUnmapPersistentlyMappedMemory
void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
@@ -82,23 +82,23 @@ $(function() {
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Size of a single memory block to allocate for resources from a small heap <= 512 MB.
Definition: vk_mem_alloc.h:223
VmaMemoryRequirements::flags
VmaMemoryRequirementFlags flags
Definition: vk_mem_alloc.h:369
VmaAllocatorFlags
VkFlags VmaAllocatorFlags
Definition: vk_mem_alloc.h:205
-
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:565
+
VmaDefragmentationStats
Statistics returned by function vmaDefragment().
Definition: vk_mem_alloc.h:571
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:284
-
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:573
-
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:556
+
VmaDefragmentationStats::deviceMemoryBlocksFreed
uint32_t deviceMemoryBlocksFreed
Number of empty VkDeviceMemory objects that have been released to the system.
Definition: vk_mem_alloc.h:579
+
VmaDefragmentationInfo::maxBytesToMove
VkDeviceSize maxBytesToMove
Maximum total numbers of bytes that can be copied while moving allocations to different places...
Definition: vk_mem_alloc.h:562
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:217
VmaAllocationInfo::size
VkDeviceSize size
Size of this allocation, in bytes.
Definition: vk_mem_alloc.h:441
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:354
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
-
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:551
-
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:569
+
VmaDefragmentationInfo
Optional configuration parameters to be passed to function vmaDefragment().
Definition: vk_mem_alloc.h:557
+
VmaDefragmentationStats::bytesFreed
VkDeviceSize bytesFreed
Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
Definition: vk_mem_alloc.h:575
VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM
Definition: vk_mem_alloc.h:363
VmaMemoryRequirements::pUserData
void * pUserData
Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
Definition: vk_mem_alloc.h:385
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:280
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
VmaMemoryRequirements::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:383
-
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:571
+
VmaDefragmentationStats::allocationsMoved
uint32_t allocationsMoved
Number of allocations that have been moved to different places.
Definition: vk_mem_alloc.h:577
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:317
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
Definition: vk_mem_alloc.h:201
@@ -134,7 +134,7 @@ $(function() {
VmaStatInfo::SuballocationCount
uint32_t SuballocationCount
Definition: vk_mem_alloc.h:271
VmaStatInfo::UnusedRangeSizeAvg
VkDeviceSize UnusedRangeSizeAvg
Definition: vk_mem_alloc.h:276
VmaAllocationInfo::offset
VkDeviceSize offset
Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
Definition: vk_mem_alloc.h:436
-
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:567
+
VmaDefragmentationStats::bytesMoved
VkDeviceSize bytesMoved
Total number of bytes that have been copied while moving allocations to different places...
Definition: vk_mem_alloc.h:573
vmaDefragment
VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
Compacts memory by moving allocations.
VmaDeviceMemoryCallbacks
struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
VmaStatInfo::SuballocationSizeMin
VkDeviceSize SuballocationSizeMin
Definition: vk_mem_alloc.h:275
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 5ee5cbf..cb343c4 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -455,7 +455,7 @@ typedef struct VmaAllocationInfo { /** \brief General purpose memory allocation. @param[out] pAllocation Handle to allocated memory. -@param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function VmaGetAllocationInfo(). +@param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo(). You should free the memory using vmaFreeMemory(). @@ -471,7 +471,7 @@ VkResult vmaAllocateMemory( /** @param[out] pAllocation Handle to allocated memory. -@param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function VmaGetAllocationInfo(). +@param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo(). You should free the memory using vmaFreeMemory(). */ @@ -526,9 +526,15 @@ void vmaUnmapMemory( /** \brief Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. -This is optional performance optimization. You should call it on Windows for -time of call to vkQueueSubmit and vkQueuePresent, for performance reasons, -because of the internal behavior of WDDM. +This is optional performance optimization. On Windows you should call it before +every call to vkQueueSubmit and vkQueuePresent. After which you can remap the +allocations again using vmaMapPersistentlyMappedMemory(). This is because of the +internal behavior of WDDM. Example: + + + vmaUnmapPersistentlyMappedMemory(allocator); + vkQueueSubmit(...) + vmaMapPersistentlyMappedMemory(allocator); After this call VmaAllocationInfo::pMappedData of some allocations may become null.