From 8bc1a7df839c59326ff7da8047482ea092babd26 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 4 Jul 2017 15:22:57 +0200 Subject: [PATCH] Changed "CONFIGURATION SECTION" to contain #ifndef so you can define these macros before including this header, not necessarily change them in the file. --- docs/html/index.html | 3 +- docs/html/vk__mem__alloc_8h_source.html | 72 +++--- src/vk_mem_alloc.h | 291 ++++++++++++++---------- 3 files changed, 207 insertions(+), 159 deletions(-) diff --git a/docs/html/index.html b/docs/html/index.html index e65d94c..bd719db 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -137,8 +137,7 @@ vmaCreateBuffer(allocator, &bufferInfo, &memReq, &buffer, nullptr, n 
vmaDestroyBuffer(allocator, buffer);

Configuration

-

Set VMA_STATS_STRING_ENABLED macro in vk_mem_alloc.h to 0 or 1 to disable/enable compilation of code for dumping internal allocator state to string in JSON format.

-

Please check "CONFIGURATION" section in vk_mem_alloc.cpp file to find macros and other definitions that you can change to connect the library to your own implementation of basic facilities like assert, min and max functions, mutex etc. C++ STL is used by default, but changing these allows you to get rid of any STL usage if you want, as many game developers tend to do.

+

Please check "CONFIGURATION SECTION" in the code to find macros that you can define before each #include of this file or change directly in this file to provide your own implementation of basic facilities like assert, min and max functions, mutex etc. C++ STL is used by default, but changing these allows you to get rid of any STL usage if you want, as many game developers tend to do.

Custom memory allocator

You can use custom memory allocator by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These functions will be passed to Vulkan, as well as used by the library itself to make any CPU-side allocations.

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index de40d22..02af352 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,63 +62,63 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
155 #include <vulkan/vulkan.h>
156 
158 
162 VK_DEFINE_HANDLE(VmaAllocator)
163 
164 typedef struct VmaAllocatorCreateInfo
166 {
168 
169  VkPhysicalDevice physicalDevice;
171 
172  VkDevice device;
174 
175  VkDeviceSize preferredLargeHeapBlockSize;
177 
178  VkDeviceSize preferredSmallHeapBlockSize;
180 
181  const VkAllocationCallbacks* pAllocationCallbacks;
182 } VmaAllocatorCreateInfo;
183 
185 VkResult vmaCreateAllocator(
186  const VmaAllocatorCreateInfo* pCreateInfo,
187  VmaAllocator* pAllocator);
188 
190 void vmaDestroyAllocator(
191  VmaAllocator allocator);
192 
197 void vmaGetPhysicalDeviceProperties(
198  VmaAllocator allocator,
199  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
200 
205 void vmaGetMemoryProperties(
206  VmaAllocator allocator,
207  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
208 
215 void vmaGetMemoryTypeProperties(
216  VmaAllocator allocator,
217  uint32_t memoryTypeIndex,
218  VkMemoryPropertyFlags* pFlags);
219 
220 typedef struct VmaStatInfo
221 {
222  uint32_t AllocationCount;
223  uint32_t SuballocationCount;
224  uint32_t UnusedRangeCount;
225  VkDeviceSize UsedBytes;
226  VkDeviceSize UnusedBytes;
227  VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax;
228  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
229 } VmaStatInfo;
230 
232 struct VmaStats
233 {
234  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
235  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
236  VmaStatInfo total;
237 };
238 
240 void vmaCalculateStats(
241  VmaAllocator allocator,
242  VmaStats* pStats);
243 
244 #define VMA_STATS_STRING_ENABLED 1
245 
246 #if VMA_STATS_STRING_ENABLED
247 
249 
251 void vmaBuildStatsString(
252  VmaAllocator allocator,
253  char** ppStatsString,
254  VkBool32 detailedMap);
255 
256 void vmaFreeStatsString(
257  VmaAllocator allocator,
258  char* pStatsString);
259 
260 #endif // #if VMA_STATS_STRING_ENABLED
261 
264 
269 typedef enum VmaMemoryUsage
270 {
272  VMA_MEMORY_USAGE_UNKNOWN = 0,
274  VMA_MEMORY_USAGE_GPU_ONLY = 1,
276  VMA_MEMORY_USAGE_CPU_ONLY = 2,
278  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
280  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
281  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
282 } VmaMemoryUsage;
283 
284 typedef struct VmaMemoryRequirements
285 {
294  VkBool32 ownMemory;
299  VmaMemoryUsage usage;
303  VkMemoryPropertyFlags requiredFlags;
308  VkMemoryPropertyFlags preferredFlags;
315  VkBool32 neverAllocate;
316 } VmaMemoryRequirements;
317 
332 VkResult vmaFindMemoryTypeIndex(
333  VmaAllocator allocator,
334  uint32_t memoryTypeBits,
335  const VmaMemoryRequirements* pMemoryRequirements,
336  uint32_t* pMemoryTypeIndex);
337 
340 
357 VkResult vmaAllocateMemory(
358  VmaAllocator allocator,
359  const VkMemoryRequirements* pVkMemoryRequirements,
360  const VmaMemoryRequirements* pVmaMemoryRequirements,
361  VkMappedMemoryRange* pMemory,
362  uint32_t* pMemoryTypeIndex);
363 
371 VkResult vmaAllocateMemoryForBuffer(
372  VmaAllocator allocator,
373  VkBuffer buffer,
374  const VmaMemoryRequirements* pMemoryRequirements,
375  VkMappedMemoryRange* pMemory,
376  uint32_t* pMemoryTypeIndex);
377 
379 VkResult vmaAllocateMemoryForImage(
380  VmaAllocator allocator,
381  VkImage image,
382  const VmaMemoryRequirements* pMemoryRequirements,
383  VkMappedMemoryRange* pMemory,
384  uint32_t* pMemoryTypeIndex);
385 
387 void vmaFreeMemory(
388  VmaAllocator allocator,
389  const VkMappedMemoryRange* pMemory);
390 
396 VkResult vmaMapMemory(
397  VmaAllocator allocator,
398  const VkMappedMemoryRange* pMemory,
399  void** ppData);
400 
401 void vmaUnmapMemory(
402  VmaAllocator allocator,
403  const VkMappedMemoryRange* pMemory);
404 
407 
429 VkResult vmaCreateBuffer(
430  VmaAllocator allocator,
431  const VkBufferCreateInfo* pCreateInfo,
432  const VmaMemoryRequirements* pMemoryRequirements,
433  VkBuffer* pBuffer,
434  VkMappedMemoryRange* pMemory,
435  uint32_t* pMemoryTypeIndex);
436 
437 void vmaDestroyBuffer(
438  VmaAllocator allocator,
439  VkBuffer buffer);
440 
442 VkResult vmaCreateImage(
443  VmaAllocator allocator,
444  const VkImageCreateInfo* pCreateInfo,
445  const VmaMemoryRequirements* pMemoryRequirements,
446  VkImage* pImage,
447  VkMappedMemoryRange* pMemory,
448  uint32_t* pMemoryTypeIndex);
449 
450 void vmaDestroyImage(
451  VmaAllocator allocator,
452  VkImage image);
453 
456 #ifdef VMA_IMPLEMENTATION
457 
458 #include <cstdlib>
459 
460 /*******************************************************************************
461 CONFIGURATION
462 
463 Change these definitions depending on your environment.
464 */
465 
466 #define VMA_USE_STL_CONTAINERS 0
467 
468 /* Set this macro to 1 to make the library including and using STL containers:
469 std::pair, std::vector, std::list, std::unordered_map.
470 
471 Set it to 0 or undefined to make the library using its own implementation of
472 the containers.
473 */
474 #if VMA_USE_STL_CONTAINERS
475 #define VMA_USE_STL_VECTOR 1
476 #define VMA_USE_STL_UNORDERED_MAP 1
477 #define VMA_USE_STL_LIST 1
478 #endif
479 
480 #if VMA_USE_STL_VECTOR
481 #include <vector>
482 #endif
483 
484 #if VMA_USE_STL_UNORDERED_MAP
485 #include <unordered_map>
486 #endif
487 
488 #if VMA_USE_STL_LIST
489 #include <list>
490 #endif
491 
492 /*
493 Following headers are used in this CONFIGURATION section only, so feel free to
494 remove them if not needed.
495 */
496 #include <cassert> // for assert
497 #include <algorithm> // for min, max
498 #include <mutex> // for std::mutex
499 
500 #ifdef _DEBUG
501  // Normal assert to check for programmer's errors, especially in Debug configuration.
502  #define VMA_ASSERT(expr) assert(expr)
503  // Assert that will be called very often, like inside data structures e.g. operator[].
504  // Making it non-empty can make program slow.
505  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
506 #else
507  #define VMA_ASSERT(expr)
508  #define VMA_HEAVY_ASSERT(expr)
509 #endif
510 
511 // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
512 #define VMA_NULL nullptr
513 
514 #define VMA_ALIGN_OF(type) (__alignof(type))
515 #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
516 #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
517 
518 #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
519 #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
520 #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
521 
522 #define VMA_DEBUG_LOG(format, ...)
523 /*
524 #define VMA_DEBUG_LOG(format, ...) do { \
525  printf(format, __VA_ARGS__); \
526  printf("\n"); \
527 } while(false)
528 */
529 
530 #if VMA_STATS_STRING_ENABLED
531 
532 static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
533 {
534  _ultoa_s(num, outStr, strLen, 10);
535 }
536 static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
537 {
538  _ui64toa_s(num, outStr, strLen, 10);
539 }
540 
541 #endif // #if VMA_STATS_STRING_ENABLED
542 
543 class VmaMutex
544 {
545 public:
546  VmaMutex() { }
547  ~VmaMutex() { }
548  void Lock() { m_Mutex.lock(); }
549  void Unlock() { m_Mutex.unlock(); }
550 private:
551  std::mutex m_Mutex;
552 };
553 
554 /*
555 Main parameter for function assessing how good is a free suballocation for a new
556 allocation request.
557 
558 - Set to true to use Best-Fit algorithm - prefer smaller blocks, as close to the
559  size of requested allocations as possible.
560 - Set to false to use Worst-Fit algorithm - prefer larger blocks, as large as
561  possible.
562 
563 Experiments in special testing environment showed that Best-Fit algorithm is
564 better.
565 */
566 static const bool VMA_BEST_FIT = true;
567 
568 /*
569 Every object will have its own allocation.
570 Enable for debugging purposes only.
571 */
572 static const bool VMA_DEBUG_ALWAYS_OWN_MEMORY = false;
573 
574 /*
575 Minimum alignment of all suballocations, in bytes.
576 Set to more than 1 for debugging purposes only. Must be power of two.
577 */
578 static const VkDeviceSize VMA_DEBUG_ALIGNMENT = 1;
579 
580 /*
581 Minimum margin between suballocations, in bytes.
582 Set nonzero for debugging purposes only.
583 */
584 static const VkDeviceSize VMA_DEBUG_MARGIN = 0;
585 
586 /*
587 Set this to 1 for debugging purposes only, to enable single mutex protecting all
588 entry calls to the library. Can be useful for debugging multithreading issues.
589 */
590 #define VMA_DEBUG_GLOBAL_MUTEX 0
591 
592 /*
593 Minimum value for VkPhysicalDeviceLimits::bufferImageGranularity.
594 Set to more than 1 for debugging purposes only. Must be power of two.
595 */
596 static const VkDeviceSize VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY = 1;
597 
598 // Maximum size of a memory heap in Vulkan to consider it "small".
599 static const VkDeviceSize VMA_SMALL_HEAP_MAX_SIZE = 512 * 1024 * 1024;
600 // Default size of a block allocated as single VkDeviceMemory from a "large" heap.
601 static const VkDeviceSize VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE = 256 * 1024 * 1024;
602 // Default size of a block allocated as single VkDeviceMemory from a "small" heap.
603 static const VkDeviceSize VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE = 64 * 1024 * 1024;
604 
605 /*******************************************************************************
606 END OF CONFIGURATION
607 */
608 
609 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
610  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
611 
612 // Returns number of bits set to 1 in (v).
613 static inline uint32_t CountBitsSet(uint32_t v)
614 {
615  uint32_t c = v - ((v >> 1) & 0x55555555);
616  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
617  c = ((c >> 4) + c) & 0x0F0F0F0F;
618  c = ((c >> 8) + c) & 0x00FF00FF;
619  c = ((c >> 16) + c) & 0x0000FFFF;
620  return c;
621 }
622 
623 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
624 // Use types like uint32_t, uint64_t as T.
625 template <typename T>
626 static inline T VmaAlignUp(T val, T align)
627 {
628  return (val + align - 1) / align * align;
629 }
630 
631 // Division with mathematical rounding to nearest number.
632 template <typename T>
633 inline T VmaRoundDiv(T x, T y)
634 {
635  return (x + (y / (T)2)) / y;
636 }
637 /*
638 Returns true if two memory blocks occupy overlapping pages.
639 ResourceA must be in less memory offset than ResourceB.
640 
641 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
642 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
643 */
644 static inline bool VmaBlocksOnSamePage(
645  VkDeviceSize resourceAOffset,
646  VkDeviceSize resourceASize,
647  VkDeviceSize resourceBOffset,
648  VkDeviceSize pageSize)
649 {
650  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
651  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
652  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
653  VkDeviceSize resourceBStart = resourceBOffset;
654  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
655  return resourceAEndPage == resourceBStartPage;
656 }
657 
658 enum VmaSuballocationType
659 {
660  VMA_SUBALLOCATION_TYPE_FREE = 0,
661  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
662  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
663  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
664  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
665  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
666  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
667 };
668 
669 /*
670 Returns true if given suballocation types could conflict and must respect
671 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
672 or linear image and another one is optimal image. If type is unknown, behave
673 conservatively.
674 */
675 static inline bool VmaIsBufferImageGranularityConflict(
676  VmaSuballocationType suballocType1,
677  VmaSuballocationType suballocType2)
678 {
679  if(suballocType1 > suballocType2)
680  VMA_SWAP(suballocType1, suballocType2);
681 
682  switch(suballocType1)
683  {
684  case VMA_SUBALLOCATION_TYPE_FREE:
685  return false;
686  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
687  return true;
688  case VMA_SUBALLOCATION_TYPE_BUFFER:
689  return
690  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
691  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
692  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
693  return
694  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
695  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
696  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
697  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
698  return
699  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
700  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
701  return false;
702  default:
703  VMA_ASSERT(0);
704  return true;
705  }
706 }
707 
708 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
709 struct VmaMutexLock
710 {
711 public:
712  VmaMutexLock(VmaMutex& mutex) : m_Mutex(mutex) { mutex.Lock(); }
713  ~VmaMutexLock() { m_Mutex.Unlock(); }
714 
715 private:
716  VmaMutex& m_Mutex;
717 };
718 
719 #if VMA_DEBUG_GLOBAL_MUTEX
720  static VmaMutex gDebugGlobalMutex;
721  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex);
722 #else
723  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
724 #endif
725 
726 // Minimum size of a free suballocation to register it in the free suballocation collection.
727 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
728 
729 /*
730 Performs binary search and returns iterator to first element that is greater or
731 equal to (key), according to comparison (cmp).
732 
733 Cmp should return true if first argument is less than second argument.
734 
735 Returned value is the found element, if present in the collection or place where
736 new element with value (key) should be inserted.
737 */
738 template <typename IterT, typename KeyT, typename CmpT>
739 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
740 {
741  size_t down = 0, up = (end - beg);
742  while(down < up)
743  {
744  const size_t mid = (down + up) / 2;
745  if(cmp(*(beg+mid), key))
746  down = mid + 1;
747  else
748  up = mid;
749  }
750  return beg + down;
751 }
752 
754 // Memory allocation
755 
756 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
757 {
758  if((pAllocationCallbacks != VMA_NULL) &&
759  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
760  {
761  return (*pAllocationCallbacks->pfnAllocation)(
762  pAllocationCallbacks->pUserData,
763  size,
764  alignment,
765  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
766  }
767  else
768  {
769  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
770  }
771 }
772 
773 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
774 {
775  if((pAllocationCallbacks != VMA_NULL) &&
776  (pAllocationCallbacks->pfnFree != VMA_NULL))
777  {
778  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
779  }
780  else
781  {
782  VMA_SYSTEM_FREE(ptr);
783  }
784 }
785 
786 template<typename T>
787 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
788 {
789  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
790 }
791 
792 template<typename T>
793 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
794 {
795  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
796 }
797 
798 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
799 
800 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
801 
802 template<typename T>
803 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
804 {
805  ptr->~T();
806  VmaFree(pAllocationCallbacks, ptr);
807 }
808 
809 template<typename T>
810 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
811 {
812  if(ptr != VMA_NULL)
813  {
814  for(size_t i = count; i--; )
815  ptr[i].~T();
816  VmaFree(pAllocationCallbacks, ptr);
817  }
818 }
819 
820 // STL-compatible allocator.
821 template<typename T>
822 class VmaStlAllocator
823 {
824 public:
825  const VkAllocationCallbacks* const m_pCallbacks;
826  typedef T value_type;
827 
828  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
829  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
830 
831  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
832  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
833 
834  template<typename U>
835  bool operator==(const VmaStlAllocator<U>& rhs) const
836  {
837  return m_pCallbacks == rhs.m_pCallbacks;
838  }
839  template<typename U>
840  bool operator!=(const VmaStlAllocator<U>& rhs) const
841  {
842  return m_pCallbacks != rhs.m_pCallbacks;
843  }
844 };
845 
846 #if VMA_USE_STL_VECTOR
847 
848 #define VmaVector std::vector
849 
850 template<typename T, typename allocatorT>
851 static void VectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
852 {
853  vec.insert(vec.begin() + index, item);
854 }
855 
856 template<typename T, typename allocatorT>
857 static void VectorRemove(std::vector<T, allocatorT>& vec, size_t index)
858 {
859  vec.erase(vec.begin() + index);
860 }
861 
862 #else // #if VMA_USE_STL_VECTOR
863 
864 /* Class with interface compatible with subset of std::vector.
865 T must be POD because constructors and destructors are not called and memcpy is
866 used for these objects. */
867 template<typename T, typename AllocatorT>
868 class VmaVector
869 {
870 public:
871  VmaVector(AllocatorT& allocator) :
872  m_Allocator(allocator),
873  m_pArray(VMA_NULL),
874  m_Count(0),
875  m_Capacity(0)
876  {
877  }
878 
879  VmaVector(size_t count, AllocatorT& allocator) :
880  m_Allocator(allocator),
881  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, count) : VMA_NULL),
882  m_Count(count),
883  m_Capacity(count)
884  {
885  }
886 
887  VmaVector(const VmaVector<T, AllocatorT>& src) :
888  m_Allocator(src.m_Allocator),
889  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, src.m_Count) : VMA_NULL),
890  m_Count(src.m_Count),
891  m_Capacity(src.m_Count)
892  {
893  if(m_Count != 0)
894  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
895  }
896 
897  ~VmaVector()
898  {
899  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
900  }
901 
902  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
903  {
904  if(&rhs != this)
905  {
906  Resize(rhs.m_Count);
907  if(m_Count != 0)
908  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
909  }
910  return *this;
911  }
912 
913  bool empty() const { return m_Count == 0; }
914  size_t size() const { return m_Count; }
915  T* data() { return m_pArray; }
916  const T* data() const { return m_pArray; }
917 
918  T& operator[](size_t index)
919  {
920  VMA_HEAVY_ASSERT(index < m_Count);
921  return m_pArray[index];
922  }
923  const T& operator[](size_t index) const
924  {
925  VMA_HEAVY_ASSERT(index < m_Count);
926  return m_pArray[index];
927  }
928 
929  T& front()
930  {
931  VMA_HEAVY_ASSERT(m_Count > 0);
932  return m_pArray[0];
933  }
934  const T& front() const
935  {
936  VMA_HEAVY_ASSERT(m_Count > 0);
937  return m_pArray[0];
938  }
939  T& back()
940  {
941  VMA_HEAVY_ASSERT(m_Count > 0);
942  return m_pArray[m_Count - 1];
943  }
944  const T& back() const
945  {
946  VMA_HEAVY_ASSERT(m_Count > 0);
947  return m_pArray[m_Count - 1];
948  }
949 
950  void reserve(size_t newCapacity, bool freeMemory = false)
951  {
952  newCapacity = VMA_MAX(newCapacity, m_Count);
953 
954  if((newCapacity < m_Capacity) && !freeMemory)
955  newCapacity = m_Capacity;
956 
957  if(newCapacity != m_Capacity)
958  {
959  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_hAllocator, newCapacity) : VMA_NULL;
960  if(m_Count != 0)
961  memcpy(newArray, m_pArray, m_Count * sizeof(T));
962  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
963  m_Capacity = newCapacity;
964  m_pArray = newArray;
965  }
966  }
967 
968  void resize(size_t newCount, bool freeMemory = false)
969  {
970  size_t newCapacity = m_Capacity;
971  if(newCount > m_Capacity)
972  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
973  else if(freeMemory)
974  newCapacity = newCount;
975 
976  if(newCapacity != m_Capacity)
977  {
978  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
979  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
980  if(elementsToCopy != 0)
981  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
982  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
983  m_Capacity = newCapacity;
984  m_pArray = newArray;
985  }
986 
987  m_Count = newCount;
988  }
989 
990  void clear(bool freeMemory = false)
991  {
992  resize(0, freeMemory);
993  }
994 
995  void insert(size_t index, const T& src)
996  {
997  VMA_HEAVY_ASSERT(index <= m_Count);
998  const size_t oldCount = size();
999  resize(oldCount + 1);
1000  if(index < oldCount)
1001  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1002  m_pArray[index] = src;
1003  }
1004 
1005  void remove(size_t index)
1006  {
1007  VMA_HEAVY_ASSERT(index < m_Count);
1008  const size_t oldCount = size();
1009  if(index < oldCount - 1)
1010  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1011  resize(oldCount - 1);
1012  }
1013 
1014  void push_back(const T& src)
1015  {
1016  const size_t newIndex = size();
1017  resize(newIndex + 1);
1018  m_pArray[newIndex] = src;
1019  }
1020 
1021  void pop_back()
1022  {
1023  VMA_HEAVY_ASSERT(m_Count > 0);
1024  resize(size() - 1);
1025  }
1026 
1027  void push_front(const T& src)
1028  {
1029  insert(0, src);
1030  }
1031 
1032  void pop_front()
1033  {
1034  VMA_HEAVY_ASSERT(m_Count > 0);
1035  remove(0);
1036  }
1037 
1038  typedef T* iterator;
1039 
1040  iterator begin() { return m_pArray; }
1041  iterator end() { return m_pArray + m_Count; }
1042 
1043 private:
1044  AllocatorT m_Allocator;
1045  T* m_pArray;
1046  size_t m_Count;
1047  size_t m_Capacity;
1048 };
1049 
1050 template<typename T, typename allocatorT>
1051 static void VectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
1052 {
1053  vec.insert(index, item);
1054 }
1055 
1056 template<typename T, typename allocatorT>
1057 static void VectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
1058 {
1059  vec.remove(index);
1060 }
1061 
1062 #endif // #if VMA_USE_STL_VECTOR
1063 
1065 // class VmaPoolAllocator
1066 
1067 /*
1068 Allocator for objects of type T using a list of arrays (pools) to speed up
1069 allocation. Number of elements that can be allocated is not bounded because
1070 allocator can create multiple blocks.
1071 */
1072 template<typename T>
1073 class VmaPoolAllocator
1074 {
1075 public:
1076  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
1077  ~VmaPoolAllocator();
1078  void Clear();
1079  T* Alloc();
1080  void Free(T* ptr);
1081 
1082 private:
1083  union Item
1084  {
1085  uint32_t NextFreeIndex;
1086  T Value;
1087  };
1088 
1089  struct ItemBlock
1090  {
1091  Item* pItems;
1092  uint32_t FirstFreeIndex;
1093  };
1094 
1095  const VkAllocationCallbacks* m_pAllocationCallbacks;
1096  size_t m_ItemsPerBlock;
1097  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
1098 
1099  ItemBlock& CreateNewBlock();
1100 };
1101 
1102 template<typename T>
1103 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
1104  m_pAllocationCallbacks(pAllocationCallbacks),
1105  m_ItemsPerBlock(itemsPerBlock),
1106  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
1107 {
1108  VMA_ASSERT(itemsPerBlock > 0);
1109 }
1110 
1111 template<typename T>
1112 VmaPoolAllocator<T>::~VmaPoolAllocator()
1113 {
1114  Clear();
1115 }
1116 
1117 template<typename T>
1118 void VmaPoolAllocator<T>::Clear()
1119 {
1120  for(size_t i = m_ItemBlocks.size(); i--; )
1121  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
1122  m_ItemBlocks.clear();
1123 }
1124 
1125 template<typename T>
1126 T* VmaPoolAllocator<T>::Alloc()
1127 {
1128  for(size_t i = m_ItemBlocks.size(); i--; )
1129  {
1130  ItemBlock& block = m_ItemBlocks[i];
1131  // This block has some free items: Use first one.
1132  if(block.FirstFreeIndex != UINT_MAX)
1133  {
1134  Item* const pItem = &block.pItems[block.FirstFreeIndex];
1135  block.FirstFreeIndex = pItem->NextFreeIndex;
1136  return &pItem->Value;
1137  }
1138  }
1139 
1140  // No block has free item: Create new one and use it.
1141  ItemBlock& newBlock = CreateNewBlock();
1142  Item* const pItem = &newBlock.pItems[0];
1143  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
1144  return &pItem->Value;
1145 }
1146 
1147 template<typename T>
1148 void VmaPoolAllocator<T>::Free(T* ptr)
1149 {
1150  // Search all memory blocks to find ptr.
1151  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
1152  {
1153  ItemBlock& block = m_ItemBlocks[i];
1154 
1155  // Casting to union.
1156  Item* pItemPtr;
1157  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
1158 
1159  // Check if pItemPtr is in address range of this block.
1160  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
1161  {
1162  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
1163  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
1164  block.FirstFreeIndex = index;
1165  return;
1166  }
1167  }
1168  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
1169 }
1170 
1171 template<typename T>
1172 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
1173 {
1174  ItemBlock newBlock = {
1175  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
1176 
1177  m_ItemBlocks.push_back(newBlock);
1178 
1179  // Setup singly-linked list of all free items in this block.
1180  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
1181  newBlock.pItems[i].NextFreeIndex = i + 1;
1182  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT_MAX;
1183  return m_ItemBlocks.back();
1184 }
1185 
1187 // class VmaRawList, VmaList
1188 
1189 #if VMA_USE_STL_LIST
1190 
1191 #define VmaList std::list
1192 
1193 #else // #if VMA_USE_STL_LIST
1194 
1195 template<typename T>
1196 struct VmaListItem
1197 {
1198  VmaListItem* pPrev;
1199  VmaListItem* pNext;
1200  T Value;
1201 };
1202 
1203 // Doubly linked list.
1204 template<typename T>
1205 class VmaRawList
1206 {
1207 public:
1208  typedef VmaListItem<T> ItemType;
1209 
1210  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
1211  ~VmaRawList();
1212  void Clear();
1213 
1214  size_t GetCount() const { return m_Count; }
1215  bool IsEmpty() const { return m_Count == 0; }
1216 
1217  ItemType* Front() { return m_pFront; }
1218  const ItemType* Front() const { return m_pFront; }
1219  ItemType* Back() { return m_pBack; }
1220  const ItemType* Back() const { return m_pBack; }
1221 
1222  ItemType* PushBack();
1223  ItemType* PushFront();
1224  ItemType* PushBack(const T& value);
1225  ItemType* PushFront(const T& value);
1226  void PopBack();
1227  void PopFront();
1228 
1229  // Item can be null - it means PushBack.
1230  ItemType* InsertBefore(ItemType* pItem);
1231  // Item can be null - it means PushFront.
1232  ItemType* InsertAfter(ItemType* pItem);
1233 
1234  ItemType* InsertBefore(ItemType* pItem, const T& value);
1235  ItemType* InsertAfter(ItemType* pItem, const T& value);
1236 
1237  void Remove(ItemType* pItem);
1238 
1239 private:
1240  const VkAllocationCallbacks* const m_pAllocationCallbacks;
1241  VmaPoolAllocator<ItemType> m_ItemAllocator;
1242  ItemType* m_pFront;
1243  ItemType* m_pBack;
1244  size_t m_Count;
1245 
1246  // Declared not defined, to block copy constructor and assignment operator.
1247  VmaRawList(const VmaRawList<T>& src);
1248  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
1249 };
1250 
1251 template<typename T>
1252 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
1253  m_pAllocationCallbacks(pAllocationCallbacks),
1254  m_ItemAllocator(pAllocationCallbacks, 128),
1255  m_pFront(VMA_NULL),
1256  m_pBack(VMA_NULL),
1257  m_Count(0)
1258 {
1259 }
1260 
1261 template<typename T>
1262 VmaRawList<T>::~VmaRawList()
1263 {
1264  // Intentionally not calling Clear, because that would be unnecessary
1265  // computations to return all items to m_ItemAllocator as free.
1266 }
1267 
1268 template<typename T>
1269 void VmaRawList<T>::Clear()
1270 {
1271  if(IsEmpty() == false)
1272  {
1273  ItemType* pItem = m_pBack;
1274  while(pItem != VMA_NULL)
1275  {
1276  ItemType* const pPrevItem = pItem->pPrev;
1277  m_ItemAllocator.Free(pItem);
1278  pItem = pPrevItem;
1279  }
1280  m_pFront = VMA_NULL;
1281  m_pBack = VMA_NULL;
1282  m_Count = 0;
1283  }
1284 }
1285 
1286 template<typename T>
1287 VmaListItem<T>* VmaRawList<T>::PushBack()
1288 {
1289  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1290  pNewItem->pNext = VMA_NULL;
1291  if(IsEmpty())
1292  {
1293  pNewItem->pPrev = VMA_NULL;
1294  m_pFront = pNewItem;
1295  m_pBack = pNewItem;
1296  m_Count = 1;
1297  }
1298  else
1299  {
1300  pNewItem->pPrev = m_pBack;
1301  m_pBack->pNext = pNewItem;
1302  m_pBack = pNewItem;
1303  ++m_Count;
1304  }
1305  return pNewItem;
1306 }
1307 
1308 template<typename T>
1309 VmaListItem<T>* VmaRawList<T>::PushFront()
1310 {
1311  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1312  pNewItem->pPrev = VMA_NULL;
1313  if(IsEmpty())
1314  {
1315  pNewItem->pNext = VMA_NULL;
1316  m_pFront = pNewItem;
1317  m_pBack = pNewItem;
1318  m_Count = 1;
1319  }
1320  else
1321  {
1322  pNewItem->pNext = m_pFront;
1323  m_pFront->pPrev = pNewItem;
1324  m_pFront = pNewItem;
1325  ++m_Count;
1326  }
1327  return pNewItem;
1328 }
1329 
1330 template<typename T>
1331 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
1332 {
1333  ItemType* const pNewItem = PushBack();
1334  pNewItem->Value = value;
1335  return pNewItem;
1336 }
1337 
1338 template<typename T>
1339 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
1340 {
1341  ItemType* const pNewItem = PushFront();
1342  pNewItem->Value = value;
1343  return newItem;
1344 }
1345 
1346 template<typename T>
1347 void VmaRawList<T>::PopBack()
1348 {
1349  VMA_HEAVY_ASSERT(m_Count > 0);
1350  ItemType* const pBackItem = m_pBack;
1351  ItemType* const pPrevItem = pBackItem->pPrev;
1352  if(pPrevItem != VMA_NULL)
1353  pPrevItem->pNext = VMA_NULL;
1354  m_pBack = pPrevItem;
1355  m_ItemAllocator.Free(pBackItem);
1356  --m_Count;
1357 }
1358 
1359 template<typename T>
1360 void VmaRawList<T>::PopFront()
1361 {
1362  VMA_HEAVY_ASSERT(m_Count > 0);
1363  ItemType* const pFrontItem = m_pFront;
1364  ItemType* const pNextItem = pFrontItem->pNext;
1365  if(pNextItem != VMA_NULL)
1366  pNextItem->pPrev = VMA_NULL;
1367  m_pFront = pNextItem;
1368  m_ItemAllocator.Free(pFrontItem);
1369  --m_Count;
1370 }
1371 
1372 template<typename T>
1373 void VmaRawList<T>::Remove(ItemType* pItem)
1374 {
1375  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
1376  VMA_HEAVY_ASSERT(m_Count > 0);
1377 
1378  if(pItem->pPrev != VMA_NULL)
1379  pItem->pPrev->pNext = pItem->pNext;
1380  else
1381  {
1382  VMA_HEAVY_ASSERT(m_pFront == pItem);
1383  m_pFront = pItem->pNext;
1384  }
1385 
1386  if(pItem->pNext != VMA_NULL)
1387  pItem->pNext->pPrev = pItem->pPrev;
1388  else
1389  {
1390  VMA_HEAVY_ASSERT(m_pBack == pItem);
1391  m_pBack = pItem->pPrev;
1392  }
1393 
1394  m_ItemAllocator.Free(pItem);
1395  --m_Count;
1396 }
1397 
1398 template<typename T>
1399 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
1400 {
1401  if(pItem != VMA_NULL)
1402  {
1403  ItemType* const prevItem = pItem->pPrev;
1404  ItemType* const newItem = m_ItemAllocator.Alloc();
1405  newItem->pPrev = prevItem;
1406  newItem->pNext = pItem;
1407  pItem->pPrev = newItem;
1408  if(prevItem != VMA_NULL)
1409  prevItem->pNext = newItem;
1410  else
1411  {
1412  VMA_HEAVY_ASSERT(m_pFront = pItem);
1413  m_pFront = newItem;
1414  }
1415  ++m_Count;
1416  return newItem;
1417  }
1418  else
1419  return PushBack();
1420 }
1421 
1422 template<typename T>
1423 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
1424 {
1425  if(pItem != VMA_NULL)
1426  {
1427  ItemType* const nextItem = pItem->pNext;
1428  ItemType* const newItem = m_ItemAllocator.Alloc();
1429  newItem->pNext = nextItem;
1430  newItem->pPrev = pItem;
1431  pItem->pNext = newItem;
1432  if(nextItem != VMA_NULL)
1433  nextItem->pPrev = newItem;
1434  else
1435  {
1436  VMA_HEAVY_ASSERT(m_pBack = pItem);
1437  m_pBack = newItem;
1438  }
1439  ++m_Count;
1440  return newItem;
1441  }
1442  else
1443  return PushFront();
1444 }
1445 
1446 template<typename T>
1447 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
1448 {
1449  ItemType* const newItem = InsertBefore(pItem);
1450  newItem->Value = value;
1451  return newItem;
1452 }
1453 
1454 template<typename T>
1455 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
1456 {
1457  ItemType* const newItem = InsertAfter(pItem);
1458  newItem->Value = value;
1459  return newItem;
1460 }
1461 
1462 template<typename T, typename AllocatorT>
1463 class VmaList
1464 {
1465 public:
1466  class iterator
1467  {
1468  public:
1469  iterator() :
1470  m_pList(VMA_NULL),
1471  m_pItem(VMA_NULL)
1472  {
1473  }
1474 
1475  T& operator*() const
1476  {
1477  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1478  return m_pItem->Value;
1479  }
1480  T* operator->() const
1481  {
1482  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1483  return &m_pItem->Value;
1484  }
1485 
1486  iterator& operator++()
1487  {
1488  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1489  m_pItem = m_pItem->pNext;
1490  return *this;
1491  }
1492  iterator& operator--()
1493  {
1494  if(m_pItem != VMA_NULL)
1495  m_pItem = m_pItem->pPrev;
1496  else
1497  {
1498  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
1499  m_pItem = m_pList->Back();
1500  }
1501  return *this;
1502  }
1503 
1504  iterator operator++(int)
1505  {
1506  iterator result = *this;
1507  ++*this;
1508  return result;
1509  }
1510  iterator operator--(int)
1511  {
1512  iterator result = *this;
1513  --*this;
1514  return result;
1515  }
1516 
1517  bool operator==(const iterator& rhs) const
1518  {
1519  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1520  return m_pItem == rhs.m_pItem;
1521  }
1522  bool operator!=(const iterator& rhs) const
1523  {
1524  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1525  return m_pItem != rhs.m_pItem;
1526  }
1527 
1528  private:
1529  VmaRawList<T>* m_pList;
1530  VmaListItem<T>* m_pItem;
1531 
1532  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
1533  m_pList(pList),
1534  m_pItem(pItem)
1535  {
1536  }
1537 
1538  friend class VmaList<T, AllocatorT>;
1539  friend class VmaList<T, AllocatorT>:: const_iterator;
1540  };
1541 
1542  class const_iterator
1543  {
1544  public:
1545  const_iterator() :
1546  m_pList(VMA_NULL),
1547  m_pItem(VMA_NULL)
1548  {
1549  }
1550 
1551  const_iterator(const iterator& src) :
1552  m_pList(src.m_pList),
1553  m_pItem(src.m_pItem)
1554  {
1555  }
1556 
1557  const T& operator*() const
1558  {
1559  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1560  return m_pItem->Value;
1561  }
1562  const T* operator->() const
1563  {
1564  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1565  return &m_pItem->Value;
1566  }
1567 
1568  const_iterator& operator++()
1569  {
1570  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1571  m_pItem = m_pItem->pNext;
1572  return *this;
1573  }
1574  const_iterator& operator--()
1575  {
1576  if(m_pItem != VMA_NULL)
1577  m_pItem = m_pItem->pPrev;
1578  else
1579  {
1580  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
1581  m_pItem = m_pList->Back();
1582  }
1583  return *this;
1584  }
1585 
1586  const_iterator operator++(int)
1587  {
1588  const_iterator result = *this;
1589  ++*this;
1590  return result;
1591  }
1592  const_iterator operator--(int)
1593  {
1594  const_iterator result = *this;
1595  --*this;
1596  return result;
1597  }
1598 
1599  bool operator==(const const_iterator& rhs) const
1600  {
1601  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1602  return m_pItem == rhs.m_pItem;
1603  }
1604  bool operator!=(const const_iterator& rhs) const
1605  {
1606  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1607  return m_pItem != rhs.m_pItem;
1608  }
1609 
1610  private:
1611  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
1612  m_pList(pList),
1613  m_pItem(pItem)
1614  {
1615  }
1616 
1617  const VmaRawList<T>* m_pList;
1618  const VmaListItem<T>* m_pItem;
1619 
1620  friend class VmaList<T, AllocatorT>;
1621  };
1622 
1623  VmaList(AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
1624 
1625  bool empty() const { return m_RawList.IsEmpty(); }
1626  size_t size() const { return m_RawList.GetCount(); }
1627 
1628  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
1629  iterator end() { return iterator(&m_RawList, VMA_NULL); }
1630 
1631  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
1632  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
1633 
1634  void clear() { m_RawList.Clear(); }
1635  void push_back(const T& value) { m_RawList.PushBack(value); }
1636  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
1637  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
1638 
1639 private:
1640  VmaRawList<T> m_RawList;
1641 };
1642 
1643 #endif // #if VMA_USE_STL_LIST
1644 
1646 // class VmaMap
1647 
1648 #if VMA_USE_STL_UNORDERED_MAP
1649 
1650 #define VmaPair std::pair
1651 
1652 #define VMA_MAP_TYPE(KeyT, ValueT) \
1653  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
1654 
1655 #else // #if VMA_USE_STL_UNORDERED_MAP
1656 
1657 template<typename T1, typename T2>
1658 struct VmaPair
1659 {
1660  T1 first;
1661  T2 second;
1662 
1663  VmaPair() : first(), second() { }
1664  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
1665 };
1666 
1667 /* Class compatible with subset of interface of std::unordered_map.
1668 KeyT, ValueT must be POD because they will be stored in VmaVector.
1669 */
1670 template<typename KeyT, typename ValueT>
1671 class VmaMap
1672 {
1673 public:
1674  typedef VmaPair<KeyT, ValueT> PairType;
1675  typedef PairType* iterator;
1676 
1677  VmaMap(VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
1678 
1679  iterator begin() { return m_Vector.begin(); }
1680  iterator end() { return m_Vector.end(); }
1681 
1682  void insert(const PairType& pair);
1683  iterator find(const KeyT& key);
1684  void erase(iterator it);
1685 
1686 private:
1687  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
1688 };
1689 
1690 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
1691 
1692 template<typename FirstT, typename SecondT>
1693 struct VmaPairFirstLess
1694 {
1695  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
1696  {
1697  return lhs.first < rhs.first;
1698  }
1699  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
1700  {
1701  return lhs.first < rhsFirst;
1702  }
1703 };
1704 
1705 template<typename KeyT, typename ValueT>
1706 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
1707 {
1708  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
1709  m_Vector.data(),
1710  m_Vector.data() + m_Vector.size(),
1711  pair,
1712  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
1713  VectorInsert(m_Vector, indexToInsert, pair);
1714 }
1715 
1716 template<typename KeyT, typename ValueT>
1717 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
1718 {
1719  PairType* it = VmaBinaryFindFirstNotLess(
1720  m_Vector.data(),
1721  m_Vector.data() + m_Vector.size(),
1722  key,
1723  VmaPairFirstLess<KeyT, ValueT>());
1724  if((it != m_Vector.end()) && (it->first == key))
1725  return it;
1726  else
1727  return m_Vector.end();
1728 }
1729 
1730 template<typename KeyT, typename ValueT>
1731 void VmaMap<KeyT, ValueT>::erase(iterator it)
1732 {
1733  VectorRemove(m_Vector, it - m_Vector.begin());
1734 }
1735 
1736 #endif // #if VMA_USE_STL_UNORDERED_MAP
1737 
1738 /*
1739 Represents a region of VmaAllocation that is either assigned and returned as
1740 allocated memory block or free.
1741 */
1742 struct VmaSuballocation
1743 {
1744  VkDeviceSize offset;
1745  VkDeviceSize size;
1746  VmaSuballocationType type;
1747 };
1748 
1749 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
1750 
1751 // Parameters of an allocation.
1752 struct VmaAllocationRequest
1753 {
1754  VmaSuballocationList::iterator freeSuballocationItem;
1755  VkDeviceSize offset;
1756 };
1757 
1758 /* Single block of memory - VkDeviceMemory with all the data about its regions
1759 assigned or free. */
1760 class VmaAllocation
1761 {
1762 public:
1763  VkDeviceMemory m_hMemory;
1764  VkDeviceSize m_Size;
1765  uint32_t m_FreeCount;
1766  VkDeviceSize m_SumFreeSize;
1767  VmaSuballocationList m_Suballocations;
1768  // Suballocations that are free and have size greater than certain threshold.
1769  // Sorted by size, ascending.
1770  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
1771 
1772  VmaAllocation(VmaAllocator hAllocator);
1773 
1774  ~VmaAllocation()
1775  {
1776  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
1777  }
1778 
1779  // Always call after construction.
1780  void Init(VkDeviceMemory newMemory, VkDeviceSize newSize);
1781  // Always call before destruction.
1782  void Destroy(VmaAllocator allocator);
1783 
1784  // Validates all data structures inside this object. If not valid, returns false.
1785  bool Validate() const;
1786 
1787  // Tries to find a place for suballocation with given parameters inside this allocation.
1788  // If succeeded, fills pAllocationRequest and returns true.
1789  // If failed, returns false.
1790  bool CreateAllocationRequest(
1791  VkDeviceSize bufferImageGranularity,
1792  VkDeviceSize allocSize,
1793  VkDeviceSize allocAlignment,
1794  VmaSuballocationType allocType,
1795  VmaAllocationRequest* pAllocationRequest);
1796 
1797  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
1798  // If yes, fills pOffset and returns true. If no, returns false.
1799  bool CheckAllocation(
1800  VkDeviceSize bufferImageGranularity,
1801  VkDeviceSize allocSize,
1802  VkDeviceSize allocAlignment,
1803  VmaSuballocationType allocType,
1804  VmaSuballocationList::const_iterator freeSuballocItem,
1805  VkDeviceSize* pOffset) const;
1806 
1807  // Returns true if this allocation is empty - contains only single free suballocation.
1808  bool IsEmpty() const;
1809 
1810  // Makes actual allocation based on request. Request must already be checked
1811  // and valid.
1812  void Alloc(
1813  const VmaAllocationRequest& request,
1814  VmaSuballocationType type,
1815  VkDeviceSize allocSize);
1816 
1817  // Frees suballocation assigned to given memory region.
1818  void Free(const VkMappedMemoryRange* pMemory);
1819 
1820 #if VMA_STATS_STRING_ENABLED
1821  void PrintDetailedMap(class VmaStringBuilder& sb) const;
1822 #endif
1823 
1824 private:
1825  // Given free suballocation, it merges it with following one, which must also be free.
1826  void MergeFreeWithNext(VmaSuballocationList::iterator item);
1827  // Releases given suballocation, making it free. Merges it with adjacent free
1828  // suballocations if applicable.
1829  void FreeSuballocation(VmaSuballocationList::iterator suballocItem);
1830  // Given free suballocation, it inserts it into sorted list of
1831  // m_FreeSuballocationsBySize if it's suitable.
1832  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
1833  // Given free suballocation, it removes it from sorted list of
1834  // m_FreeSuballocationsBySize if it's suitable.
1835  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
1836 };
1837 
1838 // Allocation for an object that has its own private VkDeviceMemory.
1839 struct VmaOwnAllocation
1840 {
1841  VkDeviceMemory m_hMemory;
1842  VkDeviceSize m_Size;
1843  VmaSuballocationType m_Type;
1844 };
1845 
1846 struct VmaOwnAllocationMemoryHandleLess
1847 {
1848  bool operator()(const VmaOwnAllocation& lhs, const VmaOwnAllocation& rhs) const
1849  {
1850  return lhs.m_hMemory < rhs.m_hMemory;
1851  }
1852  bool operator()(const VmaOwnAllocation& lhs, VkDeviceMemory rhsMem) const
1853  {
1854  return lhs.m_hMemory < rhsMem;
1855  }
1856 };
1857 
1858 /* Sequence of VmaAllocation. Represents memory blocks allocated for a specific
1859 Vulkan memory type. */
1860 struct VmaAllocationVector
1861 {
1862  // Incrementally sorted by sumFreeSize, ascending.
1863  VmaVector< VmaAllocation*, VmaStlAllocator<VmaAllocation*> > m_Allocations;
1864 
1865  VmaAllocationVector(VmaAllocator hAllocator);
1866  ~VmaAllocationVector();
1867 
1868  bool IsEmpty() const { return m_Allocations.empty(); }
1869 
1870  // Tries to free memory from any if its Allocations.
1871  // Returns index of Allocation that the memory was freed from, or -1 if not found.
1872  size_t Free(const VkMappedMemoryRange* pMemory);
1873 
1874  // Performs single step in sorting m_Allocations. They may not be fully sorted
1875  // after this call.
1876  void IncrementallySortAllocations();
1877 
1878  // Adds statistics of this AllocationVector to pStats.
1879  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const;
1880 
1881 #if VMA_STATS_STRING_ENABLED
1882  void PrintDetailedMap(class VmaStringBuilder& sb) const;
1883 #endif
1884 
1885 private:
1886  VmaAllocator m_hAllocator;
1887 };
1888 
1889 // Main allocator object.
1890 struct VmaAllocator_T
1891 {
1892  VkDevice m_hDevice;
1893  bool m_AllocationCallbacksSpecified;
1894  VkAllocationCallbacks m_AllocationCallbacks;
1895  VkDeviceSize m_PreferredLargeHeapBlockSize;
1896  VkDeviceSize m_PreferredSmallHeapBlockSize;
1897 
1898  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
1899  VkPhysicalDeviceMemoryProperties m_MemProps;
1900 
1901  VmaAllocationVector* m_pAllocations[VK_MAX_MEMORY_TYPES];
1902  /* There can be at most one allocation that is completely empty - a
1903  hysteresis to avoid pessimistic case of alternating creation and destruction
1904  of a VkDeviceMemory. */
1905  bool m_HasEmptyAllocation[VK_MAX_MEMORY_TYPES];
1906  VmaMutex m_AllocationsMutex[VK_MAX_MEMORY_TYPES];
1907 
1908  // Each vector is sorted by memory (handle value).
1909  typedef VmaVector< VmaOwnAllocation, VmaStlAllocator<VmaOwnAllocation> > OwnAllocationVectorType;
1910  OwnAllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES];
1911  VmaMutex m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
1912 
1913  // Sorted by first (VkBuffer handle value).
1914  VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange) m_BufferToMemoryMap;
1915  VmaMutex m_BufferToMemoryMapMutex;
1916  // Sorted by first (VkImage handle value).
1917  VMA_MAP_TYPE(VkImage, VkMappedMemoryRange) m_ImageToMemoryMap;
1918  VmaMutex m_ImageToMemoryMapMutex;
1919 
1920  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
1921  ~VmaAllocator_T();
1922 
1923  const VkAllocationCallbacks* GetAllocationCallbacks() const
1924  {
1925  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
1926  }
1927 
1928  VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const;
1929 
1930  VkDeviceSize GetBufferImageGranularity() const
1931  {
1932  return VMA_MAX(
1933  VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,
1934  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
1935  }
1936 
1937  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
1938  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
1939 
1940  // Main allocation function.
1941  VkResult AllocateMemory(
1942  const VkMemoryRequirements& vkMemReq,
1943  const VmaMemoryRequirements& vmaMemReq,
1944  VmaSuballocationType suballocType,
1945  VkMappedMemoryRange* pMemory,
1946  uint32_t* pMemoryTypeIndex);
1947 
1948  // Main deallocation function.
1949  void FreeMemory(const VkMappedMemoryRange* pMemory);
1950 
1951  void CalculateStats(VmaStats* pStats);
1952 
1953 #if VMA_STATS_STRING_ENABLED
1954  void PrintDetailedMap(class VmaStringBuilder& sb);
1955 #endif
1956 
1957 private:
1958  VkPhysicalDevice m_PhysicalDevice;
1959 
1960  VkResult AllocateMemoryOfType(
1961  const VkMemoryRequirements& vkMemReq,
1962  const VmaMemoryRequirements& vmaMemReq,
1963  uint32_t memTypeIndex,
1964  VmaSuballocationType suballocType,
1965  VkMappedMemoryRange* pMemory);
1966 
1967  // Allocates and registers new VkDeviceMemory specifically for single allocation.
1968  VkResult AllocateOwnMemory(
1969  VkDeviceSize size,
1970  VmaSuballocationType suballocType,
1971  uint32_t memTypeIndex,
1972  VkMappedMemoryRange* pMemory);
1973 
1974  // Tries to free pMemory as Own Memory. Returns true if found and freed.
1975  bool FreeOwnMemory(const VkMappedMemoryRange* pMemory);
1976 };
1977 
1979 // Memory allocation #2 after VmaAllocator_T definition
1980 
1981 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
1982 {
1983  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
1984 }
1985 
1986 static void VmaFree(VmaAllocator hAllocator, void* ptr)
1987 {
1988  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
1989 }
1990 
1991 template<typename T>
1992 static T* VmaAllocate(VmaAllocator hAllocator)
1993 {
1994  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
1995 }
1996 
1997 template<typename T>
1998 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
1999 {
2000  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
2001 }
2002 
2003 template<typename T>
2004 static void vma_delete(VmaAllocator hAllocator, T* ptr)
2005 {
2006  if(ptr != VMA_NULL)
2007  {
2008  ptr->~T();
2009  VmaFree(hAllocator, ptr);
2010  }
2011 }
2012 
2013 template<typename T>
2014 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
2015 {
2016  if(ptr != VMA_NULL)
2017  {
2018  for(size_t i = count; i--; )
2019  ptr[i].~T();
2020  VmaFree(hAllocator, ptr);
2021  }
2022 }
2023 
2025 // VmaStringBuilder
2026 
2027 #if VMA_STATS_STRING_ENABLED
2028 
2029 class VmaStringBuilder
2030 {
2031 public:
2032  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
2033  size_t GetLength() const { return m_Data.size(); }
2034  const char* GetData() const { return m_Data.data(); }
2035 
2036  void Add(char ch) { m_Data.push_back(ch); }
2037  void Add(const char* pStr);
2038  void AddNewLine() { Add('\n'); }
2039  void AddNumber(uint32_t num);
2040  void AddNumber(uint64_t num);
2041  void AddBool(bool b) { Add(b ? "true" : "false"); }
2042  void AddNull() { Add("null"); }
2043  void AddString(const char* pStr);
2044 
2045 private:
2046  VmaVector< char, VmaStlAllocator<char> > m_Data;
2047 };
2048 
2049 void VmaStringBuilder::Add(const char* pStr)
2050 {
2051  const size_t strLen = strlen(pStr);
2052  if(strLen > 0)
2053  {
2054  const size_t oldCount = m_Data.size();
2055  m_Data.resize(oldCount + strLen);
2056  memcpy(m_Data.data() + oldCount, pStr, strLen);
2057  }
2058 }
2059 
2060 void VmaStringBuilder::AddNumber(uint32_t num)
2061 {
2062  char buf[11];
2063  VmaUint32ToStr(buf, sizeof(buf), num);
2064  Add(buf);
2065 }
2066 
2067 void VmaStringBuilder::AddNumber(uint64_t num)
2068 {
2069  char buf[21];
2070  VmaUint64ToStr(buf, sizeof(buf), num);
2071  Add(buf);
2072 }
2073 
2074 void VmaStringBuilder::AddString(const char* pStr)
2075 {
2076  Add('"');
2077  const size_t strLen = strlen(pStr);
2078  for(size_t i = 0; i < strLen; ++i)
2079  {
2080  char ch = pStr[i];
2081  if(ch == '\'')
2082  Add("\\\\");
2083  else if(ch == '"')
2084  Add("\\\"");
2085  else if(ch >= 32)
2086  Add(ch);
2087  else switch(ch)
2088  {
2089  case '\n':
2090  Add("\\n");
2091  break;
2092  case '\r':
2093  Add("\\r");
2094  break;
2095  case '\t':
2096  Add("\\t");
2097  break;
2098  default:
2099  VMA_ASSERT(0 && "Character not currently supported.");
2100  break;
2101  }
2102  }
2103  Add('"');
2104 }
2105 
2107 
2108 // Correspond to values of enum VmaSuballocationType.
2109 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
2110  "FREE",
2111  "UNKNOWN",
2112  "BUFFER",
2113  "IMAGE_UNKNOWN",
2114  "IMAGE_LINEAR",
2115  "IMAGE_OPTIMAL",
2116 };
2117 
2118 static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat)
2119 {
2120  sb.Add("{ \"Allocations\": ");
2121  sb.AddNumber(stat.AllocationCount);
2122  sb.Add(", \"Suballocations\": ");
2123  sb.AddNumber(stat.SuballocationCount);
2124  sb.Add(", \"UnusedRanges\": ");
2125  sb.AddNumber(stat.UnusedRangeCount);
2126  sb.Add(", \"UsedBytes\": ");
2127  sb.AddNumber(stat.UsedBytes);
2128  sb.Add(", \"UnusedBytes\": ");
2129  sb.AddNumber(stat.UnusedBytes);
2130  sb.Add(", \"SuballocationSize\": { \"Min\": ");
2131  sb.AddNumber(stat.SuballocationSizeMin);
2132  sb.Add(", \"Avg\": ");
2133  sb.AddNumber(stat.SuballocationSizeAvg);
2134  sb.Add(", \"Max\": ");
2135  sb.AddNumber(stat.SuballocationSizeMax);
2136  sb.Add(" }, \"UnusedRangeSize\": { \"Min\": ");
2137  sb.AddNumber(stat.UnusedRangeSizeMin);
2138  sb.Add(", \"Avg\": ");
2139  sb.AddNumber(stat.UnusedRangeSizeAvg);
2140  sb.Add(", \"Max\": ");
2141  sb.AddNumber(stat.UnusedRangeSizeMax);
2142  sb.Add(" } }");
2143 }
2144 
2145 #endif // #if VMA_STATS_STRING_ENABLED
2146 
2147 struct VmaSuballocationItemSizeLess
2148 {
2149  bool operator()(
2150  const VmaSuballocationList::iterator lhs,
2151  const VmaSuballocationList::iterator rhs) const
2152  {
2153  return lhs->size < rhs->size;
2154  }
2155  bool operator()(
2156  const VmaSuballocationList::iterator lhs,
2157  VkDeviceSize rhsSize) const
2158  {
2159  return lhs->size < rhsSize;
2160  }
2161 };
2162 
2163 VmaAllocation::VmaAllocation(VmaAllocator hAllocator) :
2164  m_hMemory(VK_NULL_HANDLE),
2165  m_Size(0),
2166  m_FreeCount(0),
2167  m_SumFreeSize(0),
2168  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
2169  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
2170 {
2171 }
2172 
2173 void VmaAllocation::Init(VkDeviceMemory newMemory, VkDeviceSize newSize)
2174 {
2175  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2176 
2177  m_hMemory = newMemory;
2178  m_Size = newSize;
2179  m_FreeCount = 1;
2180  m_SumFreeSize = newSize;
2181 
2182  m_Suballocations.clear();
2183  m_FreeSuballocationsBySize.clear();
2184 
2185  VmaSuballocation suballoc = {};
2186  suballoc.offset = 0;
2187  suballoc.size = newSize;
2188  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2189 
2190  m_Suballocations.push_back(suballoc);
2191  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
2192  --suballocItem;
2193  m_FreeSuballocationsBySize.push_back(suballocItem);
2194 }
2195 
2196 void VmaAllocation::Destroy(VmaAllocator allocator)
2197 {
2198  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
2199  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
2200  m_hMemory = VK_NULL_HANDLE;
2201 }
2202 
2203 bool VmaAllocation::Validate() const
2204 {
2205  if((m_hMemory == VK_NULL_HANDLE) ||
2206  (m_Size == 0) ||
2207  m_Suballocations.empty())
2208  {
2209  return false;
2210  }
2211 
2212  // Expected offset of new suballocation as calculates from previous ones.
2213  VkDeviceSize calculatedOffset = 0;
2214  // Expected number of free suballocations as calculated from traversing their list.
2215  uint32_t calculatedFreeCount = 0;
2216  // Expected sum size of free suballocations as calculated from traversing their list.
2217  VkDeviceSize calculatedSumFreeSize = 0;
2218  // Expected number of free suballocations that should be registered in
2219  // m_FreeSuballocationsBySize calculated from traversing their list.
2220  size_t freeSuballocationsToRegister = 0;
2221  // True if previous visisted suballocation was free.
2222  bool prevFree = false;
2223 
2224  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2225  suballocItem != m_Suballocations.cend();
2226  ++suballocItem)
2227  {
2228  const VmaSuballocation& subAlloc = *suballocItem;
2229 
2230  // Actual offset of this suballocation doesn't match expected one.
2231  if(subAlloc.offset != calculatedOffset)
2232  return false;
2233 
2234  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
2235  // Two adjacent free suballocations are invalid. They should be merged.
2236  if(prevFree && currFree)
2237  return false;
2238  prevFree = currFree;
2239 
2240  if(currFree)
2241  {
2242  calculatedSumFreeSize += subAlloc.size;
2243  ++calculatedFreeCount;
2244  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2245  ++freeSuballocationsToRegister;
2246  }
2247 
2248  calculatedOffset += subAlloc.size;
2249  }
2250 
2251  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
2252  // match expected one.
2253  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
2254  return false;
2255 
2256  VkDeviceSize lastSize = 0;
2257  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
2258  {
2259  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
2260 
2261  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
2262  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
2263  return false;
2264  // They must be sorted by size ascending.
2265  if(suballocItem->size < lastSize)
2266  return false;
2267 
2268  lastSize = suballocItem->size;
2269  }
2270 
2271  // Check if totals match calculacted values.
2272  return
2273  (calculatedOffset == m_Size) &&
2274  (calculatedSumFreeSize == m_SumFreeSize) &&
2275  (calculatedFreeCount == m_FreeCount);
2276 }
2277 
2278 /*
2279 How many suitable free suballocations to analyze before choosing best one.
2280 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
2281  be chosen.
2282 - Set to UINT_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
2283  suballocations will be analized and best one will be chosen.
2284 - Any other value is also acceptable.
2285 */
2286 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
2287 
2288 bool VmaAllocation::CreateAllocationRequest(
2289  VkDeviceSize bufferImageGranularity,
2290  VkDeviceSize allocSize,
2291  VkDeviceSize allocAlignment,
2292  VmaSuballocationType allocType,
2293  VmaAllocationRequest* pAllocationRequest)
2294 {
2295  VMA_ASSERT(allocSize > 0);
2296  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2297  VMA_ASSERT(pAllocationRequest != VMA_NULL);
2298  VMA_HEAVY_ASSERT(Validate());
2299 
2300  // There is not enough total free space in this allocation to fullfill the request: Early return.
2301  if(m_SumFreeSize < allocSize)
2302  return false;
2303 
2304  bool found = false;
2305 
2306  // Old brute-force algorithm, linearly searching suballocations.
2307  /*
2308  uint32_t suitableSuballocationsFound = 0;
2309  for(VmaSuballocationList::iterator suballocItem = suballocations.Front();
2310  suballocItem != VMA_NULL &&
2311  suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK;
2312  suballocItem = suballocItem->Next)
2313  {
2314  if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE)
2315  {
2316  VkDeviceSize offset = 0, cost = 0;
2317  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost))
2318  {
2319  ++suitableSuballocationsFound;
2320  if(cost < costLimit)
2321  {
2322  pAllocationRequest->freeSuballocationItem = suballocItem;
2323  pAllocationRequest->offset = offset;
2324  pAllocationRequest->cost = cost;
2325  if(cost == 0)
2326  return true;
2327  costLimit = cost;
2328  betterSuballocationFound = true;
2329  }
2330  }
2331  }
2332  }
2333  */
2334 
2335  // New algorithm, efficiently searching freeSuballocationsBySize.
2336  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
2337  if(freeSuballocCount > 0)
2338  {
2339  if(VMA_BEST_FIT)
2340  {
2341  // Find first free suballocation with size not less than allocSize.
2342  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2343  m_FreeSuballocationsBySize.data(),
2344  m_FreeSuballocationsBySize.data() + freeSuballocCount,
2345  allocSize,
2346  VmaSuballocationItemSizeLess());
2347  size_t index = it - m_FreeSuballocationsBySize.data();
2348  for(; index < freeSuballocCount; ++index)
2349  {
2350  VkDeviceSize offset = 0;
2351  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
2352  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
2353  {
2354  pAllocationRequest->freeSuballocationItem = suballocItem;
2355  pAllocationRequest->offset = offset;
2356  return true;
2357  }
2358  }
2359  }
2360  else
2361  {
2362  // Search staring from biggest suballocations.
2363  for(size_t index = freeSuballocCount; index--; )
2364  {
2365  VkDeviceSize offset = 0;
2366  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
2367  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
2368  {
2369  pAllocationRequest->freeSuballocationItem = suballocItem;
2370  pAllocationRequest->offset = offset;
2371  return true;
2372  }
2373  }
2374  }
2375  }
2376 
2377  return false;
2378 }
2379 
2380 bool VmaAllocation::CheckAllocation(
2381  VkDeviceSize bufferImageGranularity,
2382  VkDeviceSize allocSize,
2383  VkDeviceSize allocAlignment,
2384  VmaSuballocationType allocType,
2385  VmaSuballocationList::const_iterator freeSuballocItem,
2386  VkDeviceSize* pOffset) const
2387 {
2388  VMA_ASSERT(allocSize > 0);
2389  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2390  VMA_ASSERT(freeSuballocItem != m_Suballocations.cend());
2391  VMA_ASSERT(pOffset != VMA_NULL);
2392 
2393  const VmaSuballocation& suballoc = *freeSuballocItem;
2394  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
2395 
2396  // Size of this suballocation is too small for this request: Early return.
2397  if(suballoc.size < allocSize)
2398  return false;
2399 
2400  // Start from offset equal to beginning of this suballocation.
2401  *pOffset = suballoc.offset;
2402 
2403  // Apply VMA_DEBUG_MARGIN at the beginning.
2404  if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin())
2405  *pOffset += VMA_DEBUG_MARGIN;
2406 
2407  // Apply alignment.
2408  const VkDeviceSize alignment = VMA_MAX(allocAlignment, VMA_DEBUG_ALIGNMENT);
2409  *pOffset = VmaAlignUp(*pOffset, alignment);
2410 
2411  // Check previous suballocations for BufferImageGranularity conflicts.
2412  // Make bigger alignment if necessary.
2413  if(bufferImageGranularity > 1)
2414  {
2415  bool bufferImageGranularityConflict = false;
2416  VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem;
2417  while(prevSuballocItem != m_Suballocations.cbegin())
2418  {
2419  --prevSuballocItem;
2420  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
2421  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
2422  {
2423  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
2424  {
2425  bufferImageGranularityConflict = true;
2426  break;
2427  }
2428  }
2429  else
2430  // Already on previous page.
2431  break;
2432  }
2433  if(bufferImageGranularityConflict)
2434  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
2435  }
2436 
2437  // Calculate padding at the beginning based on current offset.
2438  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
2439 
2440  // Calculate required margin at the end if this is not last suballocation.
2441  VmaSuballocationList::const_iterator next = freeSuballocItem;
2442  ++next;
2443  const VkDeviceSize requiredEndMargin =
2444  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
2445 
2446  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
2447  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
2448  return false;
2449 
2450  // Check next suballocations for BufferImageGranularity conflicts.
2451  // If conflict exists, allocation cannot be made here.
2452  if(bufferImageGranularity > 1)
2453  {
2454  VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem;
2455  ++nextSuballocItem;
2456  while(nextSuballocItem != m_Suballocations.cend())
2457  {
2458  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
2459  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
2460  {
2461  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
2462  return false;
2463  }
2464  else
2465  // Already on next page.
2466  break;
2467  ++nextSuballocItem;
2468  }
2469  }
2470 
2471  // All tests passed: Success. pOffset is already filled.
2472  return true;
2473 }
2474 
2475 bool VmaAllocation::IsEmpty() const
2476 {
2477  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
2478 }
2479 
2480 void VmaAllocation::Alloc(
2481  const VmaAllocationRequest& request,
2482  VmaSuballocationType type,
2483  VkDeviceSize allocSize)
2484 {
2485  VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end());
2486  VmaSuballocation& suballoc = *request.freeSuballocationItem;
2487  // Given suballocation is a free block.
2488  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
2489  // Given offset is inside this suballocation.
2490  VMA_ASSERT(request.offset >= suballoc.offset);
2491  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
2492  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
2493  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
2494 
2495  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
2496  // it to become used.
2497  UnregisterFreeSuballocation(request.freeSuballocationItem);
2498 
2499  suballoc.offset = request.offset;
2500  suballoc.size = allocSize;
2501  suballoc.type = type;
2502 
2503  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
2504  if(paddingEnd)
2505  {
2506  VmaSuballocation paddingSuballoc = {};
2507  paddingSuballoc.offset = request.offset + allocSize;
2508  paddingSuballoc.size = paddingEnd;
2509  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2510  VmaSuballocationList::iterator next = request.freeSuballocationItem;
2511  ++next;
2512  const VmaSuballocationList::iterator paddingEndItem =
2513  m_Suballocations.insert(next, paddingSuballoc);
2514  RegisterFreeSuballocation(paddingEndItem);
2515  }
2516 
2517  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
2518  if(paddingBegin)
2519  {
2520  VmaSuballocation paddingSuballoc = {};
2521  paddingSuballoc.offset = request.offset - paddingBegin;
2522  paddingSuballoc.size = paddingBegin;
2523  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2524  const VmaSuballocationList::iterator paddingBeginItem =
2525  m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc);
2526  RegisterFreeSuballocation(paddingBeginItem);
2527  }
2528 
2529  // Update totals.
2530  m_FreeCount = m_FreeCount - 1;
2531  if(paddingBegin > 0)
2532  ++m_FreeCount;
2533  if(paddingEnd > 0)
2534  ++m_FreeCount;
2535  m_SumFreeSize -= allocSize;
2536 }
2537 
2538 void VmaAllocation::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
2539 {
2540  // Change this suballocation to be marked as free.
2541  VmaSuballocation& suballoc = *suballocItem;
2542  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2543 
2544  // Update totals.
2545  ++m_FreeCount;
2546  m_SumFreeSize += suballoc.size;
2547 
2548  // Merge with previous and/or next suballocation if it's also free.
2549  bool mergeWithNext = false;
2550  bool mergeWithPrev = false;
2551 
2552  VmaSuballocationList::iterator nextItem = suballocItem;
2553  ++nextItem;
2554  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
2555  mergeWithNext = true;
2556 
2557  VmaSuballocationList::iterator prevItem = suballocItem;
2558  if(suballocItem != m_Suballocations.begin())
2559  {
2560  --prevItem;
2561  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
2562  mergeWithPrev = true;
2563  }
2564 
2565  if(mergeWithNext)
2566  {
2567  UnregisterFreeSuballocation(nextItem);
2568  MergeFreeWithNext(suballocItem);
2569  }
2570 
2571  if(mergeWithPrev)
2572  {
2573  UnregisterFreeSuballocation(prevItem);
2574  MergeFreeWithNext(prevItem);
2575  RegisterFreeSuballocation(prevItem);
2576  }
2577  else
2578  RegisterFreeSuballocation(suballocItem);
2579 }
2580 
2581 void VmaAllocation::Free(const VkMappedMemoryRange* pMemory)
2582 {
2583  // If suballocation to free has offset smaller than half of allocation size, search forward.
2584  // Otherwise search backward.
2585  const bool forwardDirection = pMemory->offset < (m_Size / 2);
2586  if(forwardDirection)
2587  {
2588  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
2589  suballocItem != m_Suballocations.end();
2590  ++suballocItem)
2591  {
2592  VmaSuballocation& suballoc = *suballocItem;
2593  if(suballoc.offset == pMemory->offset)
2594  {
2595  FreeSuballocation(suballocItem);
2596  VMA_HEAVY_ASSERT(Validate());
2597  return;
2598  }
2599  }
2600  VMA_ASSERT(0 && "Not found!");
2601  }
2602  else
2603  {
2604  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
2605  suballocItem != m_Suballocations.end();
2606  ++suballocItem)
2607  {
2608  VmaSuballocation& suballoc = *suballocItem;
2609  if(suballoc.offset == pMemory->offset)
2610  {
2611  FreeSuballocation(suballocItem);
2612  VMA_HEAVY_ASSERT(Validate());
2613  return;
2614  }
2615  }
2616  VMA_ASSERT(0 && "Not found!");
2617  }
2618 }
2619 
2620 #if VMA_STATS_STRING_ENABLED
2621 
2622 void VmaAllocation::PrintDetailedMap(class VmaStringBuilder& sb) const
2623 {
2624  sb.Add("{\n\t\t\t\"Bytes\": ");
2625  sb.AddNumber(m_Size);
2626  sb.Add(",\n\t\t\t\"FreeBytes\": ");
2627  sb.AddNumber(m_SumFreeSize);
2628  sb.Add(",\n\t\t\t\"Suballocations\": ");
2629  sb.AddNumber(m_Suballocations.size());
2630  sb.Add(",\n\t\t\t\"FreeSuballocations\": ");
2631  sb.AddNumber(m_FreeCount);
2632  sb.Add(",\n\t\t\t\"SuballocationList\": [");
2633 
2634  size_t i = 0;
2635  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2636  suballocItem != m_Suballocations.cend();
2637  ++suballocItem, ++i)
2638  {
2639  if(i > 0)
2640  sb.Add(",\n\t\t\t\t{ \"Type\": ");
2641  else
2642  sb.Add("\n\t\t\t\t{ \"Type\": ");
2643  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
2644  sb.Add(", \"Size\": ");
2645  sb.AddNumber(suballocItem->size);
2646  sb.Add(", \"Offset\": ");
2647  sb.AddNumber(suballocItem->offset);
2648  sb.Add(" }");
2649  }
2650 
2651  sb.Add("\n\t\t\t]\n\t\t}");
2652 }
2653 
2654 #endif // #if VMA_STATS_STRING_ENABLED
2655 
2656 void VmaAllocation::MergeFreeWithNext(VmaSuballocationList::iterator item)
2657 {
2658  VMA_ASSERT(item != m_Suballocations.end());
2659  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2660 
2661  VmaSuballocationList::iterator nextItem = item;
2662  ++nextItem;
2663  VMA_ASSERT(nextItem != m_Suballocations.end());
2664  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
2665 
2666  item->size += nextItem->size;
2667  --m_FreeCount;
2668  m_Suballocations.erase(nextItem);
2669 }
2670 
2671 void VmaAllocation::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
2672 {
2673  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2674  VMA_ASSERT(item->size > 0);
2675 
2676  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2677  {
2678  if(m_FreeSuballocationsBySize.empty())
2679  m_FreeSuballocationsBySize.push_back(item);
2680  else
2681  {
2682  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2683  m_FreeSuballocationsBySize.data(),
2684  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
2685  item,
2686  VmaSuballocationItemSizeLess());
2687  size_t index = it - m_FreeSuballocationsBySize.data();
2688  VectorInsert(m_FreeSuballocationsBySize, index, item);
2689  }
2690  }
2691 }
2692 
2693 void VmaAllocation::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
2694 {
2695  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2696  VMA_ASSERT(item->size > 0);
2697 
2698  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2699  {
2700  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2701  m_FreeSuballocationsBySize.data(),
2702  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
2703  item,
2704  VmaSuballocationItemSizeLess());
2705  for(size_t index = it - m_FreeSuballocationsBySize.data();
2706  index < m_FreeSuballocationsBySize.size();
2707  ++index)
2708  {
2709  if(m_FreeSuballocationsBySize[index] == item)
2710  {
2711  VectorRemove(m_FreeSuballocationsBySize, index);
2712  return;
2713  }
2714  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
2715  }
2716  VMA_ASSERT(0 && "Not found.");
2717  }
2718 }
2719 
2720 static void InitStatInfo(VmaStatInfo& outInfo)
2721 {
2722  memset(&outInfo, 0, sizeof(outInfo));
2723  outInfo.SuballocationSizeMin = UINT64_MAX;
2724  outInfo.UnusedRangeSizeMin = UINT64_MAX;
2725 }
2726 
2727 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaAllocation& alloc)
2728 {
2729  outInfo.AllocationCount = 1;
2730 
2731  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
2732  outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount;
2733  outInfo.UnusedRangeCount = alloc.m_FreeCount;
2734 
2735  outInfo.UnusedBytes = alloc.m_SumFreeSize;
2736  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
2737 
2738  outInfo.SuballocationSizeMin = UINT64_MAX;
2739  outInfo.SuballocationSizeMax = 0;
2740  outInfo.UnusedRangeSizeMin = UINT64_MAX;
2741  outInfo.UnusedRangeSizeMax = 0;
2742 
2743  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
2744  suballocItem != alloc.m_Suballocations.cend();
2745  ++suballocItem)
2746  {
2747  const VmaSuballocation& suballoc = *suballocItem;
2748  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
2749  {
2750  outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size);
2751  outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size);
2752  }
2753  else
2754  {
2755  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
2756  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
2757  }
2758  }
2759 }
2760 
2761 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
2762 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
2763 {
2764  inoutInfo.AllocationCount += srcInfo.AllocationCount;
2765  inoutInfo.SuballocationCount += srcInfo.SuballocationCount;
2766  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
2767  inoutInfo.UsedBytes += srcInfo.UsedBytes;
2768  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
2769  inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin);
2770  inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax);
2771  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
2772  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
2773 }
2774 
2775 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
2776 {
2777  inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ?
2778  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0;
2779  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
2780  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
2781 }
2782 
2783 VmaAllocationVector::VmaAllocationVector(VmaAllocator hAllocator) :
2784  m_hAllocator(hAllocator),
2785  m_Allocations(VmaStlAllocator<VmaAllocation*>(hAllocator->GetAllocationCallbacks()))
2786 {
2787 }
2788 
2789 VmaAllocationVector::~VmaAllocationVector()
2790 {
2791  for(size_t i = m_Allocations.size(); i--; )
2792  {
2793  m_Allocations[i]->Destroy(m_hAllocator);
2794  vma_delete(m_hAllocator, m_Allocations[i]);
2795  }
2796 }
2797 
2798 size_t VmaAllocationVector::Free(const VkMappedMemoryRange* pMemory)
2799 {
2800  for(uint32_t allocIndex = 0; allocIndex < m_Allocations.size(); ++allocIndex)
2801  {
2802  VmaAllocation* const pAlloc = m_Allocations[allocIndex];
2803  VMA_ASSERT(pAlloc);
2804  if(pAlloc->m_hMemory == pMemory->memory)
2805  {
2806  pAlloc->Free(pMemory);
2807  VMA_HEAVY_ASSERT(pAlloc->Validate());
2808  return allocIndex;
2809  }
2810  }
2811 
2812  return (size_t)-1;
2813 }
2814 
2815 void VmaAllocationVector::IncrementallySortAllocations()
2816 {
2817  // Bubble sort only until first swap.
2818  for(size_t i = 1; i < m_Allocations.size(); ++i)
2819  {
2820  if(m_Allocations[i - 1]->m_SumFreeSize > m_Allocations[i]->m_SumFreeSize)
2821  {
2822  VMA_SWAP(m_Allocations[i - 1], m_Allocations[i]);
2823  return;
2824  }
2825  }
2826 }
2827 
2828 #if VMA_STATS_STRING_ENABLED
2829 
2830 void VmaAllocationVector::PrintDetailedMap(class VmaStringBuilder& sb) const
2831 {
2832  for(size_t i = 0; i < m_Allocations.size(); ++i)
2833  {
2834  if(i > 0)
2835  sb.Add(",\n\t\t");
2836  else
2837  sb.Add("\n\t\t");
2838  m_Allocations[i]->PrintDetailedMap(sb);
2839  }
2840 }
2841 
2842 #endif // #if VMA_STATS_STRING_ENABLED
2843 
2844 void VmaAllocationVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const
2845 {
2846  for(uint32_t allocIndex = 0; allocIndex < m_Allocations.size(); ++allocIndex)
2847  {
2848  const VmaAllocation* const pAlloc = m_Allocations[allocIndex];
2849  VMA_ASSERT(pAlloc);
2850  VMA_HEAVY_ASSERT(pAlloc->Validate());
2851  VmaStatInfo allocationStatInfo;
2852  CalcAllocationStatInfo(allocationStatInfo, *pAlloc);
2853  VmaAddStatInfo(pStats->total, allocationStatInfo);
2854  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
2855  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
2856  }
2857 }
2858 
2860 // VmaAllocator_T
2861 
2862 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
2863  m_PhysicalDevice(pCreateInfo->physicalDevice),
2864  m_hDevice(pCreateInfo->device),
2865  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
2866  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
2867  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
2868  m_PreferredLargeHeapBlockSize(0),
2869  m_PreferredSmallHeapBlockSize(0),
2870  m_BufferToMemoryMap(VmaStlAllocator< VmaPair<VkBuffer, VkMappedMemoryRange> >(pCreateInfo->pAllocationCallbacks)),
2871  m_ImageToMemoryMap(VmaStlAllocator< VmaPair<VkImage, VkMappedMemoryRange> >(pCreateInfo->pAllocationCallbacks))
2872 {
2873  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
2874 
2875  memset(&m_MemProps, 0, sizeof(m_MemProps));
2876  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
2877 
2878  memset(&m_pAllocations, 0, sizeof(m_pAllocations));
2879  memset(&m_HasEmptyAllocation, 0, sizeof(m_HasEmptyAllocation));
2880  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
2881 
2882  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
2883  pCreateInfo->preferredLargeHeapBlockSize : VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE;
2884  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
2885  pCreateInfo->preferredSmallHeapBlockSize : VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE;
2886 
2887  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
2888  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
2889 
2890  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
2891  {
2892  m_pAllocations[i] = vma_new(this, VmaAllocationVector)(this);
2893  m_pOwnAllocations[i] = vma_new(this, OwnAllocationVectorType)(VmaStlAllocator<VmaOwnAllocation>(GetAllocationCallbacks()));
2894  }
2895 }
2896 
2897 VmaAllocator_T::~VmaAllocator_T()
2898 {
2899  for(VMA_MAP_TYPE(VkImage, VkMappedMemoryRange)::iterator it = m_ImageToMemoryMap.begin();
2900  it != m_ImageToMemoryMap.end();
2901  ++it)
2902  {
2903  vkDestroyImage(m_hDevice, it->first, GetAllocationCallbacks());
2904  }
2905 
2906  for(VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange)::iterator it = m_BufferToMemoryMap.begin();
2907  it != m_BufferToMemoryMap.end();
2908  ++it)
2909  {
2910  vkDestroyBuffer(m_hDevice, it->first, GetAllocationCallbacks());
2911  }
2912 
2913  for(uint32_t typeIndex = 0; typeIndex < GetMemoryTypeCount(); ++typeIndex)
2914  {
2915  OwnAllocationVectorType* pOwnAllocations = m_pOwnAllocations[typeIndex];
2916  VMA_ASSERT(pOwnAllocations);
2917  for(size_t allocationIndex = 0; allocationIndex < pOwnAllocations->size(); ++allocationIndex)
2918  {
2919  const VmaOwnAllocation& ownAlloc = (*pOwnAllocations)[allocationIndex];
2920  vkFreeMemory(m_hDevice, ownAlloc.m_hMemory, GetAllocationCallbacks());
2921  }
2922  }
2923 
2924  for(size_t i = GetMemoryTypeCount(); i--; )
2925  {
2926  vma_delete(this, m_pAllocations[i]);
2927  vma_delete(this, m_pOwnAllocations[i]);
2928  }
2929 }
2930 
2931 VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const
2932 {
2933  VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
2934  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
2935  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
2936 }
2937 
2938 VkResult VmaAllocator_T::AllocateMemoryOfType(
2939  const VkMemoryRequirements& vkMemReq,
2940  const VmaMemoryRequirements& vmaMemReq,
2941  uint32_t memTypeIndex,
2942  VmaSuballocationType suballocType,
2943  VkMappedMemoryRange* pMemory)
2944 {
2945  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
2946 
2947  pMemory->sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
2948  pMemory->pNext = VMA_NULL;
2949  pMemory->size = vkMemReq.size;
2950 
2951  const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex);
2952  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
2953  const bool ownMemory =
2954  vmaMemReq.ownMemory ||
2955  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
2956  ((vmaMemReq.neverAllocate == false) && (vkMemReq.size > preferredBlockSize / 2));
2957 
2958  if(ownMemory)
2959  {
2960  if(vmaMemReq.neverAllocate)
2961  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
2962  else
2963  return AllocateOwnMemory(vkMemReq.size, suballocType, memTypeIndex, pMemory);
2964  }
2965  else
2966  {
2967  VmaMutexLock lock(m_AllocationsMutex[memTypeIndex]);
2968  VmaAllocationVector* const allocationVector = m_pAllocations[memTypeIndex];
2969  VMA_ASSERT(allocationVector);
2970 
2971  // 1. Search existing allocations.
2972  // Forward order - prefer blocks with smallest amount of free space.
2973  for(size_t allocIndex = 0; allocIndex < allocationVector->m_Allocations.size(); ++allocIndex )
2974  {
2975  VmaAllocation* const pAlloc = allocationVector->m_Allocations[allocIndex];
2976  VMA_ASSERT(pAlloc);
2977  VmaAllocationRequest allocRequest = {};
2978  // Check if can allocate from pAlloc.
2979  if(pAlloc->CreateAllocationRequest(
2980  GetBufferImageGranularity(),
2981  vkMemReq.size,
2982  vkMemReq.alignment,
2983  suballocType,
2984  &allocRequest))
2985  {
2986  // We no longer have an empty Allocation.
2987  if(pAlloc->IsEmpty())
2988  m_HasEmptyAllocation[memTypeIndex] = false;
2989  // Allocate from this pAlloc.
2990  pAlloc->Alloc(allocRequest, suballocType, vkMemReq.size);
2991  // Return VkDeviceMemory and offset (size already filled above).
2992  pMemory->memory = pAlloc->m_hMemory;
2993  pMemory->offset = allocRequest.offset;
2994  VMA_HEAVY_ASSERT(pAlloc->Validate());
2995  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex);
2996  return VK_SUCCESS;
2997  }
2998  }
2999 
3000  // 2. Create new Allocation.
3001  if(vmaMemReq.neverAllocate)
3002  {
3003  VMA_DEBUG_LOG(" FAILED due to VmaMemoryRequirements::neverAllocate");
3004  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3005  }
3006  else
3007  {
3008  // Start with full preferredBlockSize.
3009  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
3010  allocInfo.memoryTypeIndex = memTypeIndex;
3011  allocInfo.allocationSize = preferredBlockSize;
3012  VkDeviceMemory mem = VK_NULL_HANDLE;
3013  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3014  if(res < 0)
3015  {
3016  // 3. Try half the size.
3017  allocInfo.allocationSize /= 2;
3018  if(allocInfo.allocationSize >= vkMemReq.size)
3019  {
3020  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3021  if(res < 0)
3022  {
3023  // 4. Try quarter the size.
3024  allocInfo.allocationSize /= 2;
3025  if(allocInfo.allocationSize >= vkMemReq.size)
3026  {
3027  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3028  }
3029  }
3030  }
3031  }
3032  if(res < 0)
3033  {
3034  // 5. Try OwnAlloc.
3035  res = AllocateOwnMemory(vkMemReq.size, suballocType, memTypeIndex, pMemory);
3036  if(res == VK_SUCCESS)
3037  {
3038  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
3039  VMA_DEBUG_LOG(" Allocated as OwnMemory");
3040  return VK_SUCCESS;
3041  }
3042  else
3043  {
3044  // Everything failed: Return error code.
3045  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
3046  return res;
3047  }
3048  }
3049 
3050  // New VkDeviceMemory successfully created. Create new Allocation for it.
3051  VmaAllocation* const pAlloc = vma_new(this, VmaAllocation)(this);
3052  pAlloc->Init(mem, allocInfo.allocationSize);
3053 
3054  allocationVector->m_Allocations.push_back(pAlloc);
3055 
3056  // Allocate from pAlloc. Because it is empty, allocRequest can be trivially filled.
3057  VmaAllocationRequest allocRequest = {};
3058  allocRequest.freeSuballocationItem = pAlloc->m_Suballocations.begin();
3059  allocRequest.offset = 0;
3060  pAlloc->Alloc(allocRequest, suballocType, vkMemReq.size);
3061  pMemory->memory = mem;
3062  pMemory->offset = allocRequest.offset;
3063  VMA_HEAVY_ASSERT(pAlloc->Validate());
3064  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
3065  return VK_SUCCESS;
3066  }
3067  }
3068 }
3069 
3070 VkResult VmaAllocator_T::AllocateOwnMemory(
3071  VkDeviceSize size,
3072  VmaSuballocationType suballocType,
3073  uint32_t memTypeIndex,
3074  VkMappedMemoryRange* pMemory)
3075 {
3076  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
3077  allocInfo.memoryTypeIndex = memTypeIndex;
3078  allocInfo.allocationSize = size;
3079 
3080  // Allocate VkDeviceMemory.
3081  VmaOwnAllocation ownAlloc = {};
3082  ownAlloc.m_Size = size;
3083  ownAlloc.m_Type = suballocType;
3084  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &ownAlloc.m_hMemory);
3085  if(res < 0)
3086  {
3087  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
3088  return res;
3089  }
3090 
3091  // Register it in m_pOwnAllocations.
3092  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex]);
3093  OwnAllocationVectorType* ownAllocations = m_pOwnAllocations[memTypeIndex];
3094  VMA_ASSERT(ownAllocations);
3095  VmaOwnAllocation* const pOwnAllocationsBeg = ownAllocations->data();
3096  VmaOwnAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + ownAllocations->size();
3097  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3098  pOwnAllocationsBeg,
3099  pOwnAllocationsEnd,
3100  ownAlloc,
3101  VmaOwnAllocationMemoryHandleLess()) - pOwnAllocationsBeg;
3102  VectorInsert(*ownAllocations, indexToInsert, ownAlloc);
3103 
3104  // Return parameters of the allocation.
3105  pMemory->sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
3106  pMemory->pNext = VMA_NULL;
3107  pMemory->memory = ownAlloc.m_hMemory;
3108  pMemory->offset = 0;
3109  pMemory->size = size;
3110 
3111  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
3112 
3113  return VK_SUCCESS;
3114 }
3115 
3116 VkResult VmaAllocator_T::AllocateMemory(
3117  const VkMemoryRequirements& vkMemReq,
3118  const VmaMemoryRequirements& vmaMemReq,
3119  VmaSuballocationType suballocType,
3120  VkMappedMemoryRange* pMemory,
3121  uint32_t* pMemoryTypeIndex)
3122 {
3123  if(vmaMemReq.ownMemory && vmaMemReq.neverAllocate)
3124  {
3125  VMA_ASSERT(0 && "Specifying VmaMemoryRequirements::ownMemory && VmaMemoryRequirements::neverAllocate makes no sense.");
3126  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3127  }
3128 
3129  // Bit mask of memory Vulkan types acceptable for this allocation.
3130  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
3131  uint32_t memTypeIndex = UINT_MAX;
3132  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
3133  if(res == VK_SUCCESS)
3134  {
3135  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pMemory);
3136  // Succeeded on first try.
3137  if(res == VK_SUCCESS)
3138  {
3139  if(pMemoryTypeIndex != VMA_NULL)
3140  *pMemoryTypeIndex = memTypeIndex;
3141  return res;
3142  }
3143  // Allocation from this memory type failed. Try other compatible memory types.
3144  else
3145  {
3146  for(;;)
3147  {
3148  // Remove old memTypeIndex from list of possibilities.
3149  memoryTypeBits &= ~(1u << memTypeIndex);
3150  // Find alternative memTypeIndex.
3151  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
3152  if(res == VK_SUCCESS)
3153  {
3154  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pMemory);
3155  // Allocation from this alternative memory type succeeded.
3156  if(res == VK_SUCCESS)
3157  {
3158  if(pMemoryTypeIndex != VMA_NULL)
3159  *pMemoryTypeIndex = memTypeIndex;
3160  return res;
3161  }
3162  // else: Allocation from this memory type failed. Try next one - next loop iteration.
3163  }
3164  // No other matching memory type index could be found.
3165  else
3166  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
3167  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3168  }
3169  }
3170  }
3171  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
3172  else
3173  return res;
3174 }
3175 
3176 void VmaAllocator_T::FreeMemory(const VkMappedMemoryRange* pMemory)
3177 {
3178  uint32_t memTypeIndex = 0;
3179  bool found = false;
3180  VmaAllocation* allocationToDelete = VMA_NULL;
3181  // Check all memory types because we don't know which one does pMemory come from.
3182  for(; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3183  {
3184  VmaMutexLock lock(m_AllocationsMutex[memTypeIndex]);
3185  VmaAllocationVector* const pAllocationVector = m_pAllocations[memTypeIndex];
3186  VMA_ASSERT(pAllocationVector);
3187  // Try to free pMemory from pAllocationVector.
3188  const size_t allocIndex = pAllocationVector->Free(pMemory);
3189  if(allocIndex != (size_t)-1)
3190  {
3191  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
3192  found = true;
3193  VmaAllocation* const pAlloc = pAllocationVector->m_Allocations[allocIndex];
3194  VMA_ASSERT(pAlloc);
3195  // pAlloc became empty after this deallocation.
3196  if(pAlloc->IsEmpty())
3197  {
3198  // Already has empty Allocation. We don't want to have two, so delete this one.
3199  if(m_HasEmptyAllocation[memTypeIndex])
3200  {
3201  allocationToDelete = pAlloc;
3202  VectorRemove(pAllocationVector->m_Allocations, allocIndex);
3203  break;
3204  }
3205  // We now have first empty Allocation.
3206  else
3207  m_HasEmptyAllocation[memTypeIndex] = true;
3208  }
3209  // Must be called after allocIndex is used, because later it may become invalid!
3210  pAllocationVector->IncrementallySortAllocations();
3211  break;
3212  }
3213  }
3214  if(found)
3215  {
3216  // Destruction of a free Allocation. Deferred until this point, outside of mutex
3217  // lock, for performance reason.
3218  if(allocationToDelete != VMA_NULL)
3219  {
3220  VMA_DEBUG_LOG(" Deleted empty allocation");
3221  allocationToDelete->Destroy(this);
3222  vma_delete(this, allocationToDelete);
3223  }
3224  return;
3225  }
3226 
3227  // pMemory not found in allocations. Try free it as Own Memory.
3228  if(FreeOwnMemory(pMemory))
3229  return;
3230 
3231  // pMemory not found as Own Memory either.
3232  VMA_ASSERT(0 && "Not found. Trying to free memory not allocated using this allocator (or some other bug).");
3233 }
3234 
3235 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
3236 {
3237  InitStatInfo(pStats->total);
3238  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
3239  InitStatInfo(pStats->memoryType[i]);
3240  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
3241  InitStatInfo(pStats->memoryHeap[i]);
3242 
3243  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3244  {
3245  VmaMutexLock allocationsLock(m_AllocationsMutex[memTypeIndex]);
3246  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3247  const VmaAllocationVector* const allocVector = m_pAllocations[memTypeIndex];
3248  VMA_ASSERT(allocVector);
3249  allocVector->AddStats(pStats, memTypeIndex, heapIndex);
3250  }
3251 
3252  VmaPostprocessCalcStatInfo(pStats->total);
3253  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
3254  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
3255  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
3256  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
3257 }
3258 
3259 bool VmaAllocator_T::FreeOwnMemory(const VkMappedMemoryRange* pMemory)
3260 {
3261  VkDeviceMemory vkMemory = VK_NULL_HANDLE;
3262 
3263  // Check all memory types because we don't know which one does pMemory come from.
3264  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3265  {
3266  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex]);
3267  OwnAllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex];
3268  VMA_ASSERT(pOwnAllocations);
3269  VmaOwnAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
3270  VmaOwnAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
3271  VmaOwnAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess(
3272  pOwnAllocationsBeg,
3273  pOwnAllocationsEnd,
3274  pMemory->memory,
3275  VmaOwnAllocationMemoryHandleLess());
3276  if((pOwnAllocationIt != pOwnAllocationsEnd) &&
3277  (pOwnAllocationIt->m_hMemory == pMemory->memory))
3278  {
3279  VMA_ASSERT(pMemory->size == pOwnAllocationIt->m_Size && pMemory->offset == 0);
3280  vkMemory = pOwnAllocationIt->m_hMemory;
3281  const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg;
3282  VectorRemove(*pOwnAllocations, ownAllocationIndex);
3283  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
3284  break;
3285  }
3286  }
3287 
3288  // Found. Free VkDeviceMemory deferred until this point, outside of mutex lock,
3289  // for performance reason.
3290  if(vkMemory != VK_NULL_HANDLE)
3291  {
3292  vkFreeMemory(m_hDevice, vkMemory, GetAllocationCallbacks());
3293  return true;
3294  }
3295  else
3296  return false;
3297 }
3298 
3299 #if VMA_STATS_STRING_ENABLED
3300 
3301 void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb)
3302 {
3303  bool ownAllocationsStarted = false;
3304  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3305  {
3306  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex]);
3307  OwnAllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex];
3308  VMA_ASSERT(pOwnAllocVector);
3309  if(pOwnAllocVector->empty() == false)
3310  {
3311  if(ownAllocationsStarted)
3312  sb.Add(",\n\t\"Type ");
3313  else
3314  {
3315  sb.Add(",\n\"OwnAllocations\": {\n\t\"Type ");
3316  ownAllocationsStarted = true;
3317  }
3318  sb.AddNumber(memTypeIndex);
3319  sb.Add("\": [");
3320 
3321  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
3322  {
3323  const VmaOwnAllocation& ownAlloc = (*pOwnAllocVector)[i];
3324  if(i > 0)
3325  sb.Add(",\n\t\t{ \"Size\": ");
3326  else
3327  sb.Add("\n\t\t{ \"Size\": ");
3328  sb.AddNumber(ownAlloc.m_Size);
3329  sb.Add(", \"Type\": ");
3330  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[ownAlloc.m_Type]);
3331  sb.Add(" }");
3332  }
3333 
3334  sb.Add("\n\t]");
3335  }
3336  }
3337  if(ownAllocationsStarted)
3338  sb.Add("\n}");
3339 
3340  {
3341  bool allocationsStarted = false;
3342  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3343  {
3344  VmaMutexLock globalAllocationsLock(m_AllocationsMutex[memTypeIndex]);
3345  if(m_pAllocations[memTypeIndex]->IsEmpty() == false)
3346  {
3347  if(allocationsStarted)
3348  sb.Add(",\n\t\"Type ");
3349  else
3350  {
3351  sb.Add(",\n\"Allocations\": {\n\t\"Type ");
3352  allocationsStarted = true;
3353  }
3354  sb.AddNumber(memTypeIndex);
3355  sb.Add("\": [");
3356 
3357  m_pAllocations[memTypeIndex]->PrintDetailedMap(sb);
3358 
3359  sb.Add("\n\t]");
3360  }
3361  }
3362  if(allocationsStarted)
3363  sb.Add("\n}");
3364  }
3365 }
3366 
3367 #endif // #if VMA_STATS_STRING_ENABLED
3368 
3369 static VkResult AllocateMemoryForImage(
3370  VmaAllocator allocator,
3371  VkImage image,
3372  const VmaMemoryRequirements* pMemoryRequirements,
3373  VmaSuballocationType suballocType,
3374  VkMappedMemoryRange* pMemory,
3375  uint32_t* pMemoryTypeIndex)
3376 {
3377  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pMemory);
3378 
3379  VkMemoryRequirements vkMemReq = {};
3380  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
3381 
3382  return allocator->AllocateMemory(
3383  vkMemReq,
3384  *pMemoryRequirements,
3385  suballocType,
3386  pMemory,
3387  pMemoryTypeIndex);
3388 }
3389 
3391 // Public interface
3392 
3393 VkResult vmaCreateAllocator(
3394  const VmaAllocatorCreateInfo* pCreateInfo,
3395  VmaAllocator* pAllocator)
3396 {
3397  VMA_ASSERT(pCreateInfo && pAllocator);
3398  VMA_DEBUG_LOG("vmaCreateAllocator");
3399  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
3400  return VK_SUCCESS;
3401 }
3402 
3403 void vmaDestroyAllocator(
3404  VmaAllocator allocator)
3405 {
3406  if(allocator != VK_NULL_HANDLE)
3407  {
3408  VMA_DEBUG_LOG("vmaDestroyAllocator");
3409  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
3410  vma_delete(&allocationCallbacks, allocator);
3411  }
3412 }
3413 
3414 void vmaGetPhysicalDeviceProperties(
3415  VmaAllocator allocator,
3416  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
3417 {
3418  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
3419  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
3420 }
3421 
3422 void vmaGetMemoryProperties(
3423  VmaAllocator allocator,
3424  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
3425 {
3426  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
3427  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
3428 }
3429 
3430 void vmaGetMemoryTypeProperties(
3431  VmaAllocator allocator,
3432  uint32_t memoryTypeIndex,
3433  VkMemoryPropertyFlags* pFlags)
3434 {
3435  VMA_ASSERT(allocator && pFlags);
3436  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
3437  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
3438 }
3439 
3440 void vmaCalculateStats(
3441  VmaAllocator allocator,
3442  VmaStats* pStats)
3443 {
3444  VMA_ASSERT(allocator && pStats);
3445  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3446  allocator->CalculateStats(pStats);
3447 }
3448 
3449 #if VMA_STATS_STRING_ENABLED
3450 
3451 void vmaBuildStatsString(
3452  VmaAllocator allocator,
3453  char** ppStatsString,
3454  VkBool32 detailedMap)
3455 {
3456  VMA_ASSERT(allocator && ppStatsString);
3457  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3458 
3459  VmaStringBuilder sb(allocator);
3460  {
3461  VmaStats stats;
3462  allocator->CalculateStats(&stats);
3463 
3464  sb.Add("{\n\"Total\": ");
3465  VmaPrintStatInfo(sb, stats.total);
3466 
3467  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
3468  {
3469  sb.Add(",\n\"Heap ");
3470  sb.AddNumber(heapIndex);
3471  sb.Add("\": {\n\t\"Size\": ");
3472  sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
3473  sb.Add(",\n\t\"Flags\": ");
3474  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
3475  sb.AddString("DEVICE_LOCAL");
3476  else
3477  sb.AddString("");
3478  if(stats.memoryHeap[heapIndex].AllocationCount > 0)
3479  {
3480  sb.Add(",\n\t\"Stats:\": ");
3481  VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]);
3482  }
3483 
3484  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
3485  {
3486  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
3487  {
3488  sb.Add(",\n\t\"Type ");
3489  sb.AddNumber(typeIndex);
3490  sb.Add("\": {\n\t\t\"Flags\": \"");
3491  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
3492  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
3493  sb.Add(" DEVICE_LOCAL");
3494  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
3495  sb.Add(" HOST_VISIBLE");
3496  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
3497  sb.Add(" HOST_COHERENT");
3498  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
3499  sb.Add(" HOST_CACHED");
3500  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
3501  sb.Add(" LAZILY_ALLOCATED");
3502  sb.Add("\"");
3503  if(stats.memoryType[typeIndex].AllocationCount > 0)
3504  {
3505  sb.Add(",\n\t\t\"Stats\": ");
3506  VmaPrintStatInfo(sb, stats.memoryType[typeIndex]);
3507  }
3508  sb.Add("\n\t}");
3509  }
3510  }
3511  sb.Add("\n}");
3512  }
3513  if(detailedMap == VK_TRUE)
3514  allocator->PrintDetailedMap(sb);
3515  sb.Add("\n}\n");
3516  }
3517 
3518  const size_t len = sb.GetLength();
3519  char* const pChars = vma_new_array(allocator, char, len + 1);
3520  if(len > 0)
3521  memcpy(pChars, sb.GetData(), len);
3522  pChars[len] = '\0';
3523  *ppStatsString = pChars;
3524 }
3525 
3526 void vmaFreeStatsString(
3527  VmaAllocator allocator,
3528  char* pStatsString)
3529 {
3530  if(pStatsString != VMA_NULL)
3531  {
3532  VMA_ASSERT(allocator);
3533  size_t len = strlen(pStatsString);
3534  vma_delete_array(allocator, pStatsString, len + 1);
3535  }
3536 }
3537 
3538 #endif // #if VMA_STATS_STRING_ENABLED
3539 
3542 VkResult vmaFindMemoryTypeIndex(
3543  VmaAllocator allocator,
3544  uint32_t memoryTypeBits,
3545  const VmaMemoryRequirements* pMemoryRequirements,
3546  uint32_t* pMemoryTypeIndex)
3547 {
3548  VMA_ASSERT(allocator != VK_NULL_HANDLE);
3549  VMA_ASSERT(pMemoryRequirements != VMA_NULL);
3550  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
3551 
3552  uint32_t requiredFlags = pMemoryRequirements->requiredFlags;
3553  uint32_t preferredFlags = pMemoryRequirements->preferredFlags;
3554  if(preferredFlags == 0)
3555  preferredFlags = requiredFlags;
3556  // preferredFlags, if not 0, must be subset of requiredFlags.
3557  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
3558 
3559  // Convert usage to requiredFlags and preferredFlags.
3560  switch(pMemoryRequirements->usage)
3561  {
3562  case VMA_MEMORY_USAGE_UNKNOWN:
3563  break;
3564  case VMA_MEMORY_USAGE_GPU_ONLY:
3565  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
3566  break;
3567  case VMA_MEMORY_USAGE_CPU_ONLY:
3568  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
3569  break;
3570  case VMA_MEMORY_USAGE_CPU_TO_GPU:
3571  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
3572  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
3573  break;
3574  case VMA_MEMORY_USAGE_GPU_TO_CPU:
3575  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
3576  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
3577  break;
3578  default:
3579  break;
3580  }
3581 
3582  *pMemoryTypeIndex = UINT_MAX;
3583  uint32_t minCost = UINT_MAX;
3584  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
3585  memTypeIndex < allocator->GetMemoryTypeCount();
3586  ++memTypeIndex, memTypeBit <<= 1)
3587  {
3588  // This memory type is acceptable according to memoryTypeBits bitmask.
3589  if((memTypeBit & memoryTypeBits) != 0)
3590  {
3591  const VkMemoryPropertyFlags currFlags =
3592  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
3593  // This memory type contains requiredFlags.
3594  if((requiredFlags & ~currFlags) == 0)
3595  {
3596  // Calculate cost as number of bits from preferredFlags not present in this memory type.
3597  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
3598  // Remember memory type with lowest cost.
3599  if(currCost < minCost)
3600  {
3601  *pMemoryTypeIndex = memTypeIndex;
3602  if(currCost == 0)
3603  return VK_SUCCESS;
3604  minCost = currCost;
3605  }
3606  }
3607  }
3608  }
3609  return (*pMemoryTypeIndex != UINT_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
3610 }
3611 
3612 VkResult vmaAllocateMemory(
3613  VmaAllocator allocator,
3614  const VkMemoryRequirements* pVkMemoryRequirements,
3615  const VmaMemoryRequirements* pVmaMemoryRequirements,
3616  VkMappedMemoryRange* pMemory,
3617  uint32_t* pMemoryTypeIndex)
3618 {
3619  VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pMemory);
3620 
3621  VMA_DEBUG_LOG("vmaAllocateMemory");
3622 
3623  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3624 
3625  return allocator->AllocateMemory(
3626  *pVkMemoryRequirements,
3627  *pVmaMemoryRequirements,
3628  VMA_SUBALLOCATION_TYPE_UNKNOWN,
3629  pMemory,
3630  pMemoryTypeIndex);
3631 }
3632 
3633 VkResult vmaAllocateMemoryForBuffer(
3634  VmaAllocator allocator,
3635  VkBuffer buffer,
3636  const VmaMemoryRequirements* pMemoryRequirements,
3637  VkMappedMemoryRange* pMemory,
3638  uint32_t* pMemoryTypeIndex)
3639 {
3640  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pMemory);
3641 
3642  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
3643 
3644  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3645 
3646  VkMemoryRequirements vkMemReq = {};
3647  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
3648 
3649  return allocator->AllocateMemory(
3650  vkMemReq,
3651  *pMemoryRequirements,
3652  VMA_SUBALLOCATION_TYPE_BUFFER,
3653  pMemory,
3654  pMemoryTypeIndex);
3655 }
3656 
3657 VkResult vmaAllocateMemoryForImage(
3658  VmaAllocator allocator,
3659  VkImage image,
3660  const VmaMemoryRequirements* pMemoryRequirements,
3661  VkMappedMemoryRange* pMemory,
3662  uint32_t* pMemoryTypeIndex)
3663 {
3664  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements);
3665 
3666  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
3667 
3668  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3669 
3670  return AllocateMemoryForImage(
3671  allocator,
3672  image,
3673  pMemoryRequirements,
3674  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
3675  pMemory,
3676  pMemoryTypeIndex);
3677 }
3678 
3679 void vmaFreeMemory(
3680  VmaAllocator allocator,
3681  const VkMappedMemoryRange* pMemory)
3682 {
3683  VMA_ASSERT(allocator && pMemory);
3684 
3685  VMA_DEBUG_LOG("vmaFreeMemory");
3686 
3687  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3688 
3689  allocator->FreeMemory(pMemory);
3690 }
3691 
3692 VkResult vmaMapMemory(
3693  VmaAllocator allocator,
3694  const VkMappedMemoryRange* pMemory,
3695  void** ppData)
3696 {
3697  VMA_ASSERT(allocator && pMemory && ppData);
3698 
3699  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3700 
3701  return vkMapMemory(allocator->m_hDevice, pMemory->memory,
3702  pMemory->offset, pMemory->size, 0, ppData);
3703 }
3704 
3705 void vmaUnmapMemory(
3706  VmaAllocator allocator,
3707  const VkMappedMemoryRange* pMemory)
3708 {
3709  VMA_ASSERT(allocator && pMemory);
3710 
3711  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3712 
3713  vkUnmapMemory(allocator->m_hDevice, pMemory->memory);
3714 }
3715 
3716 VkResult vmaCreateBuffer(
3717  VmaAllocator allocator,
3718  const VkBufferCreateInfo* pCreateInfo,
3719  const VmaMemoryRequirements* pMemoryRequirements,
3720  VkBuffer* pBuffer,
3721  VkMappedMemoryRange* pMemory,
3722  uint32_t* pMemoryTypeIndex)
3723 {
3724  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements);
3725 
3726  VMA_DEBUG_LOG("vmaCreateBuffer");
3727 
3728  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3729 
3730  // 1. Create VkBuffer.
3731  VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
3732  if(res >= 0)
3733  {
3734  VkMappedMemoryRange mem = {};
3735 
3736  // 2. vkGetBufferMemoryRequirements.
3737  VkMemoryRequirements vkMemReq = {};
3738  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
3739 
3740  // 3. Allocate memory using allocator.
3741  res = allocator->AllocateMemory(
3742  vkMemReq,
3743  *pMemoryRequirements,
3744  VMA_SUBALLOCATION_TYPE_BUFFER,
3745  &mem,
3746  pMemoryTypeIndex);
3747  if(res >= 0)
3748  {
3749  if(pMemory != VMA_NULL)
3750  {
3751  *pMemory = mem;
3752  }
3753  // 3. Bind buffer with memory.
3754  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, mem.memory, mem.offset);
3755  if(res >= 0)
3756  {
3757  // All steps succeeded.
3758  VmaMutexLock lock(allocator->m_BufferToMemoryMapMutex);
3759  allocator->m_BufferToMemoryMap.insert(VmaPair<VkBuffer, VkMappedMemoryRange>(*pBuffer, mem));
3760  return VK_SUCCESS;
3761  }
3762  allocator->FreeMemory(&mem);
3763  return res;
3764  }
3765  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
3766  return res;
3767  }
3768  return res;
3769 }
3770 
3771 void vmaDestroyBuffer(
3772  VmaAllocator allocator,
3773  VkBuffer buffer)
3774 {
3775  if(buffer != VK_NULL_HANDLE)
3776  {
3777  VMA_ASSERT(allocator);
3778 
3779  VMA_DEBUG_LOG("vmaDestroyBuffer");
3780 
3781  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3782 
3783  VkMappedMemoryRange mem = {};
3784  {
3785  VmaMutexLock lock(allocator->m_BufferToMemoryMapMutex);
3786  VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange)::iterator it = allocator->m_BufferToMemoryMap.find(buffer);
3787  if(it == allocator->m_BufferToMemoryMap.end())
3788  {
3789  VMA_ASSERT(0 && "Trying to destroy buffer that was not created using vmaCreateBuffer or already freed.");
3790  return;
3791  }
3792  mem = it->second;
3793  allocator->m_BufferToMemoryMap.erase(it);
3794  }
3795 
3796  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
3797 
3798  allocator->FreeMemory(&mem);
3799  }
3800 }
3801 
3802 VkResult vmaCreateImage(
3803  VmaAllocator allocator,
3804  const VkImageCreateInfo* pCreateInfo,
3805  const VmaMemoryRequirements* pMemoryRequirements,
3806  VkImage* pImage,
3807  VkMappedMemoryRange* pMemory,
3808  uint32_t* pMemoryTypeIndex)
3809 {
3810  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements);
3811 
3812  VMA_DEBUG_LOG("vmaCreateImage");
3813 
3814  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3815 
3816  // 1. Create VkImage.
3817  VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage);
3818  if(res >= 0)
3819  {
3820  VkMappedMemoryRange mem = {};
3821  VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
3822  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
3823  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
3824 
3825  // 2. Allocate memory using allocator.
3826  res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, &mem, pMemoryTypeIndex);
3827  if(res >= 0)
3828  {
3829  if(pMemory != VMA_NULL)
3830  *pMemory = mem;
3831  // 3. Bind image with memory.
3832  res = vkBindImageMemory(allocator->m_hDevice, *pImage, mem.memory, mem.offset);
3833  if(res >= 0)
3834  {
3835  // All steps succeeded.
3836  VmaMutexLock lock(allocator->m_ImageToMemoryMapMutex);
3837  allocator->m_ImageToMemoryMap.insert(VmaPair<VkImage, VkMappedMemoryRange>(*pImage, mem));
3838  return VK_SUCCESS;
3839  }
3840  allocator->FreeMemory(&mem);
3841  return res;
3842  }
3843  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
3844  return res;
3845  }
3846  return res;
3847 }
3848 
3849 void vmaDestroyImage(
3850  VmaAllocator allocator,
3851  VkImage image)
3852 {
3853  if(image != VK_NULL_HANDLE)
3854  {
3855  VMA_ASSERT(allocator);
3856 
3857  VMA_DEBUG_LOG("vmaDestroyImage");
3858 
3859  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3860 
3861  VkMappedMemoryRange mem = {};
3862  {
3863  VmaMutexLock lock(allocator->m_ImageToMemoryMapMutex);
3864  VMA_MAP_TYPE(VkImage, VkMappedMemoryRange)::iterator it = allocator->m_ImageToMemoryMap.find(image);
3865  if(it == allocator->m_ImageToMemoryMap.end())
3866  {
3867  VMA_ASSERT(0 && "Trying to destroy buffer that was not created using vmaCreateBuffer or already freed.");
3868  return;
3869  }
3870  mem = it->second;
3871  allocator->m_ImageToMemoryMap.erase(it);
3872  }
3873 
3874  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
3875 
3876  allocator->FreeMemory(&mem);
3877  }
3878 }
3879 
3880 #endif // #ifdef VMA_IMPLEMENTATION
3881 
3882 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
VmaMemoryRequirements
struct VmaMemoryRequirements VmaMemoryRequirements
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
151 #include <vulkan/vulkan.h>
152 
154 
158 VK_DEFINE_HANDLE(VmaAllocator)
159 
160 typedef struct VmaAllocatorCreateInfo
162 {
164 
165  VkPhysicalDevice physicalDevice;
167 
168  VkDevice device;
170 
171  VkDeviceSize preferredLargeHeapBlockSize;
173 
174  VkDeviceSize preferredSmallHeapBlockSize;
176 
177  const VkAllocationCallbacks* pAllocationCallbacks;
178 } VmaAllocatorCreateInfo;
179 
181 VkResult vmaCreateAllocator(
182  const VmaAllocatorCreateInfo* pCreateInfo,
183  VmaAllocator* pAllocator);
184 
186 void vmaDestroyAllocator(
187  VmaAllocator allocator);
188 
193 void vmaGetPhysicalDeviceProperties(
194  VmaAllocator allocator,
195  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
196 
201 void vmaGetMemoryProperties(
202  VmaAllocator allocator,
203  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
204 
211 void vmaGetMemoryTypeProperties(
212  VmaAllocator allocator,
213  uint32_t memoryTypeIndex,
214  VkMemoryPropertyFlags* pFlags);
215 
216 typedef struct VmaStatInfo
217 {
218  uint32_t AllocationCount;
219  uint32_t SuballocationCount;
220  uint32_t UnusedRangeCount;
221  VkDeviceSize UsedBytes;
222  VkDeviceSize UnusedBytes;
223  VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax;
224  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
225 } VmaStatInfo;
226 
228 struct VmaStats
229 {
230  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
231  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
232  VmaStatInfo total;
233 };
234 
236 void vmaCalculateStats(
237  VmaAllocator allocator,
238  VmaStats* pStats);
239 
240 #define VMA_STATS_STRING_ENABLED 1
241 
242 #if VMA_STATS_STRING_ENABLED
243 
245 
247 void vmaBuildStatsString(
248  VmaAllocator allocator,
249  char** ppStatsString,
250  VkBool32 detailedMap);
251 
252 void vmaFreeStatsString(
253  VmaAllocator allocator,
254  char* pStatsString);
255 
256 #endif // #if VMA_STATS_STRING_ENABLED
257 
260 
265 typedef enum VmaMemoryUsage
266 {
268  VMA_MEMORY_USAGE_UNKNOWN = 0,
270  VMA_MEMORY_USAGE_GPU_ONLY = 1,
272  VMA_MEMORY_USAGE_CPU_ONLY = 2,
274  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
276  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
277  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
278 } VmaMemoryUsage;
279 
280 typedef struct VmaMemoryRequirements
281 {
290  VkBool32 ownMemory;
295  VmaMemoryUsage usage;
299  VkMemoryPropertyFlags requiredFlags;
304  VkMemoryPropertyFlags preferredFlags;
311  VkBool32 neverAllocate;
312 } VmaMemoryRequirements;
313 
328 VkResult vmaFindMemoryTypeIndex(
329  VmaAllocator allocator,
330  uint32_t memoryTypeBits,
331  const VmaMemoryRequirements* pMemoryRequirements,
332  uint32_t* pMemoryTypeIndex);
333 
336 
353 VkResult vmaAllocateMemory(
354  VmaAllocator allocator,
355  const VkMemoryRequirements* pVkMemoryRequirements,
356  const VmaMemoryRequirements* pVmaMemoryRequirements,
357  VkMappedMemoryRange* pMemory,
358  uint32_t* pMemoryTypeIndex);
359 
367 VkResult vmaAllocateMemoryForBuffer(
368  VmaAllocator allocator,
369  VkBuffer buffer,
370  const VmaMemoryRequirements* pMemoryRequirements,
371  VkMappedMemoryRange* pMemory,
372  uint32_t* pMemoryTypeIndex);
373 
375 VkResult vmaAllocateMemoryForImage(
376  VmaAllocator allocator,
377  VkImage image,
378  const VmaMemoryRequirements* pMemoryRequirements,
379  VkMappedMemoryRange* pMemory,
380  uint32_t* pMemoryTypeIndex);
381 
383 void vmaFreeMemory(
384  VmaAllocator allocator,
385  const VkMappedMemoryRange* pMemory);
386 
392 VkResult vmaMapMemory(
393  VmaAllocator allocator,
394  const VkMappedMemoryRange* pMemory,
395  void** ppData);
396 
397 void vmaUnmapMemory(
398  VmaAllocator allocator,
399  const VkMappedMemoryRange* pMemory);
400 
403 
425 VkResult vmaCreateBuffer(
426  VmaAllocator allocator,
427  const VkBufferCreateInfo* pCreateInfo,
428  const VmaMemoryRequirements* pMemoryRequirements,
429  VkBuffer* pBuffer,
430  VkMappedMemoryRange* pMemory,
431  uint32_t* pMemoryTypeIndex);
432 
433 void vmaDestroyBuffer(
434  VmaAllocator allocator,
435  VkBuffer buffer);
436 
438 VkResult vmaCreateImage(
439  VmaAllocator allocator,
440  const VkImageCreateInfo* pCreateInfo,
441  const VmaMemoryRequirements* pMemoryRequirements,
442  VkImage* pImage,
443  VkMappedMemoryRange* pMemory,
444  uint32_t* pMemoryTypeIndex);
445 
446 void vmaDestroyImage(
447  VmaAllocator allocator,
448  VkImage image);
449 
452 #ifdef VMA_IMPLEMENTATION
453 
454 #include <cstdint>
455 #include <cstdlib>
456 
457 /*******************************************************************************
458 CONFIGURATION SECTION
459 
460 Define some of these macros before each #include of this header or change them
461 here if you need other then default behavior depending on your environment.
462 */
463 
464 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
465 //#define VMA_USE_STL_CONTAINERS 1
466 
467 /* Set this macro to 1 to make the library including and using STL containers:
468 std::pair, std::vector, std::list, std::unordered_map.
469 
470 Set it to 0 or undefined to make the library using its own implementation of
471 the containers.
472 */
473 #if VMA_USE_STL_CONTAINERS
474  #define VMA_USE_STL_VECTOR 1
475  #define VMA_USE_STL_UNORDERED_MAP 1
476  #define VMA_USE_STL_LIST 1
477 #endif
478 
479 #if VMA_USE_STL_VECTOR
480  #include <vector>
481 #endif
482 
483 #if VMA_USE_STL_UNORDERED_MAP
484  #include <unordered_map>
485 #endif
486 
487 #if VMA_USE_STL_LIST
488  #include <list>
489 #endif
490 
491 /*
492 Following headers are used in this CONFIGURATION section only, so feel free to
493 remove them if not needed.
494 */
495 #include <cassert> // for assert
496 #include <algorithm> // for min, max
497 #include <mutex> // for std::mutex
498 
499 #if !defined(_WIN32)
500  #include <malloc.h> // for aligned_alloc()
501 #endif
502 
503 // Normal assert to check for programmer's errors, especially in Debug configuration.
504 #ifndef VMA_ASSERT
505  #ifdef _DEBUG
506  #define VMA_ASSERT(expr) assert(expr)
507  #else
508  #define VMA_ASSERT(expr)
509  #endif
510 #endif
511 
512 // Assert that will be called very often, like inside data structures e.g. operator[].
513 // Making it non-empty can make program slow.
514 #ifndef VMA_HEAVY_ASSERT
515  #ifdef _DEBUG
516  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
517  #else
518  #define VMA_HEAVY_ASSERT(expr)
519  #endif
520 #endif
521 
522 #ifndef VMA_NULL
523  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
524  #define VMA_NULL nullptr
525 #endif
526 
527 #ifndef VMA_ALIGN_OF
528  #define VMA_ALIGN_OF(type) (__alignof(type))
529 #endif
530 
531 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
532  #if defined(_WIN32)
533  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
534  #else
535  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
536  #endif
537 #endif
538 
539 #ifndef VMA_SYSTEM_FREE
540  #if defined(_WIN32)
541  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
542  #else
543  #define VMA_SYSTEM_FREE(ptr) free(ptr)
544  #endif
545 #endif
546 
547 #ifndef VMA_MIN
548  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
549 #endif
550 
551 #ifndef VMA_MAX
552  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
553 #endif
554 
555 #ifndef VMA_SWAP
556  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
557 #endif
558 
559 #ifndef VMA_DEBUG_LOG
560  #define VMA_DEBUG_LOG(format, ...)
561  /*
562  #define VMA_DEBUG_LOG(format, ...) do { \
563  printf(format, __VA_ARGS__); \
564  printf("\n"); \
565  } while(false)
566  */
567 #endif
568 
569 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
570 #if VMA_STATS_STRING_ENABLED
571  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
572  {
573  _ultoa_s(num, outStr, strLen, 10);
574  }
575  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
576  {
577  _ui64toa_s(num, outStr, strLen, 10);
578  }
579 #endif
580 
581 #ifndef VMA_MUTEX
582  class VmaMutex
583  {
584  public:
585  VmaMutex() { }
586  ~VmaMutex() { }
587  void Lock() { m_Mutex.lock(); }
588  void Unlock() { m_Mutex.unlock(); }
589  private:
590  std::mutex m_Mutex;
591  };
592  #define VMA_MUTEX VmaMutex
593 #endif
594 
595 #ifndef VMA_BEST_FIT
596 
608  #define VMA_BEST_FIT (1)
609 #endif
610 
611 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
612 
616  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
617 #endif
618 
619 #ifndef VMA_DEBUG_ALIGNMENT
620 
624  #define VMA_DEBUG_ALIGNMENT (1)
625 #endif
626 
627 #ifndef VMA_DEBUG_MARGIN
628 
632  #define VMA_DEBUG_MARGIN (0)
633 #endif
634 
635 #ifndef VMA_DEBUG_GLOBAL_MUTEX
636 
640  #define VMA_DEBUG_GLOBAL_MUTEX (0)
641 #endif
642 
643 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
644 
648  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
649 #endif
650 
651 #ifndef VMA_SMALL_HEAP_MAX_SIZE
652  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
654 #endif
655 
656 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
657  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
659 #endif
660 
661 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
662  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
664 #endif
665 
666 /*******************************************************************************
667 END OF CONFIGURATION
668 */
669 
670 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
671  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
672 
673 // Returns number of bits set to 1 in (v).
674 static inline uint32_t CountBitsSet(uint32_t v)
675 {
676  uint32_t c = v - ((v >> 1) & 0x55555555);
677  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
678  c = ((c >> 4) + c) & 0x0F0F0F0F;
679  c = ((c >> 8) + c) & 0x00FF00FF;
680  c = ((c >> 16) + c) & 0x0000FFFF;
681  return c;
682 }
683 
684 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
685 // Use types like uint32_t, uint64_t as T.
686 template <typename T>
687 static inline T VmaAlignUp(T val, T align)
688 {
689  return (val + align - 1) / align * align;
690 }
691 
692 // Division with mathematical rounding to nearest number.
693 template <typename T>
694 inline T VmaRoundDiv(T x, T y)
695 {
696  return (x + (y / (T)2)) / y;
697 }
698 /*
699 Returns true if two memory blocks occupy overlapping pages.
700 ResourceA must be in less memory offset than ResourceB.
701 
702 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
703 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
704 */
705 static inline bool VmaBlocksOnSamePage(
706  VkDeviceSize resourceAOffset,
707  VkDeviceSize resourceASize,
708  VkDeviceSize resourceBOffset,
709  VkDeviceSize pageSize)
710 {
711  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
712  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
713  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
714  VkDeviceSize resourceBStart = resourceBOffset;
715  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
716  return resourceAEndPage == resourceBStartPage;
717 }
718 
719 enum VmaSuballocationType
720 {
721  VMA_SUBALLOCATION_TYPE_FREE = 0,
722  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
723  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
724  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
725  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
726  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
727  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
728 };
729 
730 /*
731 Returns true if given suballocation types could conflict and must respect
732 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
733 or linear image and another one is optimal image. If type is unknown, behave
734 conservatively.
735 */
736 static inline bool VmaIsBufferImageGranularityConflict(
737  VmaSuballocationType suballocType1,
738  VmaSuballocationType suballocType2)
739 {
740  if(suballocType1 > suballocType2)
741  VMA_SWAP(suballocType1, suballocType2);
742 
743  switch(suballocType1)
744  {
745  case VMA_SUBALLOCATION_TYPE_FREE:
746  return false;
747  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
748  return true;
749  case VMA_SUBALLOCATION_TYPE_BUFFER:
750  return
751  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
752  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
753  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
754  return
755  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
756  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
757  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
758  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
759  return
760  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
761  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
762  return false;
763  default:
764  VMA_ASSERT(0);
765  return true;
766  }
767 }
768 
769 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
770 struct VmaMutexLock
771 {
772 public:
773  VmaMutexLock(VMA_MUTEX& mutex) : m_Mutex(mutex) { mutex.Lock(); }
774  ~VmaMutexLock() { m_Mutex.Unlock(); }
775 
776 private:
777  VMA_MUTEX& m_Mutex;
778 };
779 
780 #if VMA_DEBUG_GLOBAL_MUTEX
781  static VMA_MUTEX gDebugGlobalMutex;
782  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex);
783 #else
784  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
785 #endif
786 
787 // Minimum size of a free suballocation to register it in the free suballocation collection.
788 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
789 
790 /*
791 Performs binary search and returns iterator to first element that is greater or
792 equal to (key), according to comparison (cmp).
793 
794 Cmp should return true if first argument is less than second argument.
795 
796 Returned value is the found element, if present in the collection or place where
797 new element with value (key) should be inserted.
798 */
799 template <typename IterT, typename KeyT, typename CmpT>
800 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
801 {
802  size_t down = 0, up = (end - beg);
803  while(down < up)
804  {
805  const size_t mid = (down + up) / 2;
806  if(cmp(*(beg+mid), key))
807  down = mid + 1;
808  else
809  up = mid;
810  }
811  return beg + down;
812 }
813 
815 // Memory allocation
816 
817 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
818 {
819  if((pAllocationCallbacks != VMA_NULL) &&
820  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
821  {
822  return (*pAllocationCallbacks->pfnAllocation)(
823  pAllocationCallbacks->pUserData,
824  size,
825  alignment,
826  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
827  }
828  else
829  {
830  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
831  }
832 }
833 
834 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
835 {
836  if((pAllocationCallbacks != VMA_NULL) &&
837  (pAllocationCallbacks->pfnFree != VMA_NULL))
838  {
839  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
840  }
841  else
842  {
843  VMA_SYSTEM_FREE(ptr);
844  }
845 }
846 
847 template<typename T>
848 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
849 {
850  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
851 }
852 
853 template<typename T>
854 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
855 {
856  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
857 }
858 
859 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
860 
861 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
862 
863 template<typename T>
864 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
865 {
866  ptr->~T();
867  VmaFree(pAllocationCallbacks, ptr);
868 }
869 
870 template<typename T>
871 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
872 {
873  if(ptr != VMA_NULL)
874  {
875  for(size_t i = count; i--; )
876  ptr[i].~T();
877  VmaFree(pAllocationCallbacks, ptr);
878  }
879 }
880 
881 // STL-compatible allocator.
882 template<typename T>
883 class VmaStlAllocator
884 {
885 public:
886  const VkAllocationCallbacks* const m_pCallbacks;
887  typedef T value_type;
888 
889  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
890  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
891 
892  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
893  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
894 
895  template<typename U>
896  bool operator==(const VmaStlAllocator<U>& rhs) const
897  {
898  return m_pCallbacks == rhs.m_pCallbacks;
899  }
900  template<typename U>
901  bool operator!=(const VmaStlAllocator<U>& rhs) const
902  {
903  return m_pCallbacks != rhs.m_pCallbacks;
904  }
905 
906  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
907 };
908 
909 #if VMA_USE_STL_VECTOR
910 
911 #define VmaVector std::vector
912 
913 template<typename T, typename allocatorT>
914 static void VectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
915 {
916  vec.insert(vec.begin() + index, item);
917 }
918 
919 template<typename T, typename allocatorT>
920 static void VectorRemove(std::vector<T, allocatorT>& vec, size_t index)
921 {
922  vec.erase(vec.begin() + index);
923 }
924 
925 #else // #if VMA_USE_STL_VECTOR
926 
927 /* Class with interface compatible with subset of std::vector.
928 T must be POD because constructors and destructors are not called and memcpy is
929 used for these objects. */
930 template<typename T, typename AllocatorT>
931 class VmaVector
932 {
933 public:
934  VmaVector(const AllocatorT& allocator) :
935  m_Allocator(allocator),
936  m_pArray(VMA_NULL),
937  m_Count(0),
938  m_Capacity(0)
939  {
940  }
941 
942  VmaVector(AllocatorT& allocator) :
943  m_Allocator(allocator),
944  m_pArray(VMA_NULL),
945  m_Count(0),
946  m_Capacity(0)
947  {
948  }
949 
950  VmaVector(size_t count, AllocatorT& allocator) :
951  m_Allocator(allocator),
952  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, count) : VMA_NULL),
953  m_Count(count),
954  m_Capacity(count)
955  {
956  }
957 
958  VmaVector(const VmaVector<T, AllocatorT>& src) :
959  m_Allocator(src.m_Allocator),
960  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src->m_pCallbacks, src.m_Count) : VMA_NULL),
961  m_Count(src.m_Count),
962  m_Capacity(src.m_Count)
963  {
964  if(m_Count != 0)
965  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
966  }
967 
968  ~VmaVector()
969  {
970  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
971  }
972 
973  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
974  {
975  if(&rhs != this)
976  {
977  Resize(rhs.m_Count);
978  if(m_Count != 0)
979  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
980  }
981  return *this;
982  }
983 
984  bool empty() const { return m_Count == 0; }
985  size_t size() const { return m_Count; }
986  T* data() { return m_pArray; }
987  const T* data() const { return m_pArray; }
988 
989  T& operator[](size_t index)
990  {
991  VMA_HEAVY_ASSERT(index < m_Count);
992  return m_pArray[index];
993  }
994  const T& operator[](size_t index) const
995  {
996  VMA_HEAVY_ASSERT(index < m_Count);
997  return m_pArray[index];
998  }
999 
1000  T& front()
1001  {
1002  VMA_HEAVY_ASSERT(m_Count > 0);
1003  return m_pArray[0];
1004  }
1005  const T& front() const
1006  {
1007  VMA_HEAVY_ASSERT(m_Count > 0);
1008  return m_pArray[0];
1009  }
1010  T& back()
1011  {
1012  VMA_HEAVY_ASSERT(m_Count > 0);
1013  return m_pArray[m_Count - 1];
1014  }
1015  const T& back() const
1016  {
1017  VMA_HEAVY_ASSERT(m_Count > 0);
1018  return m_pArray[m_Count - 1];
1019  }
1020 
1021  void reserve(size_t newCapacity, bool freeMemory = false)
1022  {
1023  newCapacity = VMA_MAX(newCapacity, m_Count);
1024 
1025  if((newCapacity < m_Capacity) && !freeMemory)
1026  newCapacity = m_Capacity;
1027 
1028  if(newCapacity != m_Capacity)
1029  {
1030  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
1031  if(m_Count != 0)
1032  memcpy(newArray, m_pArray, m_Count * sizeof(T));
1033  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1034  m_Capacity = newCapacity;
1035  m_pArray = newArray;
1036  }
1037  }
1038 
1039  void resize(size_t newCount, bool freeMemory = false)
1040  {
1041  size_t newCapacity = m_Capacity;
1042  if(newCount > m_Capacity)
1043  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
1044  else if(freeMemory)
1045  newCapacity = newCount;
1046 
1047  if(newCapacity != m_Capacity)
1048  {
1049  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
1050  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
1051  if(elementsToCopy != 0)
1052  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
1053  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
1054  m_Capacity = newCapacity;
1055  m_pArray = newArray;
1056  }
1057 
1058  m_Count = newCount;
1059  }
1060 
1061  void clear(bool freeMemory = false)
1062  {
1063  resize(0, freeMemory);
1064  }
1065 
1066  void insert(size_t index, const T& src)
1067  {
1068  VMA_HEAVY_ASSERT(index <= m_Count);
1069  const size_t oldCount = size();
1070  resize(oldCount + 1);
1071  if(index < oldCount)
1072  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
1073  m_pArray[index] = src;
1074  }
1075 
1076  void remove(size_t index)
1077  {
1078  VMA_HEAVY_ASSERT(index < m_Count);
1079  const size_t oldCount = size();
1080  if(index < oldCount - 1)
1081  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
1082  resize(oldCount - 1);
1083  }
1084 
1085  void push_back(const T& src)
1086  {
1087  const size_t newIndex = size();
1088  resize(newIndex + 1);
1089  m_pArray[newIndex] = src;
1090  }
1091 
1092  void pop_back()
1093  {
1094  VMA_HEAVY_ASSERT(m_Count > 0);
1095  resize(size() - 1);
1096  }
1097 
1098  void push_front(const T& src)
1099  {
1100  insert(0, src);
1101  }
1102 
1103  void pop_front()
1104  {
1105  VMA_HEAVY_ASSERT(m_Count > 0);
1106  remove(0);
1107  }
1108 
1109  typedef T* iterator;
1110 
1111  iterator begin() { return m_pArray; }
1112  iterator end() { return m_pArray + m_Count; }
1113 
1114 private:
1115  AllocatorT m_Allocator;
1116  T* m_pArray;
1117  size_t m_Count;
1118  size_t m_Capacity;
1119 };
1120 
1121 template<typename T, typename allocatorT>
1122 static void VectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
1123 {
1124  vec.insert(index, item);
1125 }
1126 
1127 template<typename T, typename allocatorT>
1128 static void VectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
1129 {
1130  vec.remove(index);
1131 }
1132 
1133 #endif // #if VMA_USE_STL_VECTOR
1134 
1136 // class VmaPoolAllocator
1137 
1138 /*
1139 Allocator for objects of type T using a list of arrays (pools) to speed up
1140 allocation. Number of elements that can be allocated is not bounded because
1141 allocator can create multiple blocks.
1142 */
1143 template<typename T>
1144 class VmaPoolAllocator
1145 {
1146 public:
1147  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
1148  ~VmaPoolAllocator();
1149  void Clear();
1150  T* Alloc();
1151  void Free(T* ptr);
1152 
1153 private:
1154  union Item
1155  {
1156  uint32_t NextFreeIndex;
1157  T Value;
1158  };
1159 
1160  struct ItemBlock
1161  {
1162  Item* pItems;
1163  uint32_t FirstFreeIndex;
1164  };
1165 
1166  const VkAllocationCallbacks* m_pAllocationCallbacks;
1167  size_t m_ItemsPerBlock;
1168  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
1169 
1170  ItemBlock& CreateNewBlock();
1171 };
1172 
1173 template<typename T>
1174 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
1175  m_pAllocationCallbacks(pAllocationCallbacks),
1176  m_ItemsPerBlock(itemsPerBlock),
1177  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
1178 {
1179  VMA_ASSERT(itemsPerBlock > 0);
1180 }
1181 
1182 template<typename T>
1183 VmaPoolAllocator<T>::~VmaPoolAllocator()
1184 {
1185  Clear();
1186 }
1187 
1188 template<typename T>
1189 void VmaPoolAllocator<T>::Clear()
1190 {
1191  for(size_t i = m_ItemBlocks.size(); i--; )
1192  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
1193  m_ItemBlocks.clear();
1194 }
1195 
1196 template<typename T>
1197 T* VmaPoolAllocator<T>::Alloc()
1198 {
1199  for(size_t i = m_ItemBlocks.size(); i--; )
1200  {
1201  ItemBlock& block = m_ItemBlocks[i];
1202  // This block has some free items: Use first one.
1203  if(block.FirstFreeIndex != UINT32_MAX)
1204  {
1205  Item* const pItem = &block.pItems[block.FirstFreeIndex];
1206  block.FirstFreeIndex = pItem->NextFreeIndex;
1207  return &pItem->Value;
1208  }
1209  }
1210 
1211  // No block has free item: Create new one and use it.
1212  ItemBlock& newBlock = CreateNewBlock();
1213  Item* const pItem = &newBlock.pItems[0];
1214  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
1215  return &pItem->Value;
1216 }
1217 
1218 template<typename T>
1219 void VmaPoolAllocator<T>::Free(T* ptr)
1220 {
1221  // Search all memory blocks to find ptr.
1222  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
1223  {
1224  ItemBlock& block = m_ItemBlocks[i];
1225 
1226  // Casting to union.
1227  Item* pItemPtr;
1228  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
1229 
1230  // Check if pItemPtr is in address range of this block.
1231  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
1232  {
1233  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
1234  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
1235  block.FirstFreeIndex = index;
1236  return;
1237  }
1238  }
1239  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
1240 }
1241 
1242 template<typename T>
1243 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
1244 {
1245  ItemBlock newBlock = {
1246  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
1247 
1248  m_ItemBlocks.push_back(newBlock);
1249 
1250  // Setup singly-linked list of all free items in this block.
1251  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
1252  newBlock.pItems[i].NextFreeIndex = i + 1;
1253  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
1254  return m_ItemBlocks.back();
1255 }
1256 
1258 // class VmaRawList, VmaList
1259 
1260 #if VMA_USE_STL_LIST
1261 
1262 #define VmaList std::list
1263 
1264 #else // #if VMA_USE_STL_LIST
1265 
1266 template<typename T>
1267 struct VmaListItem
1268 {
1269  VmaListItem* pPrev;
1270  VmaListItem* pNext;
1271  T Value;
1272 };
1273 
1274 // Doubly linked list.
1275 template<typename T>
1276 class VmaRawList
1277 {
1278 public:
1279  typedef VmaListItem<T> ItemType;
1280 
1281  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
1282  ~VmaRawList();
1283  void Clear();
1284 
1285  size_t GetCount() const { return m_Count; }
1286  bool IsEmpty() const { return m_Count == 0; }
1287 
1288  ItemType* Front() { return m_pFront; }
1289  const ItemType* Front() const { return m_pFront; }
1290  ItemType* Back() { return m_pBack; }
1291  const ItemType* Back() const { return m_pBack; }
1292 
1293  ItemType* PushBack();
1294  ItemType* PushFront();
1295  ItemType* PushBack(const T& value);
1296  ItemType* PushFront(const T& value);
1297  void PopBack();
1298  void PopFront();
1299 
1300  // Item can be null - it means PushBack.
1301  ItemType* InsertBefore(ItemType* pItem);
1302  // Item can be null - it means PushFront.
1303  ItemType* InsertAfter(ItemType* pItem);
1304 
1305  ItemType* InsertBefore(ItemType* pItem, const T& value);
1306  ItemType* InsertAfter(ItemType* pItem, const T& value);
1307 
1308  void Remove(ItemType* pItem);
1309 
1310 private:
1311  const VkAllocationCallbacks* const m_pAllocationCallbacks;
1312  VmaPoolAllocator<ItemType> m_ItemAllocator;
1313  ItemType* m_pFront;
1314  ItemType* m_pBack;
1315  size_t m_Count;
1316 
1317  // Declared not defined, to block copy constructor and assignment operator.
1318  VmaRawList(const VmaRawList<T>& src);
1319  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
1320 };
1321 
1322 template<typename T>
1323 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
1324  m_pAllocationCallbacks(pAllocationCallbacks),
1325  m_ItemAllocator(pAllocationCallbacks, 128),
1326  m_pFront(VMA_NULL),
1327  m_pBack(VMA_NULL),
1328  m_Count(0)
1329 {
1330 }
1331 
1332 template<typename T>
1333 VmaRawList<T>::~VmaRawList()
1334 {
1335  // Intentionally not calling Clear, because that would be unnecessary
1336  // computations to return all items to m_ItemAllocator as free.
1337 }
1338 
1339 template<typename T>
1340 void VmaRawList<T>::Clear()
1341 {
1342  if(IsEmpty() == false)
1343  {
1344  ItemType* pItem = m_pBack;
1345  while(pItem != VMA_NULL)
1346  {
1347  ItemType* const pPrevItem = pItem->pPrev;
1348  m_ItemAllocator.Free(pItem);
1349  pItem = pPrevItem;
1350  }
1351  m_pFront = VMA_NULL;
1352  m_pBack = VMA_NULL;
1353  m_Count = 0;
1354  }
1355 }
1356 
1357 template<typename T>
1358 VmaListItem<T>* VmaRawList<T>::PushBack()
1359 {
1360  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1361  pNewItem->pNext = VMA_NULL;
1362  if(IsEmpty())
1363  {
1364  pNewItem->pPrev = VMA_NULL;
1365  m_pFront = pNewItem;
1366  m_pBack = pNewItem;
1367  m_Count = 1;
1368  }
1369  else
1370  {
1371  pNewItem->pPrev = m_pBack;
1372  m_pBack->pNext = pNewItem;
1373  m_pBack = pNewItem;
1374  ++m_Count;
1375  }
1376  return pNewItem;
1377 }
1378 
1379 template<typename T>
1380 VmaListItem<T>* VmaRawList<T>::PushFront()
1381 {
1382  ItemType* const pNewItem = m_ItemAllocator.Alloc();
1383  pNewItem->pPrev = VMA_NULL;
1384  if(IsEmpty())
1385  {
1386  pNewItem->pNext = VMA_NULL;
1387  m_pFront = pNewItem;
1388  m_pBack = pNewItem;
1389  m_Count = 1;
1390  }
1391  else
1392  {
1393  pNewItem->pNext = m_pFront;
1394  m_pFront->pPrev = pNewItem;
1395  m_pFront = pNewItem;
1396  ++m_Count;
1397  }
1398  return pNewItem;
1399 }
1400 
1401 template<typename T>
1402 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
1403 {
1404  ItemType* const pNewItem = PushBack();
1405  pNewItem->Value = value;
1406  return pNewItem;
1407 }
1408 
1409 template<typename T>
1410 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
1411 {
1412  ItemType* const pNewItem = PushFront();
1413  pNewItem->Value = value;
1414  return pNewItem;
1415 }
1416 
1417 template<typename T>
1418 void VmaRawList<T>::PopBack()
1419 {
1420  VMA_HEAVY_ASSERT(m_Count > 0);
1421  ItemType* const pBackItem = m_pBack;
1422  ItemType* const pPrevItem = pBackItem->pPrev;
1423  if(pPrevItem != VMA_NULL)
1424  pPrevItem->pNext = VMA_NULL;
1425  m_pBack = pPrevItem;
1426  m_ItemAllocator.Free(pBackItem);
1427  --m_Count;
1428 }
1429 
1430 template<typename T>
1431 void VmaRawList<T>::PopFront()
1432 {
1433  VMA_HEAVY_ASSERT(m_Count > 0);
1434  ItemType* const pFrontItem = m_pFront;
1435  ItemType* const pNextItem = pFrontItem->pNext;
1436  if(pNextItem != VMA_NULL)
1437  pNextItem->pPrev = VMA_NULL;
1438  m_pFront = pNextItem;
1439  m_ItemAllocator.Free(pFrontItem);
1440  --m_Count;
1441 }
1442 
1443 template<typename T>
1444 void VmaRawList<T>::Remove(ItemType* pItem)
1445 {
1446  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
1447  VMA_HEAVY_ASSERT(m_Count > 0);
1448 
1449  if(pItem->pPrev != VMA_NULL)
1450  pItem->pPrev->pNext = pItem->pNext;
1451  else
1452  {
1453  VMA_HEAVY_ASSERT(m_pFront == pItem);
1454  m_pFront = pItem->pNext;
1455  }
1456 
1457  if(pItem->pNext != VMA_NULL)
1458  pItem->pNext->pPrev = pItem->pPrev;
1459  else
1460  {
1461  VMA_HEAVY_ASSERT(m_pBack == pItem);
1462  m_pBack = pItem->pPrev;
1463  }
1464 
1465  m_ItemAllocator.Free(pItem);
1466  --m_Count;
1467 }
1468 
1469 template<typename T>
1470 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
1471 {
1472  if(pItem != VMA_NULL)
1473  {
1474  ItemType* const prevItem = pItem->pPrev;
1475  ItemType* const newItem = m_ItemAllocator.Alloc();
1476  newItem->pPrev = prevItem;
1477  newItem->pNext = pItem;
1478  pItem->pPrev = newItem;
1479  if(prevItem != VMA_NULL)
1480  prevItem->pNext = newItem;
1481  else
1482  {
1483  VMA_HEAVY_ASSERT(m_pFront = pItem);
1484  m_pFront = newItem;
1485  }
1486  ++m_Count;
1487  return newItem;
1488  }
1489  else
1490  return PushBack();
1491 }
1492 
1493 template<typename T>
1494 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
1495 {
1496  if(pItem != VMA_NULL)
1497  {
1498  ItemType* const nextItem = pItem->pNext;
1499  ItemType* const newItem = m_ItemAllocator.Alloc();
1500  newItem->pNext = nextItem;
1501  newItem->pPrev = pItem;
1502  pItem->pNext = newItem;
1503  if(nextItem != VMA_NULL)
1504  nextItem->pPrev = newItem;
1505  else
1506  {
1507  VMA_HEAVY_ASSERT(m_pBack = pItem);
1508  m_pBack = newItem;
1509  }
1510  ++m_Count;
1511  return newItem;
1512  }
1513  else
1514  return PushFront();
1515 }
1516 
1517 template<typename T>
1518 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
1519 {
1520  ItemType* const newItem = InsertBefore(pItem);
1521  newItem->Value = value;
1522  return newItem;
1523 }
1524 
1525 template<typename T>
1526 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
1527 {
1528  ItemType* const newItem = InsertAfter(pItem);
1529  newItem->Value = value;
1530  return newItem;
1531 }
1532 
1533 template<typename T, typename AllocatorT>
1534 class VmaList
1535 {
1536 public:
1537  class iterator
1538  {
1539  public:
1540  iterator() :
1541  m_pList(VMA_NULL),
1542  m_pItem(VMA_NULL)
1543  {
1544  }
1545 
1546  T& operator*() const
1547  {
1548  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1549  return m_pItem->Value;
1550  }
1551  T* operator->() const
1552  {
1553  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1554  return &m_pItem->Value;
1555  }
1556 
1557  iterator& operator++()
1558  {
1559  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1560  m_pItem = m_pItem->pNext;
1561  return *this;
1562  }
1563  iterator& operator--()
1564  {
1565  if(m_pItem != VMA_NULL)
1566  m_pItem = m_pItem->pPrev;
1567  else
1568  {
1569  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
1570  m_pItem = m_pList->Back();
1571  }
1572  return *this;
1573  }
1574 
1575  iterator operator++(int)
1576  {
1577  iterator result = *this;
1578  ++*this;
1579  return result;
1580  }
1581  iterator operator--(int)
1582  {
1583  iterator result = *this;
1584  --*this;
1585  return result;
1586  }
1587 
1588  bool operator==(const iterator& rhs) const
1589  {
1590  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1591  return m_pItem == rhs.m_pItem;
1592  }
1593  bool operator!=(const iterator& rhs) const
1594  {
1595  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1596  return m_pItem != rhs.m_pItem;
1597  }
1598 
1599  private:
1600  VmaRawList<T>* m_pList;
1601  VmaListItem<T>* m_pItem;
1602 
1603  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
1604  m_pList(pList),
1605  m_pItem(pItem)
1606  {
1607  }
1608 
1609  friend class VmaList<T, AllocatorT>;
1610  friend class VmaList<T, AllocatorT>:: const_iterator;
1611  };
1612 
1613  class const_iterator
1614  {
1615  public:
1616  const_iterator() :
1617  m_pList(VMA_NULL),
1618  m_pItem(VMA_NULL)
1619  {
1620  }
1621 
1622  const_iterator(const iterator& src) :
1623  m_pList(src.m_pList),
1624  m_pItem(src.m_pItem)
1625  {
1626  }
1627 
1628  const T& operator*() const
1629  {
1630  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1631  return m_pItem->Value;
1632  }
1633  const T* operator->() const
1634  {
1635  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1636  return &m_pItem->Value;
1637  }
1638 
1639  const_iterator& operator++()
1640  {
1641  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
1642  m_pItem = m_pItem->pNext;
1643  return *this;
1644  }
1645  const_iterator& operator--()
1646  {
1647  if(m_pItem != VMA_NULL)
1648  m_pItem = m_pItem->pPrev;
1649  else
1650  {
1651  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
1652  m_pItem = m_pList->Back();
1653  }
1654  return *this;
1655  }
1656 
1657  const_iterator operator++(int)
1658  {
1659  const_iterator result = *this;
1660  ++*this;
1661  return result;
1662  }
1663  const_iterator operator--(int)
1664  {
1665  const_iterator result = *this;
1666  --*this;
1667  return result;
1668  }
1669 
1670  bool operator==(const const_iterator& rhs) const
1671  {
1672  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1673  return m_pItem == rhs.m_pItem;
1674  }
1675  bool operator!=(const const_iterator& rhs) const
1676  {
1677  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
1678  return m_pItem != rhs.m_pItem;
1679  }
1680 
1681  private:
1682  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
1683  m_pList(pList),
1684  m_pItem(pItem)
1685  {
1686  }
1687 
1688  const VmaRawList<T>* m_pList;
1689  const VmaListItem<T>* m_pItem;
1690 
1691  friend class VmaList<T, AllocatorT>;
1692  };
1693 
1694  VmaList(AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
1695  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
1696 
1697  bool empty() const { return m_RawList.IsEmpty(); }
1698  size_t size() const { return m_RawList.GetCount(); }
1699 
1700  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
1701  iterator end() { return iterator(&m_RawList, VMA_NULL); }
1702 
1703  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
1704  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
1705 
1706  void clear() { m_RawList.Clear(); }
1707  void push_back(const T& value) { m_RawList.PushBack(value); }
1708  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
1709  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
1710 
1711 private:
1712  VmaRawList<T> m_RawList;
1713 };
1714 
1715 #endif // #if VMA_USE_STL_LIST
1716 
1718 // class VmaMap
1719 
1720 #if VMA_USE_STL_UNORDERED_MAP
1721 
1722 #define VmaPair std::pair
1723 
1724 #define VMA_MAP_TYPE(KeyT, ValueT) \
1725  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
1726 
1727 #else // #if VMA_USE_STL_UNORDERED_MAP
1728 
1729 template<typename T1, typename T2>
1730 struct VmaPair
1731 {
1732  T1 first;
1733  T2 second;
1734 
1735  VmaPair() : first(), second() { }
1736  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
1737 };
1738 
1739 /* Class compatible with subset of interface of std::unordered_map.
1740 KeyT, ValueT must be POD because they will be stored in VmaVector.
1741 */
1742 template<typename KeyT, typename ValueT>
1743 class VmaMap
1744 {
1745 public:
1746  typedef VmaPair<KeyT, ValueT> PairType;
1747  typedef PairType* iterator;
1748 
1749  VmaMap(VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
1750  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
1751 
1752  iterator begin() { return m_Vector.begin(); }
1753  iterator end() { return m_Vector.end(); }
1754 
1755  void insert(const PairType& pair);
1756  iterator find(const KeyT& key);
1757  void erase(iterator it);
1758 
1759 private:
1760  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
1761 };
1762 
1763 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
1764 
1765 template<typename FirstT, typename SecondT>
1766 struct VmaPairFirstLess
1767 {
1768  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
1769  {
1770  return lhs.first < rhs.first;
1771  }
1772  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
1773  {
1774  return lhs.first < rhsFirst;
1775  }
1776 };
1777 
1778 template<typename KeyT, typename ValueT>
1779 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
1780 {
1781  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
1782  m_Vector.data(),
1783  m_Vector.data() + m_Vector.size(),
1784  pair,
1785  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
1786  VectorInsert(m_Vector, indexToInsert, pair);
1787 }
1788 
1789 template<typename KeyT, typename ValueT>
1790 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
1791 {
1792  PairType* it = VmaBinaryFindFirstNotLess(
1793  m_Vector.data(),
1794  m_Vector.data() + m_Vector.size(),
1795  key,
1796  VmaPairFirstLess<KeyT, ValueT>());
1797  if((it != m_Vector.end()) && (it->first == key))
1798  return it;
1799  else
1800  return m_Vector.end();
1801 }
1802 
1803 template<typename KeyT, typename ValueT>
1804 void VmaMap<KeyT, ValueT>::erase(iterator it)
1805 {
1806  VectorRemove(m_Vector, it - m_Vector.begin());
1807 }
1808 
1809 #endif // #if VMA_USE_STL_UNORDERED_MAP
1810 
1811 /*
1812 Represents a region of VmaAllocation that is either assigned and returned as
1813 allocated memory block or free.
1814 */
1815 struct VmaSuballocation
1816 {
1817  VkDeviceSize offset;
1818  VkDeviceSize size;
1819  VmaSuballocationType type;
1820 };
1821 
1822 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
1823 
1824 // Parameters of an allocation.
1825 struct VmaAllocationRequest
1826 {
1827  VmaSuballocationList::iterator freeSuballocationItem;
1828  VkDeviceSize offset;
1829 };
1830 
1831 /* Single block of memory - VkDeviceMemory with all the data about its regions
1832 assigned or free. */
1833 class VmaAllocation
1834 {
1835 public:
1836  VkDeviceMemory m_hMemory;
1837  VkDeviceSize m_Size;
1838  uint32_t m_FreeCount;
1839  VkDeviceSize m_SumFreeSize;
1840  VmaSuballocationList m_Suballocations;
1841  // Suballocations that are free and have size greater than certain threshold.
1842  // Sorted by size, ascending.
1843  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
1844 
1845  VmaAllocation(VmaAllocator hAllocator);
1846 
1847  ~VmaAllocation()
1848  {
1849  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
1850  }
1851 
1852  // Always call after construction.
1853  void Init(VkDeviceMemory newMemory, VkDeviceSize newSize);
1854  // Always call before destruction.
1855  void Destroy(VmaAllocator allocator);
1856 
1857  // Validates all data structures inside this object. If not valid, returns false.
1858  bool Validate() const;
1859 
1860  // Tries to find a place for suballocation with given parameters inside this allocation.
1861  // If succeeded, fills pAllocationRequest and returns true.
1862  // If failed, returns false.
1863  bool CreateAllocationRequest(
1864  VkDeviceSize bufferImageGranularity,
1865  VkDeviceSize allocSize,
1866  VkDeviceSize allocAlignment,
1867  VmaSuballocationType allocType,
1868  VmaAllocationRequest* pAllocationRequest);
1869 
1870  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
1871  // If yes, fills pOffset and returns true. If no, returns false.
1872  bool CheckAllocation(
1873  VkDeviceSize bufferImageGranularity,
1874  VkDeviceSize allocSize,
1875  VkDeviceSize allocAlignment,
1876  VmaSuballocationType allocType,
1877  VmaSuballocationList::const_iterator freeSuballocItem,
1878  VkDeviceSize* pOffset) const;
1879 
1880  // Returns true if this allocation is empty - contains only single free suballocation.
1881  bool IsEmpty() const;
1882 
1883  // Makes actual allocation based on request. Request must already be checked
1884  // and valid.
1885  void Alloc(
1886  const VmaAllocationRequest& request,
1887  VmaSuballocationType type,
1888  VkDeviceSize allocSize);
1889 
1890  // Frees suballocation assigned to given memory region.
1891  void Free(const VkMappedMemoryRange* pMemory);
1892 
1893 #if VMA_STATS_STRING_ENABLED
1894  void PrintDetailedMap(class VmaStringBuilder& sb) const;
1895 #endif
1896 
1897 private:
1898  // Given free suballocation, it merges it with following one, which must also be free.
1899  void MergeFreeWithNext(VmaSuballocationList::iterator item);
1900  // Releases given suballocation, making it free. Merges it with adjacent free
1901  // suballocations if applicable.
1902  void FreeSuballocation(VmaSuballocationList::iterator suballocItem);
1903  // Given free suballocation, it inserts it into sorted list of
1904  // m_FreeSuballocationsBySize if it's suitable.
1905  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
1906  // Given free suballocation, it removes it from sorted list of
1907  // m_FreeSuballocationsBySize if it's suitable.
1908  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
1909 };
1910 
1911 // Allocation for an object that has its own private VkDeviceMemory.
1912 struct VmaOwnAllocation
1913 {
1914  VkDeviceMemory m_hMemory;
1915  VkDeviceSize m_Size;
1916  VmaSuballocationType m_Type;
1917 };
1918 
1919 struct VmaOwnAllocationMemoryHandleLess
1920 {
1921  bool operator()(const VmaOwnAllocation& lhs, const VmaOwnAllocation& rhs) const
1922  {
1923  return lhs.m_hMemory < rhs.m_hMemory;
1924  }
1925  bool operator()(const VmaOwnAllocation& lhs, VkDeviceMemory rhsMem) const
1926  {
1927  return lhs.m_hMemory < rhsMem;
1928  }
1929 };
1930 
1931 /* Sequence of VmaAllocation. Represents memory blocks allocated for a specific
1932 Vulkan memory type. */
1933 struct VmaAllocationVector
1934 {
1935  // Incrementally sorted by sumFreeSize, ascending.
1936  VmaVector< VmaAllocation*, VmaStlAllocator<VmaAllocation*> > m_Allocations;
1937 
1938  VmaAllocationVector(VmaAllocator hAllocator);
1939  ~VmaAllocationVector();
1940 
1941  bool IsEmpty() const { return m_Allocations.empty(); }
1942 
1943  // Tries to free memory from any if its Allocations.
1944  // Returns index of Allocation that the memory was freed from, or -1 if not found.
1945  size_t Free(const VkMappedMemoryRange* pMemory);
1946 
1947  // Performs single step in sorting m_Allocations. They may not be fully sorted
1948  // after this call.
1949  void IncrementallySortAllocations();
1950 
1951  // Adds statistics of this AllocationVector to pStats.
1952  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const;
1953 
1954 #if VMA_STATS_STRING_ENABLED
1955  void PrintDetailedMap(class VmaStringBuilder& sb) const;
1956 #endif
1957 
1958 private:
1959  VmaAllocator m_hAllocator;
1960 };
1961 
1962 // Main allocator object.
1963 struct VmaAllocator_T
1964 {
1965  VkDevice m_hDevice;
1966  bool m_AllocationCallbacksSpecified;
1967  VkAllocationCallbacks m_AllocationCallbacks;
1968  VkDeviceSize m_PreferredLargeHeapBlockSize;
1969  VkDeviceSize m_PreferredSmallHeapBlockSize;
1970 
1971  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
1972  VkPhysicalDeviceMemoryProperties m_MemProps;
1973 
1974  VmaAllocationVector* m_pAllocations[VK_MAX_MEMORY_TYPES];
1975  /* There can be at most one allocation that is completely empty - a
1976  hysteresis to avoid pessimistic case of alternating creation and destruction
1977  of a VkDeviceMemory. */
1978  bool m_HasEmptyAllocation[VK_MAX_MEMORY_TYPES];
1979  VMA_MUTEX m_AllocationsMutex[VK_MAX_MEMORY_TYPES];
1980 
1981  // Each vector is sorted by memory (handle value).
1982  typedef VmaVector< VmaOwnAllocation, VmaStlAllocator<VmaOwnAllocation> > OwnAllocationVectorType;
1983  OwnAllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES];
1984  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
1985 
1986  // Sorted by first (VkBuffer handle value).
1987  VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange) m_BufferToMemoryMap;
1988  VMA_MUTEX m_BufferToMemoryMapMutex;
1989  // Sorted by first (VkImage handle value).
1990  VMA_MAP_TYPE(VkImage, VkMappedMemoryRange) m_ImageToMemoryMap;
1991  VMA_MUTEX m_ImageToMemoryMapMutex;
1992 
1993  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
1994  ~VmaAllocator_T();
1995 
1996  const VkAllocationCallbacks* GetAllocationCallbacks() const
1997  {
1998  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
1999  }
2000 
2001  VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const;
2002 
2003  VkDeviceSize GetBufferImageGranularity() const
2004  {
2005  return VMA_MAX(
2006  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
2007  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
2008  }
2009 
2010  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
2011  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
2012 
2013  // Main allocation function.
2014  VkResult AllocateMemory(
2015  const VkMemoryRequirements& vkMemReq,
2016  const VmaMemoryRequirements& vmaMemReq,
2017  VmaSuballocationType suballocType,
2018  VkMappedMemoryRange* pMemory,
2019  uint32_t* pMemoryTypeIndex);
2020 
2021  // Main deallocation function.
2022  void FreeMemory(const VkMappedMemoryRange* pMemory);
2023 
2024  void CalculateStats(VmaStats* pStats);
2025 
2026 #if VMA_STATS_STRING_ENABLED
2027  void PrintDetailedMap(class VmaStringBuilder& sb);
2028 #endif
2029 
2030 private:
2031  VkPhysicalDevice m_PhysicalDevice;
2032 
2033  VkResult AllocateMemoryOfType(
2034  const VkMemoryRequirements& vkMemReq,
2035  const VmaMemoryRequirements& vmaMemReq,
2036  uint32_t memTypeIndex,
2037  VmaSuballocationType suballocType,
2038  VkMappedMemoryRange* pMemory);
2039 
2040  // Allocates and registers new VkDeviceMemory specifically for single allocation.
2041  VkResult AllocateOwnMemory(
2042  VkDeviceSize size,
2043  VmaSuballocationType suballocType,
2044  uint32_t memTypeIndex,
2045  VkMappedMemoryRange* pMemory);
2046 
2047  // Tries to free pMemory as Own Memory. Returns true if found and freed.
2048  bool FreeOwnMemory(const VkMappedMemoryRange* pMemory);
2049 };
2050 
2052 // Memory allocation #2 after VmaAllocator_T definition
2053 
2054 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
2055 {
2056  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
2057 }
2058 
2059 static void VmaFree(VmaAllocator hAllocator, void* ptr)
2060 {
2061  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
2062 }
2063 
2064 template<typename T>
2065 static T* VmaAllocate(VmaAllocator hAllocator)
2066 {
2067  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
2068 }
2069 
2070 template<typename T>
2071 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
2072 {
2073  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
2074 }
2075 
2076 template<typename T>
2077 static void vma_delete(VmaAllocator hAllocator, T* ptr)
2078 {
2079  if(ptr != VMA_NULL)
2080  {
2081  ptr->~T();
2082  VmaFree(hAllocator, ptr);
2083  }
2084 }
2085 
2086 template<typename T>
2087 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
2088 {
2089  if(ptr != VMA_NULL)
2090  {
2091  for(size_t i = count; i--; )
2092  ptr[i].~T();
2093  VmaFree(hAllocator, ptr);
2094  }
2095 }
2096 
2098 // VmaStringBuilder
2099 
2100 #if VMA_STATS_STRING_ENABLED
2101 
2102 class VmaStringBuilder
2103 {
2104 public:
2105  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
2106  size_t GetLength() const { return m_Data.size(); }
2107  const char* GetData() const { return m_Data.data(); }
2108 
2109  void Add(char ch) { m_Data.push_back(ch); }
2110  void Add(const char* pStr);
2111  void AddNewLine() { Add('\n'); }
2112  void AddNumber(uint32_t num);
2113  void AddNumber(uint64_t num);
2114  void AddBool(bool b) { Add(b ? "true" : "false"); }
2115  void AddNull() { Add("null"); }
2116  void AddString(const char* pStr);
2117 
2118 private:
2119  VmaVector< char, VmaStlAllocator<char> > m_Data;
2120 };
2121 
2122 void VmaStringBuilder::Add(const char* pStr)
2123 {
2124  const size_t strLen = strlen(pStr);
2125  if(strLen > 0)
2126  {
2127  const size_t oldCount = m_Data.size();
2128  m_Data.resize(oldCount + strLen);
2129  memcpy(m_Data.data() + oldCount, pStr, strLen);
2130  }
2131 }
2132 
2133 void VmaStringBuilder::AddNumber(uint32_t num)
2134 {
2135  char buf[11];
2136  VmaUint32ToStr(buf, sizeof(buf), num);
2137  Add(buf);
2138 }
2139 
2140 void VmaStringBuilder::AddNumber(uint64_t num)
2141 {
2142  char buf[21];
2143  VmaUint64ToStr(buf, sizeof(buf), num);
2144  Add(buf);
2145 }
2146 
2147 void VmaStringBuilder::AddString(const char* pStr)
2148 {
2149  Add('"');
2150  const size_t strLen = strlen(pStr);
2151  for(size_t i = 0; i < strLen; ++i)
2152  {
2153  char ch = pStr[i];
2154  if(ch == '\'')
2155  Add("\\\\");
2156  else if(ch == '"')
2157  Add("\\\"");
2158  else if(ch >= 32)
2159  Add(ch);
2160  else switch(ch)
2161  {
2162  case '\n':
2163  Add("\\n");
2164  break;
2165  case '\r':
2166  Add("\\r");
2167  break;
2168  case '\t':
2169  Add("\\t");
2170  break;
2171  default:
2172  VMA_ASSERT(0 && "Character not currently supported.");
2173  break;
2174  }
2175  }
2176  Add('"');
2177 }
2178 
2180 
2181 // Correspond to values of enum VmaSuballocationType.
2182 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
2183  "FREE",
2184  "UNKNOWN",
2185  "BUFFER",
2186  "IMAGE_UNKNOWN",
2187  "IMAGE_LINEAR",
2188  "IMAGE_OPTIMAL",
2189 };
2190 
2191 static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat)
2192 {
2193  sb.Add("{ \"Allocations\": ");
2194  sb.AddNumber(stat.AllocationCount);
2195  sb.Add(", \"Suballocations\": ");
2196  sb.AddNumber(stat.SuballocationCount);
2197  sb.Add(", \"UnusedRanges\": ");
2198  sb.AddNumber(stat.UnusedRangeCount);
2199  sb.Add(", \"UsedBytes\": ");
2200  sb.AddNumber(stat.UsedBytes);
2201  sb.Add(", \"UnusedBytes\": ");
2202  sb.AddNumber(stat.UnusedBytes);
2203  sb.Add(", \"SuballocationSize\": { \"Min\": ");
2204  sb.AddNumber(stat.SuballocationSizeMin);
2205  sb.Add(", \"Avg\": ");
2206  sb.AddNumber(stat.SuballocationSizeAvg);
2207  sb.Add(", \"Max\": ");
2208  sb.AddNumber(stat.SuballocationSizeMax);
2209  sb.Add(" }, \"UnusedRangeSize\": { \"Min\": ");
2210  sb.AddNumber(stat.UnusedRangeSizeMin);
2211  sb.Add(", \"Avg\": ");
2212  sb.AddNumber(stat.UnusedRangeSizeAvg);
2213  sb.Add(", \"Max\": ");
2214  sb.AddNumber(stat.UnusedRangeSizeMax);
2215  sb.Add(" } }");
2216 }
2217 
2218 #endif // #if VMA_STATS_STRING_ENABLED
2219 
2220 struct VmaSuballocationItemSizeLess
2221 {
2222  bool operator()(
2223  const VmaSuballocationList::iterator lhs,
2224  const VmaSuballocationList::iterator rhs) const
2225  {
2226  return lhs->size < rhs->size;
2227  }
2228  bool operator()(
2229  const VmaSuballocationList::iterator lhs,
2230  VkDeviceSize rhsSize) const
2231  {
2232  return lhs->size < rhsSize;
2233  }
2234 };
2235 
2236 VmaAllocation::VmaAllocation(VmaAllocator hAllocator) :
2237  m_hMemory(VK_NULL_HANDLE),
2238  m_Size(0),
2239  m_FreeCount(0),
2240  m_SumFreeSize(0),
2241  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
2242  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
2243 {
2244 }
2245 
2246 void VmaAllocation::Init(VkDeviceMemory newMemory, VkDeviceSize newSize)
2247 {
2248  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
2249 
2250  m_hMemory = newMemory;
2251  m_Size = newSize;
2252  m_FreeCount = 1;
2253  m_SumFreeSize = newSize;
2254 
2255  m_Suballocations.clear();
2256  m_FreeSuballocationsBySize.clear();
2257 
2258  VmaSuballocation suballoc = {};
2259  suballoc.offset = 0;
2260  suballoc.size = newSize;
2261  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2262 
2263  m_Suballocations.push_back(suballoc);
2264  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
2265  --suballocItem;
2266  m_FreeSuballocationsBySize.push_back(suballocItem);
2267 }
2268 
2269 void VmaAllocation::Destroy(VmaAllocator allocator)
2270 {
2271  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
2272  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
2273  m_hMemory = VK_NULL_HANDLE;
2274 }
2275 
2276 bool VmaAllocation::Validate() const
2277 {
2278  if((m_hMemory == VK_NULL_HANDLE) ||
2279  (m_Size == 0) ||
2280  m_Suballocations.empty())
2281  {
2282  return false;
2283  }
2284 
2285  // Expected offset of new suballocation as calculates from previous ones.
2286  VkDeviceSize calculatedOffset = 0;
2287  // Expected number of free suballocations as calculated from traversing their list.
2288  uint32_t calculatedFreeCount = 0;
2289  // Expected sum size of free suballocations as calculated from traversing their list.
2290  VkDeviceSize calculatedSumFreeSize = 0;
2291  // Expected number of free suballocations that should be registered in
2292  // m_FreeSuballocationsBySize calculated from traversing their list.
2293  size_t freeSuballocationsToRegister = 0;
2294  // True if previous visisted suballocation was free.
2295  bool prevFree = false;
2296 
2297  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2298  suballocItem != m_Suballocations.cend();
2299  ++suballocItem)
2300  {
2301  const VmaSuballocation& subAlloc = *suballocItem;
2302 
2303  // Actual offset of this suballocation doesn't match expected one.
2304  if(subAlloc.offset != calculatedOffset)
2305  return false;
2306 
2307  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
2308  // Two adjacent free suballocations are invalid. They should be merged.
2309  if(prevFree && currFree)
2310  return false;
2311  prevFree = currFree;
2312 
2313  if(currFree)
2314  {
2315  calculatedSumFreeSize += subAlloc.size;
2316  ++calculatedFreeCount;
2317  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2318  ++freeSuballocationsToRegister;
2319  }
2320 
2321  calculatedOffset += subAlloc.size;
2322  }
2323 
2324  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
2325  // match expected one.
2326  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
2327  return false;
2328 
2329  VkDeviceSize lastSize = 0;
2330  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
2331  {
2332  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
2333 
2334  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
2335  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
2336  return false;
2337  // They must be sorted by size ascending.
2338  if(suballocItem->size < lastSize)
2339  return false;
2340 
2341  lastSize = suballocItem->size;
2342  }
2343 
2344  // Check if totals match calculacted values.
2345  return
2346  (calculatedOffset == m_Size) &&
2347  (calculatedSumFreeSize == m_SumFreeSize) &&
2348  (calculatedFreeCount == m_FreeCount);
2349 }
2350 
2351 /*
2352 How many suitable free suballocations to analyze before choosing best one.
2353 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
2354  be chosen.
2355 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
2356  suballocations will be analized and best one will be chosen.
2357 - Any other value is also acceptable.
2358 */
2359 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
2360 
2361 bool VmaAllocation::CreateAllocationRequest(
2362  VkDeviceSize bufferImageGranularity,
2363  VkDeviceSize allocSize,
2364  VkDeviceSize allocAlignment,
2365  VmaSuballocationType allocType,
2366  VmaAllocationRequest* pAllocationRequest)
2367 {
2368  VMA_ASSERT(allocSize > 0);
2369  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2370  VMA_ASSERT(pAllocationRequest != VMA_NULL);
2371  VMA_HEAVY_ASSERT(Validate());
2372 
2373  // There is not enough total free space in this allocation to fullfill the request: Early return.
2374  if(m_SumFreeSize < allocSize)
2375  return false;
2376 
2377  // Old brute-force algorithm, linearly searching suballocations.
2378  /*
2379  uint32_t suitableSuballocationsFound = 0;
2380  for(VmaSuballocationList::iterator suballocItem = suballocations.Front();
2381  suballocItem != VMA_NULL &&
2382  suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK;
2383  suballocItem = suballocItem->Next)
2384  {
2385  if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE)
2386  {
2387  VkDeviceSize offset = 0, cost = 0;
2388  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost))
2389  {
2390  ++suitableSuballocationsFound;
2391  if(cost < costLimit)
2392  {
2393  pAllocationRequest->freeSuballocationItem = suballocItem;
2394  pAllocationRequest->offset = offset;
2395  pAllocationRequest->cost = cost;
2396  if(cost == 0)
2397  return true;
2398  costLimit = cost;
2399  betterSuballocationFound = true;
2400  }
2401  }
2402  }
2403  }
2404  */
2405 
2406  // New algorithm, efficiently searching freeSuballocationsBySize.
2407  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
2408  if(freeSuballocCount > 0)
2409  {
2410  if(VMA_BEST_FIT)
2411  {
2412  // Find first free suballocation with size not less than allocSize.
2413  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2414  m_FreeSuballocationsBySize.data(),
2415  m_FreeSuballocationsBySize.data() + freeSuballocCount,
2416  allocSize,
2417  VmaSuballocationItemSizeLess());
2418  size_t index = it - m_FreeSuballocationsBySize.data();
2419  for(; index < freeSuballocCount; ++index)
2420  {
2421  VkDeviceSize offset = 0;
2422  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
2423  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
2424  {
2425  pAllocationRequest->freeSuballocationItem = suballocItem;
2426  pAllocationRequest->offset = offset;
2427  return true;
2428  }
2429  }
2430  }
2431  else
2432  {
2433  // Search staring from biggest suballocations.
2434  for(size_t index = freeSuballocCount; index--; )
2435  {
2436  VkDeviceSize offset = 0;
2437  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
2438  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
2439  {
2440  pAllocationRequest->freeSuballocationItem = suballocItem;
2441  pAllocationRequest->offset = offset;
2442  return true;
2443  }
2444  }
2445  }
2446  }
2447 
2448  return false;
2449 }
2450 
2451 bool VmaAllocation::CheckAllocation(
2452  VkDeviceSize bufferImageGranularity,
2453  VkDeviceSize allocSize,
2454  VkDeviceSize allocAlignment,
2455  VmaSuballocationType allocType,
2456  VmaSuballocationList::const_iterator freeSuballocItem,
2457  VkDeviceSize* pOffset) const
2458 {
2459  VMA_ASSERT(allocSize > 0);
2460  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
2461  VMA_ASSERT(freeSuballocItem != m_Suballocations.cend());
2462  VMA_ASSERT(pOffset != VMA_NULL);
2463 
2464  const VmaSuballocation& suballoc = *freeSuballocItem;
2465  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
2466 
2467  // Size of this suballocation is too small for this request: Early return.
2468  if(suballoc.size < allocSize)
2469  return false;
2470 
2471  // Start from offset equal to beginning of this suballocation.
2472  *pOffset = suballoc.offset;
2473 
2474  // Apply VMA_DEBUG_MARGIN at the beginning.
2475  if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin())
2476  *pOffset += VMA_DEBUG_MARGIN;
2477 
2478  // Apply alignment.
2479  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
2480  *pOffset = VmaAlignUp(*pOffset, alignment);
2481 
2482  // Check previous suballocations for BufferImageGranularity conflicts.
2483  // Make bigger alignment if necessary.
2484  if(bufferImageGranularity > 1)
2485  {
2486  bool bufferImageGranularityConflict = false;
2487  VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem;
2488  while(prevSuballocItem != m_Suballocations.cbegin())
2489  {
2490  --prevSuballocItem;
2491  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
2492  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
2493  {
2494  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
2495  {
2496  bufferImageGranularityConflict = true;
2497  break;
2498  }
2499  }
2500  else
2501  // Already on previous page.
2502  break;
2503  }
2504  if(bufferImageGranularityConflict)
2505  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
2506  }
2507 
2508  // Calculate padding at the beginning based on current offset.
2509  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
2510 
2511  // Calculate required margin at the end if this is not last suballocation.
2512  VmaSuballocationList::const_iterator next = freeSuballocItem;
2513  ++next;
2514  const VkDeviceSize requiredEndMargin =
2515  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
2516 
2517  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
2518  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
2519  return false;
2520 
2521  // Check next suballocations for BufferImageGranularity conflicts.
2522  // If conflict exists, allocation cannot be made here.
2523  if(bufferImageGranularity > 1)
2524  {
2525  VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem;
2526  ++nextSuballocItem;
2527  while(nextSuballocItem != m_Suballocations.cend())
2528  {
2529  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
2530  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
2531  {
2532  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
2533  return false;
2534  }
2535  else
2536  // Already on next page.
2537  break;
2538  ++nextSuballocItem;
2539  }
2540  }
2541 
2542  // All tests passed: Success. pOffset is already filled.
2543  return true;
2544 }
2545 
2546 bool VmaAllocation::IsEmpty() const
2547 {
2548  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
2549 }
2550 
2551 void VmaAllocation::Alloc(
2552  const VmaAllocationRequest& request,
2553  VmaSuballocationType type,
2554  VkDeviceSize allocSize)
2555 {
2556  VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end());
2557  VmaSuballocation& suballoc = *request.freeSuballocationItem;
2558  // Given suballocation is a free block.
2559  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
2560  // Given offset is inside this suballocation.
2561  VMA_ASSERT(request.offset >= suballoc.offset);
2562  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
2563  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
2564  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
2565 
2566  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
2567  // it to become used.
2568  UnregisterFreeSuballocation(request.freeSuballocationItem);
2569 
2570  suballoc.offset = request.offset;
2571  suballoc.size = allocSize;
2572  suballoc.type = type;
2573 
2574  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
2575  if(paddingEnd)
2576  {
2577  VmaSuballocation paddingSuballoc = {};
2578  paddingSuballoc.offset = request.offset + allocSize;
2579  paddingSuballoc.size = paddingEnd;
2580  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2581  VmaSuballocationList::iterator next = request.freeSuballocationItem;
2582  ++next;
2583  const VmaSuballocationList::iterator paddingEndItem =
2584  m_Suballocations.insert(next, paddingSuballoc);
2585  RegisterFreeSuballocation(paddingEndItem);
2586  }
2587 
2588  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
2589  if(paddingBegin)
2590  {
2591  VmaSuballocation paddingSuballoc = {};
2592  paddingSuballoc.offset = request.offset - paddingBegin;
2593  paddingSuballoc.size = paddingBegin;
2594  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2595  const VmaSuballocationList::iterator paddingBeginItem =
2596  m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc);
2597  RegisterFreeSuballocation(paddingBeginItem);
2598  }
2599 
2600  // Update totals.
2601  m_FreeCount = m_FreeCount - 1;
2602  if(paddingBegin > 0)
2603  ++m_FreeCount;
2604  if(paddingEnd > 0)
2605  ++m_FreeCount;
2606  m_SumFreeSize -= allocSize;
2607 }
2608 
2609 void VmaAllocation::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
2610 {
2611  // Change this suballocation to be marked as free.
2612  VmaSuballocation& suballoc = *suballocItem;
2613  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
2614 
2615  // Update totals.
2616  ++m_FreeCount;
2617  m_SumFreeSize += suballoc.size;
2618 
2619  // Merge with previous and/or next suballocation if it's also free.
2620  bool mergeWithNext = false;
2621  bool mergeWithPrev = false;
2622 
2623  VmaSuballocationList::iterator nextItem = suballocItem;
2624  ++nextItem;
2625  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
2626  mergeWithNext = true;
2627 
2628  VmaSuballocationList::iterator prevItem = suballocItem;
2629  if(suballocItem != m_Suballocations.begin())
2630  {
2631  --prevItem;
2632  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
2633  mergeWithPrev = true;
2634  }
2635 
2636  if(mergeWithNext)
2637  {
2638  UnregisterFreeSuballocation(nextItem);
2639  MergeFreeWithNext(suballocItem);
2640  }
2641 
2642  if(mergeWithPrev)
2643  {
2644  UnregisterFreeSuballocation(prevItem);
2645  MergeFreeWithNext(prevItem);
2646  RegisterFreeSuballocation(prevItem);
2647  }
2648  else
2649  RegisterFreeSuballocation(suballocItem);
2650 }
2651 
2652 void VmaAllocation::Free(const VkMappedMemoryRange* pMemory)
2653 {
2654  // If suballocation to free has offset smaller than half of allocation size, search forward.
2655  // Otherwise search backward.
2656  const bool forwardDirection = pMemory->offset < (m_Size / 2);
2657  if(forwardDirection)
2658  {
2659  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
2660  suballocItem != m_Suballocations.end();
2661  ++suballocItem)
2662  {
2663  VmaSuballocation& suballoc = *suballocItem;
2664  if(suballoc.offset == pMemory->offset)
2665  {
2666  FreeSuballocation(suballocItem);
2667  VMA_HEAVY_ASSERT(Validate());
2668  return;
2669  }
2670  }
2671  VMA_ASSERT(0 && "Not found!");
2672  }
2673  else
2674  {
2675  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
2676  suballocItem != m_Suballocations.end();
2677  ++suballocItem)
2678  {
2679  VmaSuballocation& suballoc = *suballocItem;
2680  if(suballoc.offset == pMemory->offset)
2681  {
2682  FreeSuballocation(suballocItem);
2683  VMA_HEAVY_ASSERT(Validate());
2684  return;
2685  }
2686  }
2687  VMA_ASSERT(0 && "Not found!");
2688  }
2689 }
2690 
2691 #if VMA_STATS_STRING_ENABLED
2692 
2693 void VmaAllocation::PrintDetailedMap(class VmaStringBuilder& sb) const
2694 {
2695  sb.Add("{\n\t\t\t\"Bytes\": ");
2696  sb.AddNumber(m_Size);
2697  sb.Add(",\n\t\t\t\"FreeBytes\": ");
2698  sb.AddNumber(m_SumFreeSize);
2699  sb.Add(",\n\t\t\t\"Suballocations\": ");
2700  sb.AddNumber(m_Suballocations.size());
2701  sb.Add(",\n\t\t\t\"FreeSuballocations\": ");
2702  sb.AddNumber(m_FreeCount);
2703  sb.Add(",\n\t\t\t\"SuballocationList\": [");
2704 
2705  size_t i = 0;
2706  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
2707  suballocItem != m_Suballocations.cend();
2708  ++suballocItem, ++i)
2709  {
2710  if(i > 0)
2711  sb.Add(",\n\t\t\t\t{ \"Type\": ");
2712  else
2713  sb.Add("\n\t\t\t\t{ \"Type\": ");
2714  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
2715  sb.Add(", \"Size\": ");
2716  sb.AddNumber(suballocItem->size);
2717  sb.Add(", \"Offset\": ");
2718  sb.AddNumber(suballocItem->offset);
2719  sb.Add(" }");
2720  }
2721 
2722  sb.Add("\n\t\t\t]\n\t\t}");
2723 }
2724 
2725 #endif // #if VMA_STATS_STRING_ENABLED
2726 
2727 void VmaAllocation::MergeFreeWithNext(VmaSuballocationList::iterator item)
2728 {
2729  VMA_ASSERT(item != m_Suballocations.end());
2730  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2731 
2732  VmaSuballocationList::iterator nextItem = item;
2733  ++nextItem;
2734  VMA_ASSERT(nextItem != m_Suballocations.end());
2735  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
2736 
2737  item->size += nextItem->size;
2738  --m_FreeCount;
2739  m_Suballocations.erase(nextItem);
2740 }
2741 
2742 void VmaAllocation::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
2743 {
2744  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2745  VMA_ASSERT(item->size > 0);
2746 
2747  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2748  {
2749  if(m_FreeSuballocationsBySize.empty())
2750  m_FreeSuballocationsBySize.push_back(item);
2751  else
2752  {
2753  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2754  m_FreeSuballocationsBySize.data(),
2755  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
2756  item,
2757  VmaSuballocationItemSizeLess());
2758  size_t index = it - m_FreeSuballocationsBySize.data();
2759  VectorInsert(m_FreeSuballocationsBySize, index, item);
2760  }
2761  }
2762 }
2763 
2764 void VmaAllocation::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
2765 {
2766  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
2767  VMA_ASSERT(item->size > 0);
2768 
2769  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
2770  {
2771  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
2772  m_FreeSuballocationsBySize.data(),
2773  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
2774  item,
2775  VmaSuballocationItemSizeLess());
2776  for(size_t index = it - m_FreeSuballocationsBySize.data();
2777  index < m_FreeSuballocationsBySize.size();
2778  ++index)
2779  {
2780  if(m_FreeSuballocationsBySize[index] == item)
2781  {
2782  VectorRemove(m_FreeSuballocationsBySize, index);
2783  return;
2784  }
2785  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
2786  }
2787  VMA_ASSERT(0 && "Not found.");
2788  }
2789 }
2790 
2791 static void InitStatInfo(VmaStatInfo& outInfo)
2792 {
2793  memset(&outInfo, 0, sizeof(outInfo));
2794  outInfo.SuballocationSizeMin = UINT64_MAX;
2795  outInfo.UnusedRangeSizeMin = UINT64_MAX;
2796 }
2797 
2798 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaAllocation& alloc)
2799 {
2800  outInfo.AllocationCount = 1;
2801 
2802  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
2803  outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount;
2804  outInfo.UnusedRangeCount = alloc.m_FreeCount;
2805 
2806  outInfo.UnusedBytes = alloc.m_SumFreeSize;
2807  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
2808 
2809  outInfo.SuballocationSizeMin = UINT64_MAX;
2810  outInfo.SuballocationSizeMax = 0;
2811  outInfo.UnusedRangeSizeMin = UINT64_MAX;
2812  outInfo.UnusedRangeSizeMax = 0;
2813 
2814  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
2815  suballocItem != alloc.m_Suballocations.cend();
2816  ++suballocItem)
2817  {
2818  const VmaSuballocation& suballoc = *suballocItem;
2819  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
2820  {
2821  outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size);
2822  outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size);
2823  }
2824  else
2825  {
2826  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
2827  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
2828  }
2829  }
2830 }
2831 
2832 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
2833 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
2834 {
2835  inoutInfo.AllocationCount += srcInfo.AllocationCount;
2836  inoutInfo.SuballocationCount += srcInfo.SuballocationCount;
2837  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
2838  inoutInfo.UsedBytes += srcInfo.UsedBytes;
2839  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
2840  inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin);
2841  inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax);
2842  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
2843  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
2844 }
2845 
2846 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
2847 {
2848  inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ?
2849  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0;
2850  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
2851  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
2852 }
2853 
2854 VmaAllocationVector::VmaAllocationVector(VmaAllocator hAllocator) :
2855  m_hAllocator(hAllocator),
2856  m_Allocations(VmaStlAllocator<VmaAllocation*>(hAllocator->GetAllocationCallbacks()))
2857 {
2858 }
2859 
2860 VmaAllocationVector::~VmaAllocationVector()
2861 {
2862  for(size_t i = m_Allocations.size(); i--; )
2863  {
2864  m_Allocations[i]->Destroy(m_hAllocator);
2865  vma_delete(m_hAllocator, m_Allocations[i]);
2866  }
2867 }
2868 
2869 size_t VmaAllocationVector::Free(const VkMappedMemoryRange* pMemory)
2870 {
2871  for(uint32_t allocIndex = 0; allocIndex < m_Allocations.size(); ++allocIndex)
2872  {
2873  VmaAllocation* const pAlloc = m_Allocations[allocIndex];
2874  VMA_ASSERT(pAlloc);
2875  if(pAlloc->m_hMemory == pMemory->memory)
2876  {
2877  pAlloc->Free(pMemory);
2878  VMA_HEAVY_ASSERT(pAlloc->Validate());
2879  return allocIndex;
2880  }
2881  }
2882 
2883  return (size_t)-1;
2884 }
2885 
2886 void VmaAllocationVector::IncrementallySortAllocations()
2887 {
2888  // Bubble sort only until first swap.
2889  for(size_t i = 1; i < m_Allocations.size(); ++i)
2890  {
2891  if(m_Allocations[i - 1]->m_SumFreeSize > m_Allocations[i]->m_SumFreeSize)
2892  {
2893  VMA_SWAP(m_Allocations[i - 1], m_Allocations[i]);
2894  return;
2895  }
2896  }
2897 }
2898 
2899 #if VMA_STATS_STRING_ENABLED
2900 
2901 void VmaAllocationVector::PrintDetailedMap(class VmaStringBuilder& sb) const
2902 {
2903  for(size_t i = 0; i < m_Allocations.size(); ++i)
2904  {
2905  if(i > 0)
2906  sb.Add(",\n\t\t");
2907  else
2908  sb.Add("\n\t\t");
2909  m_Allocations[i]->PrintDetailedMap(sb);
2910  }
2911 }
2912 
2913 #endif // #if VMA_STATS_STRING_ENABLED
2914 
2915 void VmaAllocationVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const
2916 {
2917  for(uint32_t allocIndex = 0; allocIndex < m_Allocations.size(); ++allocIndex)
2918  {
2919  const VmaAllocation* const pAlloc = m_Allocations[allocIndex];
2920  VMA_ASSERT(pAlloc);
2921  VMA_HEAVY_ASSERT(pAlloc->Validate());
2922  VmaStatInfo allocationStatInfo;
2923  CalcAllocationStatInfo(allocationStatInfo, *pAlloc);
2924  VmaAddStatInfo(pStats->total, allocationStatInfo);
2925  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
2926  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
2927  }
2928 }
2929 
2931 // VmaAllocator_T
2932 
2933 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
2934  m_PhysicalDevice(pCreateInfo->physicalDevice),
2935  m_hDevice(pCreateInfo->device),
2936  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
2937  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
2938  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
2939  m_PreferredLargeHeapBlockSize(0),
2940  m_PreferredSmallHeapBlockSize(0),
2941  m_BufferToMemoryMap(VmaStlAllocator< VmaPair<VkBuffer, VkMappedMemoryRange> >(pCreateInfo->pAllocationCallbacks)),
2942  m_ImageToMemoryMap(VmaStlAllocator< VmaPair<VkImage, VkMappedMemoryRange> >(pCreateInfo->pAllocationCallbacks))
2943 {
2944  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
2945 
2946  memset(&m_MemProps, 0, sizeof(m_MemProps));
2947  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
2948 
2949  memset(&m_pAllocations, 0, sizeof(m_pAllocations));
2950  memset(&m_HasEmptyAllocation, 0, sizeof(m_HasEmptyAllocation));
2951  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
2952 
2953  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
2954  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
2955  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
2956  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
2957 
2958  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
2959  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
2960 
2961  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
2962  {
2963  m_pAllocations[i] = vma_new(this, VmaAllocationVector)(this);
2964  m_pOwnAllocations[i] = vma_new(this, OwnAllocationVectorType)(VmaStlAllocator<VmaOwnAllocation>(GetAllocationCallbacks()));
2965  }
2966 }
2967 
2968 VmaAllocator_T::~VmaAllocator_T()
2969 {
2970  for(VMA_MAP_TYPE(VkImage, VkMappedMemoryRange)::iterator it = m_ImageToMemoryMap.begin();
2971  it != m_ImageToMemoryMap.end();
2972  ++it)
2973  {
2974  vkDestroyImage(m_hDevice, it->first, GetAllocationCallbacks());
2975  }
2976 
2977  for(VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange)::iterator it = m_BufferToMemoryMap.begin();
2978  it != m_BufferToMemoryMap.end();
2979  ++it)
2980  {
2981  vkDestroyBuffer(m_hDevice, it->first, GetAllocationCallbacks());
2982  }
2983 
2984  for(uint32_t typeIndex = 0; typeIndex < GetMemoryTypeCount(); ++typeIndex)
2985  {
2986  OwnAllocationVectorType* pOwnAllocations = m_pOwnAllocations[typeIndex];
2987  VMA_ASSERT(pOwnAllocations);
2988  for(size_t allocationIndex = 0; allocationIndex < pOwnAllocations->size(); ++allocationIndex)
2989  {
2990  const VmaOwnAllocation& ownAlloc = (*pOwnAllocations)[allocationIndex];
2991  vkFreeMemory(m_hDevice, ownAlloc.m_hMemory, GetAllocationCallbacks());
2992  }
2993  }
2994 
2995  for(size_t i = GetMemoryTypeCount(); i--; )
2996  {
2997  vma_delete(this, m_pAllocations[i]);
2998  vma_delete(this, m_pOwnAllocations[i]);
2999  }
3000 }
3001 
3002 VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const
3003 {
3004  VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
3005  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
3006  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
3007 }
3008 
3009 VkResult VmaAllocator_T::AllocateMemoryOfType(
3010  const VkMemoryRequirements& vkMemReq,
3011  const VmaMemoryRequirements& vmaMemReq,
3012  uint32_t memTypeIndex,
3013  VmaSuballocationType suballocType,
3014  VkMappedMemoryRange* pMemory)
3015 {
3016  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
3017 
3018  pMemory->sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
3019  pMemory->pNext = VMA_NULL;
3020  pMemory->size = vkMemReq.size;
3021 
3022  const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex);
3023  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
3024  const bool ownMemory =
3025  vmaMemReq.ownMemory ||
3026  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
3027  ((vmaMemReq.neverAllocate == false) && (vkMemReq.size > preferredBlockSize / 2));
3028 
3029  if(ownMemory)
3030  {
3031  if(vmaMemReq.neverAllocate)
3032  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3033  else
3034  return AllocateOwnMemory(vkMemReq.size, suballocType, memTypeIndex, pMemory);
3035  }
3036  else
3037  {
3038  VmaMutexLock lock(m_AllocationsMutex[memTypeIndex]);
3039  VmaAllocationVector* const allocationVector = m_pAllocations[memTypeIndex];
3040  VMA_ASSERT(allocationVector);
3041 
3042  // 1. Search existing allocations.
3043  // Forward order - prefer blocks with smallest amount of free space.
3044  for(size_t allocIndex = 0; allocIndex < allocationVector->m_Allocations.size(); ++allocIndex )
3045  {
3046  VmaAllocation* const pAlloc = allocationVector->m_Allocations[allocIndex];
3047  VMA_ASSERT(pAlloc);
3048  VmaAllocationRequest allocRequest = {};
3049  // Check if can allocate from pAlloc.
3050  if(pAlloc->CreateAllocationRequest(
3051  GetBufferImageGranularity(),
3052  vkMemReq.size,
3053  vkMemReq.alignment,
3054  suballocType,
3055  &allocRequest))
3056  {
3057  // We no longer have an empty Allocation.
3058  if(pAlloc->IsEmpty())
3059  m_HasEmptyAllocation[memTypeIndex] = false;
3060  // Allocate from this pAlloc.
3061  pAlloc->Alloc(allocRequest, suballocType, vkMemReq.size);
3062  // Return VkDeviceMemory and offset (size already filled above).
3063  pMemory->memory = pAlloc->m_hMemory;
3064  pMemory->offset = allocRequest.offset;
3065  VMA_HEAVY_ASSERT(pAlloc->Validate());
3066  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex);
3067  return VK_SUCCESS;
3068  }
3069  }
3070 
3071  // 2. Create new Allocation.
3072  if(vmaMemReq.neverAllocate)
3073  {
3074  VMA_DEBUG_LOG(" FAILED due to VmaMemoryRequirements::neverAllocate");
3075  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3076  }
3077  else
3078  {
3079  // Start with full preferredBlockSize.
3080  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
3081  allocInfo.memoryTypeIndex = memTypeIndex;
3082  allocInfo.allocationSize = preferredBlockSize;
3083  VkDeviceMemory mem = VK_NULL_HANDLE;
3084  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3085  if(res < 0)
3086  {
3087  // 3. Try half the size.
3088  allocInfo.allocationSize /= 2;
3089  if(allocInfo.allocationSize >= vkMemReq.size)
3090  {
3091  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3092  if(res < 0)
3093  {
3094  // 4. Try quarter the size.
3095  allocInfo.allocationSize /= 2;
3096  if(allocInfo.allocationSize >= vkMemReq.size)
3097  {
3098  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
3099  }
3100  }
3101  }
3102  }
3103  if(res < 0)
3104  {
3105  // 5. Try OwnAlloc.
3106  res = AllocateOwnMemory(vkMemReq.size, suballocType, memTypeIndex, pMemory);
3107  if(res == VK_SUCCESS)
3108  {
3109  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
3110  VMA_DEBUG_LOG(" Allocated as OwnMemory");
3111  return VK_SUCCESS;
3112  }
3113  else
3114  {
3115  // Everything failed: Return error code.
3116  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
3117  return res;
3118  }
3119  }
3120 
3121  // New VkDeviceMemory successfully created. Create new Allocation for it.
3122  VmaAllocation* const pAlloc = vma_new(this, VmaAllocation)(this);
3123  pAlloc->Init(mem, allocInfo.allocationSize);
3124 
3125  allocationVector->m_Allocations.push_back(pAlloc);
3126 
3127  // Allocate from pAlloc. Because it is empty, allocRequest can be trivially filled.
3128  VmaAllocationRequest allocRequest = {};
3129  allocRequest.freeSuballocationItem = pAlloc->m_Suballocations.begin();
3130  allocRequest.offset = 0;
3131  pAlloc->Alloc(allocRequest, suballocType, vkMemReq.size);
3132  pMemory->memory = mem;
3133  pMemory->offset = allocRequest.offset;
3134  VMA_HEAVY_ASSERT(pAlloc->Validate());
3135  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
3136  return VK_SUCCESS;
3137  }
3138  }
3139 }
3140 
3141 VkResult VmaAllocator_T::AllocateOwnMemory(
3142  VkDeviceSize size,
3143  VmaSuballocationType suballocType,
3144  uint32_t memTypeIndex,
3145  VkMappedMemoryRange* pMemory)
3146 {
3147  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
3148  allocInfo.memoryTypeIndex = memTypeIndex;
3149  allocInfo.allocationSize = size;
3150 
3151  // Allocate VkDeviceMemory.
3152  VmaOwnAllocation ownAlloc = {};
3153  ownAlloc.m_Size = size;
3154  ownAlloc.m_Type = suballocType;
3155  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &ownAlloc.m_hMemory);
3156  if(res < 0)
3157  {
3158  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
3159  return res;
3160  }
3161 
3162  // Register it in m_pOwnAllocations.
3163  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex]);
3164  OwnAllocationVectorType* ownAllocations = m_pOwnAllocations[memTypeIndex];
3165  VMA_ASSERT(ownAllocations);
3166  VmaOwnAllocation* const pOwnAllocationsBeg = ownAllocations->data();
3167  VmaOwnAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + ownAllocations->size();
3168  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3169  pOwnAllocationsBeg,
3170  pOwnAllocationsEnd,
3171  ownAlloc,
3172  VmaOwnAllocationMemoryHandleLess()) - pOwnAllocationsBeg;
3173  VectorInsert(*ownAllocations, indexToInsert, ownAlloc);
3174 
3175  // Return parameters of the allocation.
3176  pMemory->sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
3177  pMemory->pNext = VMA_NULL;
3178  pMemory->memory = ownAlloc.m_hMemory;
3179  pMemory->offset = 0;
3180  pMemory->size = size;
3181 
3182  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
3183 
3184  return VK_SUCCESS;
3185 }
3186 
3187 VkResult VmaAllocator_T::AllocateMemory(
3188  const VkMemoryRequirements& vkMemReq,
3189  const VmaMemoryRequirements& vmaMemReq,
3190  VmaSuballocationType suballocType,
3191  VkMappedMemoryRange* pMemory,
3192  uint32_t* pMemoryTypeIndex)
3193 {
3194  if(vmaMemReq.ownMemory && vmaMemReq.neverAllocate)
3195  {
3196  VMA_ASSERT(0 && "Specifying VmaMemoryRequirements::ownMemory && VmaMemoryRequirements::neverAllocate makes no sense.");
3197  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3198  }
3199 
3200  // Bit mask of memory Vulkan types acceptable for this allocation.
3201  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
3202  uint32_t memTypeIndex = UINT32_MAX;
3203  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
3204  if(res == VK_SUCCESS)
3205  {
3206  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pMemory);
3207  // Succeeded on first try.
3208  if(res == VK_SUCCESS)
3209  {
3210  if(pMemoryTypeIndex != VMA_NULL)
3211  *pMemoryTypeIndex = memTypeIndex;
3212  return res;
3213  }
3214  // Allocation from this memory type failed. Try other compatible memory types.
3215  else
3216  {
3217  for(;;)
3218  {
3219  // Remove old memTypeIndex from list of possibilities.
3220  memoryTypeBits &= ~(1u << memTypeIndex);
3221  // Find alternative memTypeIndex.
3222  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
3223  if(res == VK_SUCCESS)
3224  {
3225  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pMemory);
3226  // Allocation from this alternative memory type succeeded.
3227  if(res == VK_SUCCESS)
3228  {
3229  if(pMemoryTypeIndex != VMA_NULL)
3230  *pMemoryTypeIndex = memTypeIndex;
3231  return res;
3232  }
3233  // else: Allocation from this memory type failed. Try next one - next loop iteration.
3234  }
3235  // No other matching memory type index could be found.
3236  else
3237  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
3238  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
3239  }
3240  }
3241  }
3242  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
3243  else
3244  return res;
3245 }
3246 
3247 void VmaAllocator_T::FreeMemory(const VkMappedMemoryRange* pMemory)
3248 {
3249  uint32_t memTypeIndex = 0;
3250  bool found = false;
3251  VmaAllocation* allocationToDelete = VMA_NULL;
3252  // Check all memory types because we don't know which one does pMemory come from.
3253  for(; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3254  {
3255  VmaMutexLock lock(m_AllocationsMutex[memTypeIndex]);
3256  VmaAllocationVector* const pAllocationVector = m_pAllocations[memTypeIndex];
3257  VMA_ASSERT(pAllocationVector);
3258  // Try to free pMemory from pAllocationVector.
3259  const size_t allocIndex = pAllocationVector->Free(pMemory);
3260  if(allocIndex != (size_t)-1)
3261  {
3262  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
3263  found = true;
3264  VmaAllocation* const pAlloc = pAllocationVector->m_Allocations[allocIndex];
3265  VMA_ASSERT(pAlloc);
3266  // pAlloc became empty after this deallocation.
3267  if(pAlloc->IsEmpty())
3268  {
3269  // Already has empty Allocation. We don't want to have two, so delete this one.
3270  if(m_HasEmptyAllocation[memTypeIndex])
3271  {
3272  allocationToDelete = pAlloc;
3273  VectorRemove(pAllocationVector->m_Allocations, allocIndex);
3274  break;
3275  }
3276  // We now have first empty Allocation.
3277  else
3278  m_HasEmptyAllocation[memTypeIndex] = true;
3279  }
3280  // Must be called after allocIndex is used, because later it may become invalid!
3281  pAllocationVector->IncrementallySortAllocations();
3282  break;
3283  }
3284  }
3285  if(found)
3286  {
3287  // Destruction of a free Allocation. Deferred until this point, outside of mutex
3288  // lock, for performance reason.
3289  if(allocationToDelete != VMA_NULL)
3290  {
3291  VMA_DEBUG_LOG(" Deleted empty allocation");
3292  allocationToDelete->Destroy(this);
3293  vma_delete(this, allocationToDelete);
3294  }
3295  return;
3296  }
3297 
3298  // pMemory not found in allocations. Try free it as Own Memory.
3299  if(FreeOwnMemory(pMemory))
3300  return;
3301 
3302  // pMemory not found as Own Memory either.
3303  VMA_ASSERT(0 && "Not found. Trying to free memory not allocated using this allocator (or some other bug).");
3304 }
3305 
3306 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
3307 {
3308  InitStatInfo(pStats->total);
3309  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
3310  InitStatInfo(pStats->memoryType[i]);
3311  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
3312  InitStatInfo(pStats->memoryHeap[i]);
3313 
3314  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3315  {
3316  VmaMutexLock allocationsLock(m_AllocationsMutex[memTypeIndex]);
3317  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
3318  const VmaAllocationVector* const allocVector = m_pAllocations[memTypeIndex];
3319  VMA_ASSERT(allocVector);
3320  allocVector->AddStats(pStats, memTypeIndex, heapIndex);
3321  }
3322 
3323  VmaPostprocessCalcStatInfo(pStats->total);
3324  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
3325  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
3326  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
3327  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
3328 }
3329 
3330 bool VmaAllocator_T::FreeOwnMemory(const VkMappedMemoryRange* pMemory)
3331 {
3332  VkDeviceMemory vkMemory = VK_NULL_HANDLE;
3333 
3334  // Check all memory types because we don't know which one does pMemory come from.
3335  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3336  {
3337  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex]);
3338  OwnAllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex];
3339  VMA_ASSERT(pOwnAllocations);
3340  VmaOwnAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
3341  VmaOwnAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
3342  VmaOwnAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess(
3343  pOwnAllocationsBeg,
3344  pOwnAllocationsEnd,
3345  pMemory->memory,
3346  VmaOwnAllocationMemoryHandleLess());
3347  if((pOwnAllocationIt != pOwnAllocationsEnd) &&
3348  (pOwnAllocationIt->m_hMemory == pMemory->memory))
3349  {
3350  VMA_ASSERT(pMemory->size == pOwnAllocationIt->m_Size && pMemory->offset == 0);
3351  vkMemory = pOwnAllocationIt->m_hMemory;
3352  const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg;
3353  VectorRemove(*pOwnAllocations, ownAllocationIndex);
3354  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
3355  break;
3356  }
3357  }
3358 
3359  // Found. Free VkDeviceMemory deferred until this point, outside of mutex lock,
3360  // for performance reason.
3361  if(vkMemory != VK_NULL_HANDLE)
3362  {
3363  vkFreeMemory(m_hDevice, vkMemory, GetAllocationCallbacks());
3364  return true;
3365  }
3366  else
3367  return false;
3368 }
3369 
3370 #if VMA_STATS_STRING_ENABLED
3371 
3372 void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb)
3373 {
3374  bool ownAllocationsStarted = false;
3375  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3376  {
3377  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex]);
3378  OwnAllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex];
3379  VMA_ASSERT(pOwnAllocVector);
3380  if(pOwnAllocVector->empty() == false)
3381  {
3382  if(ownAllocationsStarted)
3383  sb.Add(",\n\t\"Type ");
3384  else
3385  {
3386  sb.Add(",\n\"OwnAllocations\": {\n\t\"Type ");
3387  ownAllocationsStarted = true;
3388  }
3389  sb.AddNumber(memTypeIndex);
3390  sb.Add("\": [");
3391 
3392  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
3393  {
3394  const VmaOwnAllocation& ownAlloc = (*pOwnAllocVector)[i];
3395  if(i > 0)
3396  sb.Add(",\n\t\t{ \"Size\": ");
3397  else
3398  sb.Add("\n\t\t{ \"Size\": ");
3399  sb.AddNumber(ownAlloc.m_Size);
3400  sb.Add(", \"Type\": ");
3401  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[ownAlloc.m_Type]);
3402  sb.Add(" }");
3403  }
3404 
3405  sb.Add("\n\t]");
3406  }
3407  }
3408  if(ownAllocationsStarted)
3409  sb.Add("\n}");
3410 
3411  {
3412  bool allocationsStarted = false;
3413  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
3414  {
3415  VmaMutexLock globalAllocationsLock(m_AllocationsMutex[memTypeIndex]);
3416  if(m_pAllocations[memTypeIndex]->IsEmpty() == false)
3417  {
3418  if(allocationsStarted)
3419  sb.Add(",\n\t\"Type ");
3420  else
3421  {
3422  sb.Add(",\n\"Allocations\": {\n\t\"Type ");
3423  allocationsStarted = true;
3424  }
3425  sb.AddNumber(memTypeIndex);
3426  sb.Add("\": [");
3427 
3428  m_pAllocations[memTypeIndex]->PrintDetailedMap(sb);
3429 
3430  sb.Add("\n\t]");
3431  }
3432  }
3433  if(allocationsStarted)
3434  sb.Add("\n}");
3435  }
3436 }
3437 
3438 #endif // #if VMA_STATS_STRING_ENABLED
3439 
3440 static VkResult AllocateMemoryForImage(
3441  VmaAllocator allocator,
3442  VkImage image,
3443  const VmaMemoryRequirements* pMemoryRequirements,
3444  VmaSuballocationType suballocType,
3445  VkMappedMemoryRange* pMemory,
3446  uint32_t* pMemoryTypeIndex)
3447 {
3448  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pMemory);
3449 
3450  VkMemoryRequirements vkMemReq = {};
3451  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
3452 
3453  return allocator->AllocateMemory(
3454  vkMemReq,
3455  *pMemoryRequirements,
3456  suballocType,
3457  pMemory,
3458  pMemoryTypeIndex);
3459 }
3460 
3462 // Public interface
3463 
3464 VkResult vmaCreateAllocator(
3465  const VmaAllocatorCreateInfo* pCreateInfo,
3466  VmaAllocator* pAllocator)
3467 {
3468  VMA_ASSERT(pCreateInfo && pAllocator);
3469  VMA_DEBUG_LOG("vmaCreateAllocator");
3470  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
3471  return VK_SUCCESS;
3472 }
3473 
3474 void vmaDestroyAllocator(
3475  VmaAllocator allocator)
3476 {
3477  if(allocator != VK_NULL_HANDLE)
3478  {
3479  VMA_DEBUG_LOG("vmaDestroyAllocator");
3480  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
3481  vma_delete(&allocationCallbacks, allocator);
3482  }
3483 }
3484 
3485 void vmaGetPhysicalDeviceProperties(
3486  VmaAllocator allocator,
3487  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
3488 {
3489  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
3490  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
3491 }
3492 
3493 void vmaGetMemoryProperties(
3494  VmaAllocator allocator,
3495  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
3496 {
3497  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
3498  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
3499 }
3500 
3501 void vmaGetMemoryTypeProperties(
3502  VmaAllocator allocator,
3503  uint32_t memoryTypeIndex,
3504  VkMemoryPropertyFlags* pFlags)
3505 {
3506  VMA_ASSERT(allocator && pFlags);
3507  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
3508  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
3509 }
3510 
3511 void vmaCalculateStats(
3512  VmaAllocator allocator,
3513  VmaStats* pStats)
3514 {
3515  VMA_ASSERT(allocator && pStats);
3516  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3517  allocator->CalculateStats(pStats);
3518 }
3519 
3520 #if VMA_STATS_STRING_ENABLED
3521 
3522 void vmaBuildStatsString(
3523  VmaAllocator allocator,
3524  char** ppStatsString,
3525  VkBool32 detailedMap)
3526 {
3527  VMA_ASSERT(allocator && ppStatsString);
3528  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3529 
3530  VmaStringBuilder sb(allocator);
3531  {
3532  VmaStats stats;
3533  allocator->CalculateStats(&stats);
3534 
3535  sb.Add("{\n\"Total\": ");
3536  VmaPrintStatInfo(sb, stats.total);
3537 
3538  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
3539  {
3540  sb.Add(",\n\"Heap ");
3541  sb.AddNumber(heapIndex);
3542  sb.Add("\": {\n\t\"Size\": ");
3543  sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
3544  sb.Add(",\n\t\"Flags\": ");
3545  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
3546  sb.AddString("DEVICE_LOCAL");
3547  else
3548  sb.AddString("");
3549  if(stats.memoryHeap[heapIndex].AllocationCount > 0)
3550  {
3551  sb.Add(",\n\t\"Stats:\": ");
3552  VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]);
3553  }
3554 
3555  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
3556  {
3557  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
3558  {
3559  sb.Add(",\n\t\"Type ");
3560  sb.AddNumber(typeIndex);
3561  sb.Add("\": {\n\t\t\"Flags\": \"");
3562  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
3563  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
3564  sb.Add(" DEVICE_LOCAL");
3565  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
3566  sb.Add(" HOST_VISIBLE");
3567  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
3568  sb.Add(" HOST_COHERENT");
3569  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
3570  sb.Add(" HOST_CACHED");
3571  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
3572  sb.Add(" LAZILY_ALLOCATED");
3573  sb.Add("\"");
3574  if(stats.memoryType[typeIndex].AllocationCount > 0)
3575  {
3576  sb.Add(",\n\t\t\"Stats\": ");
3577  VmaPrintStatInfo(sb, stats.memoryType[typeIndex]);
3578  }
3579  sb.Add("\n\t}");
3580  }
3581  }
3582  sb.Add("\n}");
3583  }
3584  if(detailedMap == VK_TRUE)
3585  allocator->PrintDetailedMap(sb);
3586  sb.Add("\n}\n");
3587  }
3588 
3589  const size_t len = sb.GetLength();
3590  char* const pChars = vma_new_array(allocator, char, len + 1);
3591  if(len > 0)
3592  memcpy(pChars, sb.GetData(), len);
3593  pChars[len] = '\0';
3594  *ppStatsString = pChars;
3595 }
3596 
3597 void vmaFreeStatsString(
3598  VmaAllocator allocator,
3599  char* pStatsString)
3600 {
3601  if(pStatsString != VMA_NULL)
3602  {
3603  VMA_ASSERT(allocator);
3604  size_t len = strlen(pStatsString);
3605  vma_delete_array(allocator, pStatsString, len + 1);
3606  }
3607 }
3608 
3609 #endif // #if VMA_STATS_STRING_ENABLED
3610 
3613 VkResult vmaFindMemoryTypeIndex(
3614  VmaAllocator allocator,
3615  uint32_t memoryTypeBits,
3616  const VmaMemoryRequirements* pMemoryRequirements,
3617  uint32_t* pMemoryTypeIndex)
3618 {
3619  VMA_ASSERT(allocator != VK_NULL_HANDLE);
3620  VMA_ASSERT(pMemoryRequirements != VMA_NULL);
3621  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
3622 
3623  uint32_t requiredFlags = pMemoryRequirements->requiredFlags;
3624  uint32_t preferredFlags = pMemoryRequirements->preferredFlags;
3625  if(preferredFlags == 0)
3626  preferredFlags = requiredFlags;
3627  // preferredFlags, if not 0, must be subset of requiredFlags.
3628  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
3629 
3630  // Convert usage to requiredFlags and preferredFlags.
3631  switch(pMemoryRequirements->usage)
3632  {
3633  case VMA_MEMORY_USAGE_UNKNOWN:
3634  break;
3635  case VMA_MEMORY_USAGE_GPU_ONLY:
3636  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
3637  break;
3638  case VMA_MEMORY_USAGE_CPU_ONLY:
3639  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
3640  break;
3641  case VMA_MEMORY_USAGE_CPU_TO_GPU:
3642  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
3643  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
3644  break;
3645  case VMA_MEMORY_USAGE_GPU_TO_CPU:
3646  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
3647  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
3648  break;
3649  default:
3650  break;
3651  }
3652 
3653  *pMemoryTypeIndex = UINT32_MAX;
3654  uint32_t minCost = UINT32_MAX;
3655  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
3656  memTypeIndex < allocator->GetMemoryTypeCount();
3657  ++memTypeIndex, memTypeBit <<= 1)
3658  {
3659  // This memory type is acceptable according to memoryTypeBits bitmask.
3660  if((memTypeBit & memoryTypeBits) != 0)
3661  {
3662  const VkMemoryPropertyFlags currFlags =
3663  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
3664  // This memory type contains requiredFlags.
3665  if((requiredFlags & ~currFlags) == 0)
3666  {
3667  // Calculate cost as number of bits from preferredFlags not present in this memory type.
3668  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
3669  // Remember memory type with lowest cost.
3670  if(currCost < minCost)
3671  {
3672  *pMemoryTypeIndex = memTypeIndex;
3673  if(currCost == 0)
3674  return VK_SUCCESS;
3675  minCost = currCost;
3676  }
3677  }
3678  }
3679  }
3680  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
3681 }
3682 
3683 VkResult vmaAllocateMemory(
3684  VmaAllocator allocator,
3685  const VkMemoryRequirements* pVkMemoryRequirements,
3686  const VmaMemoryRequirements* pVmaMemoryRequirements,
3687  VkMappedMemoryRange* pMemory,
3688  uint32_t* pMemoryTypeIndex)
3689 {
3690  VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pMemory);
3691 
3692  VMA_DEBUG_LOG("vmaAllocateMemory");
3693 
3694  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3695 
3696  return allocator->AllocateMemory(
3697  *pVkMemoryRequirements,
3698  *pVmaMemoryRequirements,
3699  VMA_SUBALLOCATION_TYPE_UNKNOWN,
3700  pMemory,
3701  pMemoryTypeIndex);
3702 }
3703 
3704 VkResult vmaAllocateMemoryForBuffer(
3705  VmaAllocator allocator,
3706  VkBuffer buffer,
3707  const VmaMemoryRequirements* pMemoryRequirements,
3708  VkMappedMemoryRange* pMemory,
3709  uint32_t* pMemoryTypeIndex)
3710 {
3711  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pMemory);
3712 
3713  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
3714 
3715  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3716 
3717  VkMemoryRequirements vkMemReq = {};
3718  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
3719 
3720  return allocator->AllocateMemory(
3721  vkMemReq,
3722  *pMemoryRequirements,
3723  VMA_SUBALLOCATION_TYPE_BUFFER,
3724  pMemory,
3725  pMemoryTypeIndex);
3726 }
3727 
3728 VkResult vmaAllocateMemoryForImage(
3729  VmaAllocator allocator,
3730  VkImage image,
3731  const VmaMemoryRequirements* pMemoryRequirements,
3732  VkMappedMemoryRange* pMemory,
3733  uint32_t* pMemoryTypeIndex)
3734 {
3735  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements);
3736 
3737  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
3738 
3739  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3740 
3741  return AllocateMemoryForImage(
3742  allocator,
3743  image,
3744  pMemoryRequirements,
3745  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
3746  pMemory,
3747  pMemoryTypeIndex);
3748 }
3749 
3750 void vmaFreeMemory(
3751  VmaAllocator allocator,
3752  const VkMappedMemoryRange* pMemory)
3753 {
3754  VMA_ASSERT(allocator && pMemory);
3755 
3756  VMA_DEBUG_LOG("vmaFreeMemory");
3757 
3758  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3759 
3760  allocator->FreeMemory(pMemory);
3761 }
3762 
3763 VkResult vmaMapMemory(
3764  VmaAllocator allocator,
3765  const VkMappedMemoryRange* pMemory,
3766  void** ppData)
3767 {
3768  VMA_ASSERT(allocator && pMemory && ppData);
3769 
3770  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3771 
3772  return vkMapMemory(allocator->m_hDevice, pMemory->memory,
3773  pMemory->offset, pMemory->size, 0, ppData);
3774 }
3775 
3776 void vmaUnmapMemory(
3777  VmaAllocator allocator,
3778  const VkMappedMemoryRange* pMemory)
3779 {
3780  VMA_ASSERT(allocator && pMemory);
3781 
3782  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3783 
3784  vkUnmapMemory(allocator->m_hDevice, pMemory->memory);
3785 }
3786 
3787 VkResult vmaCreateBuffer(
3788  VmaAllocator allocator,
3789  const VkBufferCreateInfo* pCreateInfo,
3790  const VmaMemoryRequirements* pMemoryRequirements,
3791  VkBuffer* pBuffer,
3792  VkMappedMemoryRange* pMemory,
3793  uint32_t* pMemoryTypeIndex)
3794 {
3795  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements);
3796 
3797  VMA_DEBUG_LOG("vmaCreateBuffer");
3798 
3799  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3800 
3801  // 1. Create VkBuffer.
3802  VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
3803  if(res >= 0)
3804  {
3805  VkMappedMemoryRange mem = {};
3806 
3807  // 2. vkGetBufferMemoryRequirements.
3808  VkMemoryRequirements vkMemReq = {};
3809  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
3810 
3811  // 3. Allocate memory using allocator.
3812  res = allocator->AllocateMemory(
3813  vkMemReq,
3814  *pMemoryRequirements,
3815  VMA_SUBALLOCATION_TYPE_BUFFER,
3816  &mem,
3817  pMemoryTypeIndex);
3818  if(res >= 0)
3819  {
3820  if(pMemory != VMA_NULL)
3821  {
3822  *pMemory = mem;
3823  }
3824  // 3. Bind buffer with memory.
3825  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, mem.memory, mem.offset);
3826  if(res >= 0)
3827  {
3828  // All steps succeeded.
3829  VmaMutexLock lock(allocator->m_BufferToMemoryMapMutex);
3830  allocator->m_BufferToMemoryMap.insert(VmaPair<VkBuffer, VkMappedMemoryRange>(*pBuffer, mem));
3831  return VK_SUCCESS;
3832  }
3833  allocator->FreeMemory(&mem);
3834  return res;
3835  }
3836  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
3837  return res;
3838  }
3839  return res;
3840 }
3841 
3842 void vmaDestroyBuffer(
3843  VmaAllocator allocator,
3844  VkBuffer buffer)
3845 {
3846  if(buffer != VK_NULL_HANDLE)
3847  {
3848  VMA_ASSERT(allocator);
3849 
3850  VMA_DEBUG_LOG("vmaDestroyBuffer");
3851 
3852  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3853 
3854  VkMappedMemoryRange mem = {};
3855  {
3856  VmaMutexLock lock(allocator->m_BufferToMemoryMapMutex);
3857  VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange)::iterator it = allocator->m_BufferToMemoryMap.find(buffer);
3858  if(it == allocator->m_BufferToMemoryMap.end())
3859  {
3860  VMA_ASSERT(0 && "Trying to destroy buffer that was not created using vmaCreateBuffer or already freed.");
3861  return;
3862  }
3863  mem = it->second;
3864  allocator->m_BufferToMemoryMap.erase(it);
3865  }
3866 
3867  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
3868 
3869  allocator->FreeMemory(&mem);
3870  }
3871 }
3872 
3873 VkResult vmaCreateImage(
3874  VmaAllocator allocator,
3875  const VkImageCreateInfo* pCreateInfo,
3876  const VmaMemoryRequirements* pMemoryRequirements,
3877  VkImage* pImage,
3878  VkMappedMemoryRange* pMemory,
3879  uint32_t* pMemoryTypeIndex)
3880 {
3881  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements);
3882 
3883  VMA_DEBUG_LOG("vmaCreateImage");
3884 
3885  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3886 
3887  // 1. Create VkImage.
3888  VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage);
3889  if(res >= 0)
3890  {
3891  VkMappedMemoryRange mem = {};
3892  VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
3893  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
3894  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
3895 
3896  // 2. Allocate memory using allocator.
3897  res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, &mem, pMemoryTypeIndex);
3898  if(res >= 0)
3899  {
3900  if(pMemory != VMA_NULL)
3901  *pMemory = mem;
3902  // 3. Bind image with memory.
3903  res = vkBindImageMemory(allocator->m_hDevice, *pImage, mem.memory, mem.offset);
3904  if(res >= 0)
3905  {
3906  // All steps succeeded.
3907  VmaMutexLock lock(allocator->m_ImageToMemoryMapMutex);
3908  allocator->m_ImageToMemoryMap.insert(VmaPair<VkImage, VkMappedMemoryRange>(*pImage, mem));
3909  return VK_SUCCESS;
3910  }
3911  allocator->FreeMemory(&mem);
3912  return res;
3913  }
3914  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
3915  return res;
3916  }
3917  return res;
3918 }
3919 
3920 void vmaDestroyImage(
3921  VmaAllocator allocator,
3922  VkImage image)
3923 {
3924  if(image != VK_NULL_HANDLE)
3925  {
3926  VMA_ASSERT(allocator);
3927 
3928  VMA_DEBUG_LOG("vmaDestroyImage");
3929 
3930  VMA_DEBUG_GLOBAL_MUTEX_LOCK
3931 
3932  VkMappedMemoryRange mem = {};
3933  {
3934  VmaMutexLock lock(allocator->m_ImageToMemoryMapMutex);
3935  VMA_MAP_TYPE(VkImage, VkMappedMemoryRange)::iterator it = allocator->m_ImageToMemoryMap.find(image);
3936  if(it == allocator->m_ImageToMemoryMap.end())
3937  {
3938  VMA_ASSERT(0 && "Trying to destroy buffer that was not created using vmaCreateBuffer or already freed.");
3939  return;
3940  }
3941  mem = it->second;
3942  allocator->m_ImageToMemoryMap.erase(it);
3943  }
3944 
3945  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
3946 
3947  allocator->FreeMemory(&mem);
3948  }
3949 }
3950 
3951 #endif // #ifdef VMA_IMPLEMENTATION
3952 
3953 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
VmaMemoryRequirements
struct VmaMemoryRequirements VmaMemoryRequirements
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, const VkMappedMemoryRange *pMemory)
-
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:169
+
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:165
vmaMapMemory
VkResult vmaMapMemory(VmaAllocator allocator, const VkMappedMemoryRange *pMemory, void **ppData)
-
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for writing on device and readback on host.
Definition: vk_mem_alloc.h:280
-
VmaMemoryRequirements::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:299
+
VMA_MEMORY_USAGE_GPU_TO_CPU
Memory will be used for writing on device and readback on host.
Definition: vk_mem_alloc.h:276
+
VmaMemoryRequirements::usage
VmaMemoryUsage usage
Intended usage of memory.
Definition: vk_mem_alloc.h:295
vmaAllocateMemoryForImage
VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaMemoryRequirements *pMemoryRequirements, VkMappedMemoryRange *pMemory, uint32_t *pMemoryTypeIndex)
Function similar to vmaAllocateMemoryForBuffer().
-
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom allocation callbacks.
Definition: vk_mem_alloc.h:181
-
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:165
-
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Size of a single memory block to allocate for resources from a small heap <= 512 MB.
Definition: vk_mem_alloc.h:178
-
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:236
-
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:172
+
VmaAllocatorCreateInfo::pAllocationCallbacks
const VkAllocationCallbacks * pAllocationCallbacks
Custom allocation callbacks.
Definition: vk_mem_alloc.h:177
+
VmaAllocatorCreateInfo
Description of a Allocator to be created.
Definition: vk_mem_alloc.h:161
+
VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
VkDeviceSize preferredSmallHeapBlockSize
Size of a single memory block to allocate for resources from a small heap <= 512 MB.
Definition: vk_mem_alloc.h:174
+
VmaStats::total
VmaStatInfo total
Definition: vk_mem_alloc.h:232
+
VmaAllocatorCreateInfo::device
VkDevice device
Vulkan device.
Definition: vk_mem_alloc.h:168
vmaAllocateMemory
VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaMemoryRequirements *pVmaMemoryRequirements, VkMappedMemoryRange *pMemory, uint32_t *pMemoryTypeIndex)
General purpose memory allocation.
vmaBuildStatsString
void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
Builds and returns statistics as string in JSON format.
vmaDestroyBuffer
void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer)
vmaCreateImage
VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkImage *pImage, VkMappedMemoryRange *pMemory, uint32_t *pMemoryTypeIndex)
Function similar to vmaCreateBuffer().
-
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:232
+
VmaStats
General statistics from current state of Allocator.
Definition: vk_mem_alloc.h:228
vmaCreateAllocator
VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
Creates Allocator object.
-
VmaMemoryRequirements::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:308
-
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:269
+
VmaMemoryRequirements::preferredFlags
VkMemoryPropertyFlags preferredFlags
Flags that preferably should be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:304
+
VmaMemoryUsage
VmaMemoryUsage
Definition: vk_mem_alloc.h:265
vmaDestroyAllocator
void vmaDestroyAllocator(VmaAllocator allocator)
Destroys allocator object.
vmaCreateBuffer
VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkBuffer *pBuffer, VkMappedMemoryRange *pMemory, uint32_t *pMemoryTypeIndex)
vmaCalculateStats
void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
Retrieves statistics from current state of the Allocator.
-
VmaStatInfo
Definition: vk_mem_alloc.h:220
-
VmaMemoryRequirements::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:303
-
VmaMemoryRequirements
Definition: vk_mem_alloc.h:284
-
VmaMemoryRequirements::neverAllocate
VkBool32 neverAllocate
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:315
-
VmaStatInfo::UnusedRangeSizeMax
VkDeviceSize UnusedRangeSizeMax
Definition: vk_mem_alloc.h:228
-
VmaStatInfo::SuballocationSizeMax
VkDeviceSize SuballocationSizeMax
Definition: vk_mem_alloc.h:227
+
VmaStatInfo
Definition: vk_mem_alloc.h:216
+
VmaMemoryRequirements::requiredFlags
VkMemoryPropertyFlags requiredFlags
Flags that must be set in a Memory Type chosen for an allocation.
Definition: vk_mem_alloc.h:299
+
VmaMemoryRequirements
Definition: vk_mem_alloc.h:280
+
VmaMemoryRequirements::neverAllocate
VkBool32 neverAllocate
Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
Definition: vk_mem_alloc.h:311
+
VmaStatInfo::UnusedRangeSizeMax
VkDeviceSize UnusedRangeSizeMax
Definition: vk_mem_alloc.h:224
+
VmaStatInfo::SuballocationSizeMax
VkDeviceSize SuballocationSizeMax
Definition: vk_mem_alloc.h:223
VmaAllocatorCreateInfo
struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
Description of a Allocator to be created.
-
VmaMemoryRequirements::ownMemory
VkBool32 ownMemory
Set to true if this allocation should have its own memory block.
Definition: vk_mem_alloc.h:294
-
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:234
+
VmaMemoryRequirements::ownMemory
VkBool32 ownMemory
Set to true if this allocation should have its own memory block.
Definition: vk_mem_alloc.h:290
+
VmaStats::memoryType
VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
Definition: vk_mem_alloc.h:230
vmaDestroyImage
void vmaDestroyImage(VmaAllocator allocator, VkImage image)
-
VmaStatInfo::AllocationCount
uint32_t AllocationCount
Definition: vk_mem_alloc.h:222
+
VmaStatInfo::AllocationCount
uint32_t AllocationCount
Definition: vk_mem_alloc.h:218
vmaGetPhysicalDeviceProperties
void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
-
VmaStatInfo::UsedBytes
VkDeviceSize UsedBytes
Definition: vk_mem_alloc.h:225
-
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Size of a single memory block to allocate for resources.
Definition: vk_mem_alloc.h:175
-
VmaStatInfo::UnusedRangeCount
uint32_t UnusedRangeCount
Definition: vk_mem_alloc.h:224
-
VMA_MEMORY_USAGE_CPU_ONLY
Memory will be mapped on host. Could be used for transfer to device.
Definition: vk_mem_alloc.h:276
+
VmaStatInfo::UsedBytes
VkDeviceSize UsedBytes
Definition: vk_mem_alloc.h:221
+
VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
VkDeviceSize preferredLargeHeapBlockSize
Size of a single memory block to allocate for resources.
Definition: vk_mem_alloc.h:171
+
VmaStatInfo::UnusedRangeCount
uint32_t UnusedRangeCount
Definition: vk_mem_alloc.h:220
+
VMA_MEMORY_USAGE_CPU_ONLY
Memory will be mapped on host. Could be used for transfer to device.
Definition: vk_mem_alloc.h:272
vmaGetMemoryProperties
void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
-
VmaStatInfo::SuballocationCount
uint32_t SuballocationCount
Definition: vk_mem_alloc.h:223
-
VmaStatInfo::UnusedRangeSizeAvg
VkDeviceSize UnusedRangeSizeAvg
Definition: vk_mem_alloc.h:228
-
VmaStatInfo::SuballocationSizeMin
VkDeviceSize SuballocationSizeMin
Definition: vk_mem_alloc.h:227
+
VmaStatInfo::SuballocationCount
uint32_t SuballocationCount
Definition: vk_mem_alloc.h:219
+
VmaStatInfo::UnusedRangeSizeAvg
VkDeviceSize UnusedRangeSizeAvg
Definition: vk_mem_alloc.h:224
+
VmaStatInfo::SuballocationSizeMin
VkDeviceSize SuballocationSizeMin
Definition: vk_mem_alloc.h:223
vmaAllocateMemoryForBuffer
VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaMemoryRequirements *pMemoryRequirements, VkMappedMemoryRange *pMemory, uint32_t *pMemoryTypeIndex)
-
VmaStatInfo::SuballocationSizeAvg
VkDeviceSize SuballocationSizeAvg
Definition: vk_mem_alloc.h:227
+
VmaStatInfo::SuballocationSizeAvg
VkDeviceSize SuballocationSizeAvg
Definition: vk_mem_alloc.h:223
vmaFreeStatsString
void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
-
VMA_MEMORY_USAGE_UNKNOWN
No intended memory usage specified.
Definition: vk_mem_alloc.h:272
-
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:281
-
VMA_MEMORY_USAGE_CPU_TO_GPU
Memory will be used for frequent (dynamic) updates from host and reads on device. ...
Definition: vk_mem_alloc.h:278
+
VMA_MEMORY_USAGE_UNKNOWN
No intended memory usage specified.
Definition: vk_mem_alloc.h:268
+
VMA_MEMORY_USAGE_MAX_ENUM
Definition: vk_mem_alloc.h:277
+
VMA_MEMORY_USAGE_CPU_TO_GPU
Memory will be used for frequent (dynamic) updates from host and reads on device. ...
Definition: vk_mem_alloc.h:274
vmaGetMemoryTypeProperties
void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
Given Memory Type Index, returns Property Flags of this memory type.
-
VMA_MEMORY_USAGE_GPU_ONLY
Memory will be used on device only, no need to be mapped on host.
Definition: vk_mem_alloc.h:274
+
VMA_MEMORY_USAGE_GPU_ONLY
Memory will be used on device only, no need to be mapped on host.
Definition: vk_mem_alloc.h:270
VmaStatInfo
struct VmaStatInfo VmaStatInfo
-
VmaStatInfo::UnusedBytes
VkDeviceSize UnusedBytes
Definition: vk_mem_alloc.h:226
-
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:235
+
VmaStatInfo::UnusedBytes
VkDeviceSize UnusedBytes
Definition: vk_mem_alloc.h:222
+
VmaStats::memoryHeap
VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
Definition: vk_mem_alloc.h:231
vmaFindMemoryTypeIndex
VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaMemoryRequirements *pMemoryRequirements, uint32_t *pMemoryTypeIndex)
vmaFreeMemory
void vmaFreeMemory(VmaAllocator allocator, const VkMappedMemoryRange *pMemory)
Frees memory previously allocated using vmaAllocateMemoryForBuffer() or vmaAllocateMemoryForImage().
-
VmaStatInfo::UnusedRangeSizeMin
VkDeviceSize UnusedRangeSizeMin
Definition: vk_mem_alloc.h:228
+
VmaStatInfo::UnusedRangeSizeMin
VkDeviceSize UnusedRangeSizeMin
Definition: vk_mem_alloc.h:224
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 8d66b2d..f8db677 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -130,15 +130,11 @@ This function would also free memory bound to it. \section configuration Configuration -Set VMA_STATS_STRING_ENABLED macro in vk_mem_alloc.h to 0 or 1 to disable/enable -compilation of code for dumping internal allocator state to string in JSON -format. - -Please check "CONFIGURATION" section in vk_mem_alloc.cpp file to find macros and -other definitions that you can change to connect the library to your own -implementation of basic facilities like assert, min and max functions, mutex -etc. C++ STL is used by default, but changing these allows you to get rid of any -STL usage if you want, as many game developers tend to do. +Please check "CONFIGURATION SECTION" in the code to find macros that you can define +before each #include of this file or change directly in this file to provide +your own implementation of basic facilities like assert, min and max functions, +mutex etc. C++ STL is used by default, but changing these allows you to get rid +of any STL usage if you want, as many game developers tend to do. \section custom_memory_allocator Custom memory allocator @@ -459,12 +455,14 @@ void vmaDestroyImage( #include /******************************************************************************* -CONFIGURATION +CONFIGURATION SECTION -Change these definitions depending on your environment. +Define some of these macros before each #include of this header or change them +here if you need other then default behavior depending on your environment. */ -#define VMA_USE_STL_CONTAINERS 0 +// Define this macro to 1 to make the library use STL containers instead of its own implementation. +//#define VMA_USE_STL_CONTAINERS 1 /* Set this macro to 1 to make the library including and using STL containers: std::pair, std::vector, std::list, std::unordered_map. @@ -473,21 +471,21 @@ Set it to 0 or undefined to make the library using its own implementation of the containers. */ #if VMA_USE_STL_CONTAINERS -#define VMA_USE_STL_VECTOR 1 -#define VMA_USE_STL_UNORDERED_MAP 1 -#define VMA_USE_STL_LIST 1 + #define VMA_USE_STL_VECTOR 1 + #define VMA_USE_STL_UNORDERED_MAP 1 + #define VMA_USE_STL_LIST 1 #endif #if VMA_USE_STL_VECTOR -#include + #include #endif #if VMA_USE_STL_UNORDERED_MAP -#include + #include #endif #if VMA_USE_STL_LIST -#include + #include #endif /* @@ -502,117 +500,168 @@ remove them if not needed. #include // for aligned_alloc() #endif - -#ifdef _DEBUG - // Normal assert to check for programmer's errors, especially in Debug configuration. - #define VMA_ASSERT(expr) assert(expr) - // Assert that will be called very often, like inside data structures e.g. operator[]. - // Making it non-empty can make program slow. - #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) -#else - #define VMA_ASSERT(expr) - #define VMA_HEAVY_ASSERT(expr) +// Normal assert to check for programmer's errors, especially in Debug configuration. +#ifndef VMA_ASSERT + #ifdef _DEBUG + #define VMA_ASSERT(expr) assert(expr) + #else + #define VMA_ASSERT(expr) + #endif #endif -// Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0. -#define VMA_NULL nullptr - -#define VMA_ALIGN_OF(type) (__alignof(type)) - -#if defined(_WIN32) - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment))) - #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr) -#else - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) )) - #define VMA_SYSTEM_FREE(ptr) free(ptr) +// Assert that will be called very often, like inside data structures e.g. operator[]. +// Making it non-empty can make program slow. +#ifndef VMA_HEAVY_ASSERT + #ifdef _DEBUG + #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) + #else + #define VMA_HEAVY_ASSERT(expr) + #endif #endif -#define VMA_MIN(v1, v2) (std::min((v1), (v2))) -#define VMA_MAX(v1, v2) (std::max((v1), (v2))) -#define VMA_SWAP(v1, v2) std::swap((v1), (v2)) +#ifndef VMA_NULL + // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0. + #define VMA_NULL nullptr +#endif -#define VMA_DEBUG_LOG(format, ...) -/* -#define VMA_DEBUG_LOG(format, ...) do { \ - printf(format, __VA_ARGS__); \ - printf("\n"); \ -} while(false) -*/ +#ifndef VMA_ALIGN_OF + #define VMA_ALIGN_OF(type) (__alignof(type)) +#endif +#ifndef VMA_SYSTEM_ALIGNED_MALLOC + #if defined(_WIN32) + #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment))) + #else + #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) )) + #endif +#endif + +#ifndef VMA_SYSTEM_FREE + #if defined(_WIN32) + #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr) + #else + #define VMA_SYSTEM_FREE(ptr) free(ptr) + #endif +#endif + +#ifndef VMA_MIN + #define VMA_MIN(v1, v2) (std::min((v1), (v2))) +#endif + +#ifndef VMA_MAX + #define VMA_MAX(v1, v2) (std::max((v1), (v2))) +#endif + +#ifndef VMA_SWAP + #define VMA_SWAP(v1, v2) std::swap((v1), (v2)) +#endif + +#ifndef VMA_DEBUG_LOG + #define VMA_DEBUG_LOG(format, ...) + /* + #define VMA_DEBUG_LOG(format, ...) do { \ + printf(format, __VA_ARGS__); \ + printf("\n"); \ + } while(false) + */ +#endif + +// Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString. #if VMA_STATS_STRING_ENABLED + static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num) + { + _ultoa_s(num, outStr, strLen, 10); + } + static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num) + { + _ui64toa_s(num, outStr, strLen, 10); + } +#endif -static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num) -{ - _ultoa_s(num, outStr, strLen, 10); -} -static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num) -{ - _ui64toa_s(num, outStr, strLen, 10); -} +#ifndef VMA_MUTEX + class VmaMutex + { + public: + VmaMutex() { } + ~VmaMutex() { } + void Lock() { m_Mutex.lock(); } + void Unlock() { m_Mutex.unlock(); } + private: + std::mutex m_Mutex; + }; + #define VMA_MUTEX VmaMutex +#endif -#endif // #if VMA_STATS_STRING_ENABLED +#ifndef VMA_BEST_FIT + /** + Main parameter for function assessing how good is a free suballocation for a new + allocation request. -class VmaMutex -{ -public: - VmaMutex() { } - ~VmaMutex() { } - void Lock() { m_Mutex.lock(); } - void Unlock() { m_Mutex.unlock(); } -private: - std::mutex m_Mutex; -}; + - Set to 1 to use Best-Fit algorithm - prefer smaller blocks, as close to the + size of requested allocations as possible. + - Set to 0 to use Worst-Fit algorithm - prefer larger blocks, as large as + possible. -/* -Main parameter for function assessing how good is a free suballocation for a new -allocation request. + Experiments in special testing environment showed that Best-Fit algorithm is + better. + */ + #define VMA_BEST_FIT (1) +#endif -- Set to true to use Best-Fit algorithm - prefer smaller blocks, as close to the - size of requested allocations as possible. -- Set to false to use Worst-Fit algorithm - prefer larger blocks, as large as - possible. +#ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY + /** + Every object will have its own allocation. + Define to 1 for debugging purposes only. + */ + #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0) +#endif -Experiments in special testing environment showed that Best-Fit algorithm is -better. -*/ -static const bool VMA_BEST_FIT = true; +#ifndef VMA_DEBUG_ALIGNMENT + /** + Minimum alignment of all suballocations, in bytes. + Set to more than 1 for debugging purposes only. Must be power of two. + */ + #define VMA_DEBUG_ALIGNMENT (1) +#endif -/* -Every object will have its own allocation. -Enable for debugging purposes only. -*/ -static const bool VMA_DEBUG_ALWAYS_OWN_MEMORY = false; +#ifndef VMA_DEBUG_MARGIN + /** + Minimum margin between suballocations, in bytes. + Set nonzero for debugging purposes only. + */ + #define VMA_DEBUG_MARGIN (0) +#endif -/* -Minimum alignment of all suballocations, in bytes. -Set to more than 1 for debugging purposes only. Must be power of two. -*/ -static const VkDeviceSize VMA_DEBUG_ALIGNMENT = 1; +#ifndef VMA_DEBUG_GLOBAL_MUTEX + /** + Set this to 1 for debugging purposes only, to enable single mutex protecting all + entry calls to the library. Can be useful for debugging multithreading issues. + */ + #define VMA_DEBUG_GLOBAL_MUTEX (0) +#endif -/* -Minimum margin between suballocations, in bytes. -Set nonzero for debugging purposes only. -*/ -static const VkDeviceSize VMA_DEBUG_MARGIN = 0; +#ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY + /** + Minimum value for VkPhysicalDeviceLimits::bufferImageGranularity. + Set to more than 1 for debugging purposes only. Must be power of two. + */ + #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1) +#endif -/* -Set this to 1 for debugging purposes only, to enable single mutex protecting all -entry calls to the library. Can be useful for debugging multithreading issues. -*/ -#define VMA_DEBUG_GLOBAL_MUTEX 0 +#ifndef VMA_SMALL_HEAP_MAX_SIZE + /// Maximum size of a memory heap in Vulkan to consider it "small". + #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024) +#endif -/* -Minimum value for VkPhysicalDeviceLimits::bufferImageGranularity. -Set to more than 1 for debugging purposes only. Must be power of two. -*/ -static const VkDeviceSize VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY = 1; +#ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE + /// Default size of a block allocated as single VkDeviceMemory from a "large" heap. + #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024) +#endif -// Maximum size of a memory heap in Vulkan to consider it "small". -static const VkDeviceSize VMA_SMALL_HEAP_MAX_SIZE = 512 * 1024 * 1024; -// Default size of a block allocated as single VkDeviceMemory from a "large" heap. -static const VkDeviceSize VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE = 256 * 1024 * 1024; -// Default size of a block allocated as single VkDeviceMemory from a "small" heap. -static const VkDeviceSize VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE = 64 * 1024 * 1024; +#ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE + /// Default size of a block allocated as single VkDeviceMemory from a "small" heap. + #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024) +#endif /******************************************************************************* END OF CONFIGURATION @@ -721,15 +770,15 @@ static inline bool VmaIsBufferImageGranularityConflict( struct VmaMutexLock { public: - VmaMutexLock(VmaMutex& mutex) : m_Mutex(mutex) { mutex.Lock(); } + VmaMutexLock(VMA_MUTEX& mutex) : m_Mutex(mutex) { mutex.Lock(); } ~VmaMutexLock() { m_Mutex.Unlock(); } private: - VmaMutex& m_Mutex; + VMA_MUTEX& m_Mutex; }; #if VMA_DEBUG_GLOBAL_MUTEX - static VmaMutex gDebugGlobalMutex; + static VMA_MUTEX gDebugGlobalMutex; #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex); #else #define VMA_DEBUG_GLOBAL_MUTEX_LOCK @@ -1927,19 +1976,19 @@ struct VmaAllocator_T hysteresis to avoid pessimistic case of alternating creation and destruction of a VkDeviceMemory. */ bool m_HasEmptyAllocation[VK_MAX_MEMORY_TYPES]; - VmaMutex m_AllocationsMutex[VK_MAX_MEMORY_TYPES]; + VMA_MUTEX m_AllocationsMutex[VK_MAX_MEMORY_TYPES]; // Each vector is sorted by memory (handle value). typedef VmaVector< VmaOwnAllocation, VmaStlAllocator > OwnAllocationVectorType; OwnAllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES]; - VmaMutex m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES]; + VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES]; // Sorted by first (VkBuffer handle value). VMA_MAP_TYPE(VkBuffer, VkMappedMemoryRange) m_BufferToMemoryMap; - VmaMutex m_BufferToMemoryMapMutex; + VMA_MUTEX m_BufferToMemoryMapMutex; // Sorted by first (VkImage handle value). VMA_MAP_TYPE(VkImage, VkMappedMemoryRange) m_ImageToMemoryMap; - VmaMutex m_ImageToMemoryMapMutex; + VMA_MUTEX m_ImageToMemoryMapMutex; VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo); ~VmaAllocator_T(); @@ -1954,7 +2003,7 @@ struct VmaAllocator_T VkDeviceSize GetBufferImageGranularity() const { return VMA_MAX( - VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY, + static_cast(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY), m_PhysicalDeviceProperties.limits.bufferImageGranularity); } @@ -2427,7 +2476,7 @@ bool VmaAllocation::CheckAllocation( *pOffset += VMA_DEBUG_MARGIN; // Apply alignment. - const VkDeviceSize alignment = VMA_MAX(allocAlignment, VMA_DEBUG_ALIGNMENT); + const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast(VMA_DEBUG_ALIGNMENT)); *pOffset = VmaAlignUp(*pOffset, alignment); // Check previous suballocations for BufferImageGranularity conflicts. @@ -2902,9 +2951,9 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations)); m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ? - pCreateInfo->preferredLargeHeapBlockSize : VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE; + pCreateInfo->preferredLargeHeapBlockSize : static_cast(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE); m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ? - pCreateInfo->preferredSmallHeapBlockSize : VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE; + pCreateInfo->preferredSmallHeapBlockSize : static_cast(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE); vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties); vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);