From 976f920c158cb5e8982030091c7fff3fbf542354 Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Tue, 12 Sep 2017 20:45:14 +0200 Subject: [PATCH] Version 2.0.0-alpha.3 - major update. --- bin/VulkanSample_Release_2015.exe | Bin 88064 -> 96768 bytes docs/html/annotated.html | 18 +- docs/html/classes.html | 6 +- docs/html/functions.html | 68 +- docs/html/functions_vars.html | 68 +- docs/html/globals.html | 99 +- docs/html/globals_enum.html | 9 +- docs/html/globals_eval.html | 39 +- docs/html/globals_func.html | 30 +- docs/html/globals_type.html | 30 +- docs/html/group__general.html | 34 + docs/html/group__layer1.html | 117 +- docs/html/group__layer2.html | 414 +- docs/html/group__layer3.html | 32 +- docs/html/index.html | 203 +- docs/html/search/all_0.js | 5 +- docs/html/search/all_1.js | 2 + docs/html/search/all_3.js | 3 +- docs/html/search/all_6.js | 5 +- docs/html/search/all_8.js | 5 +- docs/html/search/all_9.js | 2 +- docs/html/search/all_a.js | 6 +- docs/html/search/all_c.js | 5 +- docs/html/search/all_d.js | 41 +- docs/html/search/classes_0.js | 4 +- docs/html/search/enums_0.js | 5 +- docs/html/search/enumvalues_0.js | 15 +- docs/html/search/functions_0.js | 18 +- docs/html/search/typedefs_1.js | 10 +- docs/html/search/variables_0.js | 5 +- docs/html/search/variables_1.js | 2 + docs/html/search/variables_3.js | 3 +- docs/html/search/variables_4.js | 5 +- docs/html/search/variables_6.js | 5 +- docs/html/search/variables_7.js | 2 +- docs/html/search/variables_8.js | 6 +- docs/html/search/variables_a.js | 5 +- ...ct_vma_allocation_create_info-members.html | 82 + .../struct_vma_allocation_create_info.html | 208 + docs/html/struct_vma_allocation_info.html | 17 +- ...uct_vma_allocator_create_info-members.html | 11 +- .../struct_vma_allocator_create_info.html | 30 +- .../html/struct_vma_defragmentation_info.html | 4 +- .../struct_vma_defragmentation_stats.html | 8 +- .../struct_vma_pool_create_info-members.html | 82 + docs/html/struct_vma_pool_create_info.html | 212 + docs/html/struct_vma_pool_stats-members.html | 80 + docs/html/struct_vma_pool_stats.html | 167 + docs/html/struct_vma_stat_info-members.html | 8 +- docs/html/struct_vma_stat_info.html | 55 +- docs/html/vk__mem__alloc_8h.html | 114 +- docs/html/vk__mem__alloc_8h_source.html | 181 +- src/VulkanSample.cpp | 49 +- src/vk_mem_alloc.h | 4083 ++++++++++++----- 54 files changed, 5196 insertions(+), 1521 deletions(-) create mode 100644 docs/html/struct_vma_allocation_create_info-members.html create mode 100644 docs/html/struct_vma_allocation_create_info.html create mode 100644 docs/html/struct_vma_pool_create_info-members.html create mode 100644 docs/html/struct_vma_pool_create_info.html create mode 100644 docs/html/struct_vma_pool_stats-members.html create mode 100644 docs/html/struct_vma_pool_stats.html diff --git a/bin/VulkanSample_Release_2015.exe b/bin/VulkanSample_Release_2015.exe index c213b25c77969a61f888231aa7691a2b4e9ecae2..4eb1623491e2eab28235c662c76001e3e5888d8f 100644 GIT binary patch literal 96768 zcmd?SeS8$v6*s<{WFaVF7G*WQP*zYC}bI(2Z+%vPGYZ`62Hk&OEAH%TO)+41qFMt2|S2u1C7_wo2?dg86oUz^> zeC3Qu)i=y{*UXzcd)_bSxTpW}=9}k+-M^aYo)@{KwuNMwyxH&_I z5BwKjpX0m+LRs6StL==| zw$^UzAViIJ+v@&&%lwMlZBHZDagp6tlTnSN0Dh12Gs(%{106pFm^859L=(D*pU=5%-!}!H(f*&a+h3M~PrR3;=WZak=-Fh!&r8yCWeWa|6#T7e z_;)S%Ym)G1FuYc9N{-#uIfyQZznWwBk|Z;5?VSjYxE1lHA~q@7#C$N~l#|dQ_4tx) zih7))jz^J_Z@7N%M5)+}p1P-6yyy1Bl>l&rs7`E;BFURFZW2EAT2KBI#-TZ0a#0&! zDXLw(r9_+2>e*=Z$Yy)EU_&8|g4m^t!;~fzajmDAkX_%iy-j8}2sQsHYS5FfsFOU- zpz8M&265@0cr_s`CQMq`Q~4aR)Nq|SkdU|QUw>~HGIcKOaR&*}tBL?66X24lDu@~(s(QP?~Yd?A! z-Dbq%lF|$urPp6KEr|rYdu$Let9D|DVw`=VWaojh`e%;M2GU9xB*XP_0ret145-nG zA`6=5Pbt#?vJ6G^Q)1h~gACV`fN6}y!~I07;cBM~q64hu_KP!WKoIV3WNc~!C8$48 z5xznNwe!)S(bm{?oFcXxuH6Hu8Et}BKlwwfCyJO@sfY;`KT_jCt^p<|dvD5z`)UlA zR}qfAH==VTTNI2WM(@+0<`@j#_{DZb!?12R9`xup62CPtS&8m2W;H6t=DLxEw#ZQ> zwl6$fQ5Sov6tPngw>uSW&=CEdzZ2gshlox>joe)rA-()7Dr3MP@RYo)UviL2uJu%d zDcL6^R|FdE0c_XZR?`9Pa%z5wBG5f!JB<|Bzu4o&aJhZgbvC@1Rthu~ z-M&e_t9`6=g5upE)IjtVLmE%1_ps1g9{1LjM6U5m7bY?H08%&@D1VmE^K<>qZd$eTN{Qx4i zc^gLb?V972UTlf#{lcT8x;=bGv^zI)O%M1SeURDcP&C?gan`-=$C;{rbnn!xdvBZf zzGUq^A)go?Fj|7;UrNhetRc`};=Mv##aQ3usbsER@4L=-E$N4}R6ydTdaNCw8|nZo zD<*DvgUmNeP@;<)zpw?h5si`Iz9VD1?csARnMn#j5cEJxkN}JVO+-&8)()F5dV4p< z;U-V1wBpY+PNt|OtGUwEhFjH?=8J=fc5Vtn#5sG;S@{oi-Y1b2);fF z+Omg4*!As?nQRDXSLBx*;`!oj!1@KYEDDJ0!N`er{i5e>wtyIiDc*l7n_zC3V|fkG zsSPmUiUMMnIyzt94_%kl`b}2ripJ{$0MYK5zLs%C;K#SUz(*yCvEUl!7Wu?a^cCE& zBwp3u{)+1C(!;$`+0&?Q#=_DA)tBG&LbYd&)$dGH*X8M_rf}6*xbl;@pyHsrNXxCX zDx+H`WaCnM1LnnFG|XJPX(A-aHwa#nRYo@=rxzMHw4u|C-K zo0FuZWej8C*4J_8m_3glHQDo*-u*33Bm4CDEcSSk$kN=j9}`)T9!Mj*)IxS~Z)BSv zHo1nLHDn`umWQxM&+f6nlhTNZik=l)^OJkF`-s`I$1J4HNs`k;b$J@;Y1v3e_eMIT zH&Xq^-mP~3^}hySoY}I?~aK~(cfaaxM#b*Fnji>iImnW7#CzxU=6E+T~X|VU2%Qq z9k_4pShCkI+6hSO{McS;x3G~{qiBycU}0{6%^~VNb+ksSKDRp1uln+WDvVwAR%gOj zM4LQhVVLgJFL~K!E7_xnS9A^IUJ@rerHFPFXmKYe_;T|$Ll-b0$j9IYIPVE1Cw2wW zK(A98sGx8f)_}2If-T~eu^cTbLe5um_gU4F{lfy;Zgv^Fh~&1_>R;I7WP21wr}&z7 z+hw=7!zaL?#g^&{>^{GQTTD^s1K&7opyFXEC?)_g_A6uZ#fe6nw{)R1sEq>)uti6W zpyY2cT!vW-C1i|&$E9SuB~denc|+_6^@X%Yo(#))GAv{4Uptj$AM4;*P|UnOBzF6y zb)ki_X!nbQup8w1ZIP3MWxp(*Kk&l1vTO?hV(6y%uAR17QCE9lf`QzpJsr&S7d-8h ziX}^fC~HGLD4tq{BosR$)nf;T>t(d3EZG;p-mLjRp3?kP9&J)CXoN*^in9A4c^`7) zKFOIEpC!$2&!448x5tBJRfY5WDB3mlu3YRmmGU>|wJFW-q4qaQB2Pi3m^@Aqc^A;uzCy-cT9qGEg#UtkOgH2o zRW$h@QW44^H44@eFS-iR5%OFlV6!}%r6H)!@VIA5OXYv)&!UUjexLekHS+We}p}+RR00G7hP0j3zKlTqqh~>r5KhwTitE2YU{DTgma7P zM*cx$S5b#GYH!-J=;S|q;&m#nq`qtNQR>f2R8WB{AogQppvf=!8-gA$ct{1uW`zDP zAmYB&$HOBbqA+yjqmmPBY(6kZ$-M$2)PYpqFAW6Y<<0X?#?4)lGlYV^$bKRXyaH2h z9Hzk3{GgaTg&0UZrB239BaTFuI*rH}2&70#oF#$6l*phkr7ufh1L6*LXwZbsYX^$L zwf3&(Gxbl_B)NJDPZM(s=?sc{JWc3aNjyYevKKrs&ulWBK)|+^IxW5qiq3$Vqo{oq zwNGiihp*%FkrNO;%x?v?2AJ6jXF$kKW!NUA-0!K8mahVwB7&aUp!O>x2v*R{Q?$X$ z-y$Kp$y4oH4i!PA@wXjYSi(4U0TtO3?^YeX<&a3 z=HZ+9vjWoXt)X(cMyh`S_kv=iM=@Sej=mlgUxv>5Vpd}a%E}QEwywO880QU$3fiFu zaw`gzvtAFBzY)33C%iD}U-gx5zTF|(d}4F7+p(}vStag_+I|W`b9WdAw9(%^2AkW6 zjAnvCBQ!mA!%ypmA6xxOMGF~$+?N&6tQ`GP8MZy3=679$deQEj1>3uhi+=5p?upmO zBQG+sv6*O%>v0R*YO+n@A_!)l61iI5al!Jq6s_RNJHt~G9JVQxyJftVmlG(Lozfkb z;0Bmthko}2%bOS65FFMVoF77&sF2LJ`!@7?50hXB?#@2Ye8xA6SvU!E`_Ob9nV0Z5&>*3czo+X=OI-J5i0iIU`Bo{m5d7kr0ny(>m4;yib6W!CqYI@w*3%X~ z`VC~@UaN=Z%)|=W8Wi4q^gyb=9k)W-L?fhK2dTgue=~H}TeJ=agcriFDOlbfndTD} zkb-7^`F3<=lTYlXfxV-W9I7-OmF(_F^&@ew>vWTDX~l558jzf7QtT(V4w)20{+SVJwBW9p)&VSo9&Y#o9zR9 z`Xhf5K2+9$&-b|hG}0~j?6Jzy_7yGHg{FZ+np=EqI+d?+2g#(lFeJW`oRB(ga)(cJ z2Zzmfk_T5zFANR)P#P+C=ocOj2@|qST~t(JDC!v8mn)FEgX$sxeH5;SbcG57A_VPv zU1>=?sLs@}BzFgLckzfIf0*o+Rz(eUz;U21=~hHVzP@ccS(tJM7PAgQk5&!?R);Oh zW@eGxLCbCj*_-;}erz^{X8@U8NJt`#gisO!v`a*?krWu#LijeJyaUMes{aMii9jSS zFM^4t`3y;JR>va$emK_{TNE66dofv)wKSqNwB*%6ykJ^-lOsDVyGe^Kp*MQU&tr5G{r(P?*9vtvNRIHVD=16U(lBn;=oI~PL!N|Sj2~*1bGzpCI5L% zfR4Y+D6}q*Kny~8v{#xe!g7tJ}iE=l>H4e$j8_RK1Mp5flqDsBQ)X0=EKLh^cX%y^n-ki zi+W?ow=iJi`oY(PYzNvsvt+ugvTz}!B9-J#q*4AA2^BGmTUuXBm zm6L={;}SBuwvi|6ux{O#MtZS@ba-#1&;Q~I zf&S8&wlKu=b(#gw@euYv@`v6?^mlvHGB*jPxxz<#Qa+=1qrN0`2IXRDp{{bnJXvmk z*OaCOy>aCxYillN7Ow91s8){-ypBB?DWwc05D-h9`rtIK6M!pNHoY)B6Qc$V)!$2_ z=P#Yn@NML(tmal&&4Gzn5N@QSBpg0R583H_Cj)Vgel0^@LFZl5zC;9)ns))g(Sc_M zoLwJbwFo&zqe}wfYT8KYPr_^siKQ^|3@LUC_DDh6(J9eIbVv&w928K<;dz-9^8r}o z`mFX}`>n}%5Z|dGVPNtu1s)%W*D(UOY$JEVzP_=bn*^6zyKx+%ppEoM`Z$>3R&jg9a8_ z_RC@02s@CD@8Fxe4&El5>y_-YyqdHU1<9KTb(Y~53rNnKv^^BW_CMACAN7|h$M%2$WL3fBq0<*f$jV()@0ok~=wHY5pjWf?>4rPQ$h8W9o$E;zTDvFn1v7 z;N-X*>kQJ2PYJRG^kx8iL-6Z8FeNDNCVs(y75754A;g;k%^i7x=5Clwi#^4(?;4ZY z%VtH~T8MZ7ZIF%qf!mc_7=6w>m}!uumk?UZukQ?&A&kr ztpSQ*B2MP+mWKF2N7-@VlNBSbL^r_)w`txd)D!N&)Xf3yHtb*EM@h^df*m(%Q_Vo` z4yH7!LkV07cBoy%j=hQ|Xted8H_>IPk+iAAIm=X{r3kwt>?7z@9Yd~=ZyDPOxH1r3 zp_u+r#7Ic>>xoT=QsZmv42Vku;sIisB2NEYDZ9}fJ~8R{HC#LXL$NmMjc%wjb`?W@ zCg4MUVn`9fbI4EZEuHIxk9cdk-m6wUOaanf)b?RuqOciQc02UX_Zdc`qFn)9hKM0{ zNMMMdg|Syo2L0?XTyGyF^q@O-6}o4&6%cpvL_kef5IzhV-2RGaLrtF*FNCn8ho~xI zIUr2L6o{2VyNn8ACIuZ7`>sPY(`lXQU=DDUU2`1YMRq88uW7FV|8*1m*Httz6LPW1 z>D8XKxDJT}w7(Od$|Z+n@p|;u{&t^q&!(W*7)P&tqHnXF6Ya43$~T4k`84_a&U5+@ zGwjm3{D3wV(OvJrA>+#&4I zWHGi_?1hI`C;JUfu0q9tb6_1IpLdtlvreoowm|fyIwbapbN8y)8%?i#QmA7$Ejui^SY2U7qxL`m4tIaoy1`S>{f zYIR}4dpZ@VweGIm0NgNQV;e5v)wBri?-9?mqQoy+upen7y01P=EAq-XmmcS9?#%Vo z9e_@{%1Is%?sic0^(DN8)N(+@zmgwNE1W*H=5$5G)^Mq}D5TaxSmm~W>MfNU6mRE* zJdEj)zBy8CIckFW#FIezShp0tg$ouSE)WMaK}3vID&h_3fBgmwcr1SFWxl%GN7y14 zE4%*_C?DaOcYY>}Y{HzDyotk?bOS|vQ;H#811Rq@XA|EZ(aeL6F@%R!g*6U|f};^T zce;((#40*7s)P$1k&1Kv=TW)hqyLvt@teaU^RPhVcj_}Q zfLc==Qmf6ecnOxXPe{$3c^F&twL+?Ygp0PSu_hXpXe-q12k_nD5T=H{YO%|&vnECZ z@~hw^zyu7G{lT19Q^*zydB8kp;>zzZ!x#uE>~P zSkq99*(F0>d(Eyk*r+(Y_&FpDD*>NYtRBaMnr#Lv<*K zV=x;P90$;(!eLLK`~e~!-r)*q`Q^}&Xu-D;B{r#k83e|MQ}bZioK*{MRiZngamD0D zf7!2#A}|5qJyj>8EN_5UDd-%+*Cq%3X=#03@_GmXI10C2ANu3L!D+5u}qr++fC!_XAU%BD$ zJjeb=HSyy6|ARo@Z>ou0<~+0p4Qo(H8Oj1;L47Lv_=mKGB5=+O*Mi9XztI&{&=nZ0 zq_Uu8r@HHOlRMxFU;8%Vq}vGh@onTqB(3k?P3!wL$T#{()0s;Q&X;KANjk%NI7}3= zB^iTNp(MQIyoC`^32jhasHin=#AP9Bxa5dme2+y32WD_4`bx#W<@$JX)u~CZI(;&z zIFqOdh?|Qc-6^F~OiHDcmh3W^@NDnkVA;v164sr_$6bS^BCIZOIN_TV8Jd_xEAT7& zC}MS#T?RNd{0&AVx}*rXI^^nJB9X9#YZP$@iL_Eau5iKhp!kkN{a1>4u&O|tl%3Zc zmW&qU${BFpp{_CFji6a8pw$>k`5W_3!H#M-j#e($^0p!t0WrK+;@taubP8%&(SF+i z4Ej!ZcDnku+(C%#);T~Kx4GcwK@;HwF{!zGg2;ots1z$@Mas(|R@FRQAIz%6bbW-3}e-!j!9V$Lp5Yxz`m^ls}z!s6N98| zf;gLMLSKF2>U^UuX@U$%njmPopO1s{H_&*?ny)W->trM3IVv$+*L9G1kZzUg??eg6 zN#HRu!$^4Za&R3~t6-K?!B8Jx=x?aPqFUwlt5rovG54E^?f}@Os91(oL~$g>Ds`xa zd__>5Rf$g(KGpct;8Pn^AEDdaUG))K*HIfL=p!utDjT`DYQhaNeZHW>HzE{kDV_9E zIc&B5)L&vOtDg!Eu4J93OYe}${35j$&`gt=9HH%S_*muv z4`5-BjTrzOxju`+*Kk5+QHXiW!M~XJz5O5QYB-eLrQgrhO#cVAwQ2WF8V+XcEr(6@7s55cE_?NV|Van9RmeqM!qBMoV-BFuro9>_Xjxjq%)>&@EsyoqhkT67)T zo@bNWp2Ku49QXd*dU`@WSr(uAFm2Sy57I+0yv|^b_~3Vt>**n#cAQ21gUF(Ie#7

{mpMGcb&zTK1B8 z7*yqt@N7;{4IK>~-xgFSLoe+Ms?)IQd$BCW!@pFtio$3uqFv5GM8HSEvK&Pky*()G zR1+mfgXPDWCH<(8E)T^G1j|k^%LY=5D3OBYr`#+sXg7yuCbfpod-?*uy+^$Gtu-ZVQ*in8I<;7H9)~`o7aQ>#0;Rp68mr$-S z{3+>jpCQ|P_9b~Twr>G@#2KK#qYf=;?`1$bW=MD*q8)mU2O}S6aRc4oyVZ zu_T(?Y{#4F3337Qm>{8(=@FQFIP7k~aBqi~H zI#$7_qGXS6eLm(XT3VG0durq++(lU_E})O9^kFCw0WT;SxKH8i~j;VKXUxnyC>lj6G^Y55vp^ce}c{6!wGvS4C(vd;bfGb=-!gkrFUmC%nc4You79O-^hvj13jw z0tUHF|KKSm^Vsp#8OQhhd>`U7DEQbb3s~@<$_U02oq^_q2rcaPMc;)6d@H04`t^8m z=WfyqK0J>r#YjOTmlJ_Dght%0D`h#-9m@&7SV78)9{Ymvych?i2!tdV4vHOqD^>&% ze7Ifg$L%8WdAs!yU?Y~;lPUh8ijJkfpgp+o2Pm;Ppd^OE`Y26c^I8tx@q3|_j*H=a+ z_t>;y61fBB#x1!mqk}NRbsVI~FAih(<|)0N){f7CwI@gd=zOB3^HM@5#+h5TkU(dX z8#Zioag;#Ma6-)>L<7nr+haTcsx+x)yoT#vdq@J}1bMEwgO@YYbsSJDv5Zv0m0MW| zuMPW-@vsG)$3d|<8L}Hvan5v)wV5(^Qsm*`S+8RETZI{3?`Z`);LUN$YW3-|T3ZxQ zm118*btyc>-U(XXOMDol{g#*F+}6?|Si0Y&eK8y)NelexwzPrKhCB`=#$g5N$nzd~ zM>%a<^Z<_T66S8huJ2k+vn*A&I-~AARF}47a0!7bnC?}_v`4%XueLV-3eUZ-d?G&} z+C?*VVbPAxe<|YVg zDz_3YKUND%xrOkH)>LQmKoK1n)n@dvB;NIIa@Nvto9`Q-NOBp;*5-rg`;@*LNPk9Y-C=ay=GVXNXtu)U*H`SM5h;#aR{oSL4rGiPbvOJ zn-9c~idUfKEA+)MQUVE?ZHj<)eJS##G;S|p*kCgG2^JPcD+6BvU>G1cA_AR>Tu!v9 z6-x9Igc}632TPB$*SpBk%v_dbqo|bHd6f+)uApCyu^mlBmY1dyv z&t~vKt>nIwwCAue%VMWrU{3ATfA%SH5UP+K`ToF2lyjr=o~K2@O>PgxC*(XD-U?!% z7ozVJ2wBJS6UfDh%F$1|a!9?(2xFD=d)_Tf-}TBAK@NWU%T_Yg(?i-7c_zI+VMo1_ z1cHj=hy?FXjdv=!h|@i6w(axygfA}>W>9aKkpW|qh*8^tXmdrdylvj2*tYu1wAGNL z1(hM~jyy1%;%|Kdb0DJ^-u@{%y};xpOzA#)2c_=|`i(l69%RfLD^!J>p(c#ei^^Z1Y988*=&f@Nj?mj_{R;1jf0- zHEtXdJ`Z49S06udPejL!8OJVfy%14 z2!!Qhb`u$xSpA8ZNP%BmRn!HWdvvj1KxJbQw(bw4`xLL)+%ta#(O(|&6&}~h?Q`MVtBIV7v`PPK<>8A5b{jf35>Z9 zF`}y%eHg$bg-(Xc`+p&Dh)LiZWUYh1UfiNCr-)r^LJ_gO0F#={Ov%5ObSL*TsjFFk z^W$ZoimM8v2T8+y4L;-L2VKXDxkX)AV#gJWBER?&i{(0AESEJ~izR%}KJj+xPm)V! zReIT5wk^G6-j95vxPtZP$ux?6*V7u^b%IZ9DIHfTt&G!%0}FKLs_|NXtS2w*Y(m&+ z2(}3#|D6Hx8j4|}Kvy+?l}i>ee0^zsq-c2qX}XtpiQ!bARsISVo1 z2w+e6%N`;%CapLfhuT5yDHT^~v2`TFY8p=br=6S@ zl*&Y!k748+W$~?Foe6hu3r=N^@_M0Q4-#F31Izk9K(op9hosoEAgx7Sh{bxBu5G00 za!F7uaKi0?qMiDwHxT7x5z0V8zIcxS#YJL8CGY_+^aAgy&H(r0F=c|gxOcOgGvF4% zn~GgB^kl=F{qSqU2L{}DKZ1V#ZD>vFzW_rA2P%84<-P2%miMa^BeqL_t2Yg^Gulp) z9NlS>JoR}Zc`|l6lK$Xxq?Zhw!=g{Wwg7(406kc6a#nlX_h0qK`^oHd-{FUsKO()} zB7cH`oeQfIdpYzAwuOH6t!B5#2bJvBM`pJ^p>FXAt?*%L)8_heF|vbtdbl@bOENGn z;sg0-TjZC3BaQ1|ZEsx9nz&xFSop`@4SO;gPO{Jq(I+|zH}o3yFK5xMw2)w?FXYh~ z$xXjO+`eEvRokTx?Tu|+R&|oGzhPn$_}?%{St5By7Pt)&x2MBnxD;ZE$e(7awz)Tv z6fhmsnKc^ms#GC@4PlSA>oeL$IOSx1><^Ol z?E+|7V@t4nIsyS-BL6N^dkZ8ji`wh5TD&l$#T2z=Of3Rbe}LZaW>6QI-tVE`f0*9a z7|;|DoY17`4T4^%LO&i}M%c$u9LUdE2w#_9#dS0e$C!K+GZap+G2f6FK;@;jgTL66 zR(CPv8x;{rATkSw83`pS?}E`1Lid~D>%tj?h|KmOJPEp1DL)(#M^La!TK?qMh`@mo ztiKmC)nwId=o&uEBAsIn>?9mZD*{g}u}a*b_vOyeJD2)^QwuS!N5aJcV<)B1cS%@` z=nj3~bm9=d!RGB&^8^bHH4Z>d!a>dM(Er77+~yKeeb&pC^kEUIADNA11M9`YDEk1h zUR*=ULHh&kjBw7hG&w#{`PkW}#|tOVN_71yQm#_{DCi?r`TxSD?-|zk_dQE|?LkUW znv^qHgKI7gq|>Y#ysj7Y*T14$OgJeD1{>SE(O1)8r0!|afogmS;N~lDSlwS5rcU*r znyK!ybX8Yx3To3$Csox-i{3fU5WSNZlKy$>Um4ndu>}xhfF9i++Os)Oh}gdY`k9DN zhhzremNg~vAvPLA3l#%iWF%rTM0OMWxJWTzNnvZ1md^$xOWDKdw52|JOBvKh4>}(D zWs0#WDJ!+4EF&H}S%;-<4%cC6ySe2@q)kq@-9y?o&LfD?Vmz53RMcif*Ns3DWalgH8h9gyG3jNe^a$Gu*+a zd%-QtfTMMn-UqUX2=)m=b@x2GUv}-rUQBJc+vlWKDYypM`KH< z{G)jvm;-r3Z_RpjRyW$OPxm^dSucQRo$;lCc?Ec!fmx$Iu_a5RUY2?0znlU2%s*41 zep5r5Fr_3%Y)p?-74lG$mNB|)3)!gIPBZe0xJ`d`1r+ZgDcs47Yr>V+C%!UVBR0{l7jG~?b3w6HLBLZnEpRt+Y#Gib=}_^qKM`YAv(p%6 zc14d!E^EJ)exg)+!qbL0*7e2kq3C~IgQ~+iFxc^=iCu+qdMQQ?r!RPFDHV>KWEWz5 zWDFZzoxDCM&d(W&7rfAUglNXQ=;YD}g{qLDybAVr6FzJ3S&Nh3Rn$kf1puFR(1Z9E zc-QF*r%-rq&)+~VaMR!<*Mt8g=|rcKk3NuY-#d*c;dk^cq6(E%p^Cst>76E2vq&F3 z7eK80D00QLbrFwbMQ2ozsm9T~ujr+RfP=C9`zgW(XCSmFRwp!tiF5;FBf-c$6EPqP zO|ttpQ{zP7)U{M44PBlAZS`Yi+khnWVA7f5h#y2($YsxvH$+CUVSV& z4qQvU?n%es6dkYr!!RhQWS4&HlRYVVEUTVH_-9Ob7H5|*tjSple5r&;+tBGMVD7=) z_vcWTXl40f?hJGaFe6`(34D#c24ctZ_I=5;pOJDRB`QAtJYbJ z2#!m_dJ?gKIobbM`2U#IBH`cHoW!sGy(j(~vY;)5|70?=_&17SnSW56eor-PP6Ta2 zl6vs(uQw68Jn)Yuveth&KQUSj>9m8RH8{Y>CKGSYI8wiL4TQu_ks{Qyk&HEqpNVUF za)|kv>d$jo@^Eh!G=>&5(Ayc%EV(XA$?lpuLCafEw|F_;lL^I@mhY+Q+Q$)GNkChG z^INo-QF~8k$KpP^w^Cy8wjvYK%QyC z3{i3qM=!N4z*0(l7{P;-#fO)E(VGva#fi~W<*6rnj_ci7(BK0CHwkhfm`ow9ANtCk zF}(?6VHRRsD#!&ermUV_#8S*n#v<2vn0cHesb7Qh6zx`X!p3KdHwpo(|XmY~Wm}nm7-e)oF;;f3q zus@~H6lI~w&w{3IY#;4kE%@U_IS-oXj)&@aRt^_%QpOwzM7hDHA7#$4yVS1l)l2N9C|3LJGetxJU$`a zP|B>h`S~P|^Vakf#rG#=@gSwFwljnI<=+-ps*?Iw96iW{!X_yA>H1RwEo^Y)6|zGM|{Q>x%^?)qRU@` zccAAgGc$|S$(aLqcw(y0OloOz0DqR=PgwB0-ozQ6$FF()4~^e_=|(f?brHR=$&41V z%zWjhWw!igJWPQn>DQCF?IuU#w9f1vOwa1U#LONHPa{K=?h#rq^OBwDy@rrnu~`Pd7nT*{>UTT(Qq9!Lv5z5VPhT8>1XJCW@H)O zNyWlKwWJNCl5r+$H(X7qsfcb8773>W*q}Gz8B_iF)i4qBh9HvRGI)qq5dkce;PKmiQ9oSHp87w1I*bEct$64(35XgQ)BupxBp+{MbHOROX3;I5B+7 z2WbzVMc-2Wdtd`!<59$K5XOo#x18vX{#|{|3_|D-EB_XrgT*rBi|)wrg^_-- z+t`E?-F!JWUjD|s-$ySq*3$u!Pm&QqLoZ6lfebcmd|HjrCf{|NL1v%MV!$8>;cYsP z>B;W^LT9#xPeaI{QQ^e4nBVMZGi3zFs|Lz4m~GatRYbw-r`iz$w)dYTKn3~OLDKft z&5w_`Vcue&iH#NiiU9(d%)Kyggn9p~UC9AX2(QCN1Tm=%XbiTs!jg%y3d8jd#u+XD zozdhIn&_5ViW{SNa|JtH3OT?rQU~ggU7mWx)|o>U9p~d^xcsL6SQ}BpKZT$qVhy-} zoZ9f3OZAyLWV{WL8fl&5gmpICk@nDOabL6}2WwXjt`Un@qu)>SkmiOqxkk}6P-W9k zxsB&$8`%q9y{m$+_#r+e+U=Y-k)Gxj462N>Iq?+0$Ro`|!hH=_!{2c}$Q1|) zbG-a?nv;FO$ro@9PFCm>r>8hs0Zul8czstb@s3_%Fb^-#L}!?%75YN#7Vy9i#F1

DWOOF8|$>K(y*})VyzIcLq zpXSEcw=eE_DjbDXIntT#O@jz42* z!%U_Z&0Axo-Ccp%sZQ!Y(rkS_Z3%&=(s0R)GIZE*O{bjD+K5an8HAYW=A}r1~XHfT%{^ zaBU*b7$Tq$3^f(!)%$SCD~b7)d|w1K6TM9nZ)3%|*?uS>{lY&PM)aE;#4CK0k0;!* z_*NONjgO)+tzbI<4A)v*U@)lRRER5d(PQ}i06Z-Vk9WbIzEDe=>rs@H7d(V-66<_A zB3Gm__Dd z9$D)n9-g435$~VNem#M3*omkTJcK60upD(fegh|n!77C^msj-%%Vm1qoZN)Z8VDr3 zeR7;$;efOi?+HoBYi+bdilfN#n>;D#wzL%I(Fi`^bpT-liG<@VylD3)G=ikz>C4pr zP~Jg|IWGX_$}z2hT7baTU@lP9=}xMJp(w?J#SlIGEI=H{(Ke3kR2d=Uzx^AH0!g3P zC2LpUxk@ZXv?9UjI5}T0#ETna@!DCmFe1o!m$3lm<>{l$_osY2cV9T;qZ6 z32Sj3$8{XnZMbg3bqB6HaD5ophZXcD@}wdL`{{>Yw1Q`lU|5zjp2_gsj`|DTP)FT~ z#9z7yq>h0sPsryn!cfo}OtFuO;rbL@CHC*8%EI+)ZjlCHLazZBz|-1s2|#?$(P)~? zr(vKGARq7VU*|!<^SKTuz(Qh|53h}}#vz1v5M+%5UfhUrK&+`X4z7dvJr=@kxO8Ac z`0z*&wUrBZ=nuaE!(JV=4wqe?biZ8BSe48{IEspZYU09CXCNU-FD6NEBT0uewS(R^ zq#gA8D|mULCFw9__d~w*kpz{H4-=bcB#X1<{J%&yT;>flB_Z9v<6fktyZDEs`zh8j zN$I{kTe^S#T2i{vEZB;`+mv)q$mS~E!@|-%CMDg)rgY!KS`qHn4C(elq;c9q(k)~3 zz=9sq%{5KNKESnspLrrV9T(x}^dNq-@LWGlaAFrRmL&_X_v<4$3(f=xvX|pgCC52t zek^{)!&xvC7xc>e%Zr8MY`Q^D$BI$&s4e;MXy*K={2Km;YfO2PT>5R7|6OvcfZycua15m-x$kk|F1_nj zmfY8HNs^oFiffov$?VZ6R77`A;=+O{lwb?1u)$65MP;p8307A!tCK1gE7ao;F?7M?P8Q>Awfj^ zp70sHYh1)N_*mqhTq9IL*276lN#(5ekWq?FryHA21aU9I=94#_84;<9z(NNzO0B`# zQCk?qD-ZD8!kRS{39{_t3an((&>CGDTHEfIhI(7^rL4zS#TtB7uEJMU1HRVO!je#g zqs2f6Xz7f_fw^Q)B4$H6fTdl_N0tMhuNKtJ!A~LPmGn(T$&o~i-a3zeEUvhqXAbIc zf#Hm~t$ZSeUkGFnQqhT6k#z-|j!F};^Q;1&_&yP%_tKn=780=$R&gR$fm}YyV7Qj1 z3X83rUwsC$44Hy^q;w3{>IuA7S3=xy#z*@yd{xs|4W#N?YlW^fSLjF`Vo0FJB8~XK zAc!>KgX2KFI@EAo^A~gS!NY`C2N5~67*vutR2+qRrQc2vJLJ>w3wKx^e9>ly{wP8R zFvDh&FR2#c*a%cisjw@0sIV4VpvPIEj>!eO!dw;}`zx^-fj~RZmV^mkr{9jrh2#3B zuAk}AHqv0z!f3=()r zuu?wLge1Xb=&22pq6VK@+^^$Hb9<(;ZEm=3W+anqd}8M;zcjbt0ovK-zCro3-$h=U za)|P-50RfG&Ao{3Q^ENkm<6Nmw+b9#D>L{$%B1~gv!t81;RoMlNj2Nxzj$Mov=1-; zo!dN1TCj7LPg)Si#Q}WUak1}%S%_P0sIJCb*y zH~ueXBdiV52cp0A`_Ed_zEp)7oCDjTGEo!&DCI4GAG<lD-><6!6FpoMtcs&&Ms;d89WJ^2anlWbQ$%@W_hHjn<}u%}hnYUx%5#4qXTn^%pI6Zc<5n1-W$Kj z9fH~9lZxga#@C__St=UShOY^6d`(=7uc=M=n%;=78|z?Nky!)6ztH8W;Re<}xPH||0kUqC<7RC^i*s+1wzr2t^xH%JQXy* z5PWnTA&$U+gN0(gp$)SDuryt0M!0WBiWeQ>Wk(a@`qB%`!sRG534u9Rq*q(T%X$`H ziDGid7b)T$ay=9)<^Ktn^3w|G^;(qkR+-F}f(C4M4A-B*j-%K!*>Y^Q$ni>abDmQE z@5o*_Vel&4K|v_A(%`VIc%B4jWUuQVS3>b8$QxBjBU=zAob;w8Acmdww5N`Uif*a0 zUc>|Y*YW|aDW%gpysZV=1CzJSjvp`}HJ4!>gF)|qfxB$sQY39qbG!;7vGM*GKiUkE zUq@oNe)cE!64CCi3KNNXR0Wt;6Xq#~DM7cOt*Z2cC9J^8@pI5gzz0ph1_nG1fYj0q ztECxe>2?!fHUr#nFSj&9Z@Q3M>Swh?fRjzYOBv7$K+yOmR1SjJu*}l~zsj)9cpqyn z9Jcm|%U@QG2LUk7mFE$gz_?0!&T=z;$|-zGwDi0PUQ1P*7w*TQ*5%Dz_#LRiF1V^s zQ`GyN;D@3Po(}BbQ#~UoAp0NQ{qO+1i-D_Q@R}o^$?%SqH{(5@Ek2_~*Sl!ucSF)o zLS=DL9Z}mN9E*|S1jbQ*SQ>ZOv{&?3{#1^aQD~#9;4`PmtoOrK2l#Fxv}%k@44h=p z_x<(w7O_DfOJ#FJQ8$t;l3=p{8RW?5fO@nE#|Yq3V`PgYVo&3n?vtA5f)}g;;CY!k zWNvRn!O5Sa?T`~OgOrg6@xzqp?chXgwN(WMF>=t{PShf0+11ytZlt@|9u~r)E~Hs& zT2mOCt>}i~lKx12a#ay7?E9PqA_^YkdsA1@y~%}&>hj_~Nk)Yw8Bl};^$c`58la_6 z*G8~Nw0w^QQV0mwan!0Pn96`>Gp4~ua37ufh>~ILc=0eE^x79bhu#ll>|pGJU^*0> z%XP}Qj$;D>$M!|u!MLqM9$PiqErM^Pp<_k3KvHZbrDoza6U9tO-2$Z2OsdSJ8VQrW z2h!{l`~79;ptRyQXqQw4?T#Hsze7dQ9|nv@4JTsME?A0jd>@pkF+Z+@>PEtYgz+R8 z4*_9v6E4aoVuX$hg`*R#2DSce=76|y16>}0yM}84x@>GNzn0z+c^j01{tR@5XjdKm zxZ_@|nxt@9GU+r&L4iJZF(H_QER$AHMx<%i;#w43!1*fV6R{Z<1c*uHZnARf{j|jd z#Whv|gs0IgsYT)jQj}C6%gZ7dFSp_Y&dBWvd5hVz!Q+4p-jYV@pgQU??&p|77}exV z!OF6b_^GX=ZkSugNAAhovxZTAv>iKC+MFtyqng06Lbzcy#350obd44KFm{)gV@O6( z%7li8D1x7x`tL5yPrgkOaA;-9ivF4lcj$YX*)i0G%PtRXGRUn-9RLOEQWd}bJ@@fp z6o6?M!<0bQ(tu(cY9@_lvdT=F%w#rg`VJ!tc_rV>@4jKkZj`Wr933?QCAVX< zuf_-*)$^#aL@XZ(5tQI&DTjN6HW7Ot-!>8xjcLo;LExN&^bvLtIBch!8=8sCM%eq* zv?zEIC=J)r-|=en8^n2(Y(Mt=CHYh@sa@-5*7|GXaX0I zW4yu|==h2R)q_0TWcVhU;2*Pc@E(D1$5z8t5&2V^q_u!yRIk`V6hPY(kPH z!Pwz4BT%rwjZYCi#rTxsqu@i#s=^~&9=tBWWMGnEUh5Wq1oR?knPcMgtoS`iP0WAI(A@nA1T`BZiyOM3`4#{0vt| z8Z+&70ZcnlY)2l11_EX()HlCFTEPkj*+@moutW;WMiUFY{T?QLWg3HrG2D*ZiC70J zoq!JyjZQ77vBPGP&#Z>6Y)u?%JY!%KC@46G2vId25V<4SMrtDEVziP#rN z+-M0hLnTPM=?2)%0uWn;1RSiPt3<38$*+krc)pM&QGl<@Duk~KIX*bUx`8ZE;Wo7} zn{%s?E99ILHBUqicnXn4;x?0FBxzSSrL`Y4hiZlOQlra5u`ODch)Y8%nkQD1{=# zH~qm~&$+tGz?lw{T{;qM*H6N2CL}q>y$N{)Q=r8dXL^&4PMCFN+HusJY}HK9#s>5U zUi2mPPz`s&)nK#LT?TIx4Kapf$ts$xrmLBlioP|O2>|J8EhUNA6G(^v*!B3PHbC@x zWRS#>0J0FhK~iKU#Yk{&fN!!LAP{t$v}VVBHXD8uAa**gQTaTEBRz4d*$nAgG*fIQ zrDj6P6onN?(9{SdIDWu44PzoU5;==I77<>CXhPek03x;5S%U(7@I3AYvSx?YTRHUd z1uF|2Tad-%1jy;!f^U5T(WhxTiu5;cAq`FI6bTVn3F5lVD-udvk=R1r#gV0V6PlVY zM(=haWn6<|H|LX%|045>NXETl^U748u~J-tRbDC-j&na}5XHQ@)x4^}6{xN>A*+zU z5XLu~Et=y-hJqJ@j0#ws8g-1>f*|^Wl~6H;>kjM=cwMLP6j4yeh0BFixY7xhqWA_b zz7fR;>@&iEWDgywJu@zCl=n!21Jgdag9FpXAuugLfoc2sdRVs-H8?PB7hgvW4our* z0(aA?-GbTN_`Wa#wRxeT-E}hPZ8Nw1dlP&N+TKiUNBJ6?E`-rg(A#{zj()>!&oY6r z_u8SKI-7bic>1Tz%1M-@w`#*u&sSyc*A{|zhN~0Y0rStK8rFX8C)|k3fC^QHxeim( z{B^E=y@&i?`vH7E-*McrGZl*d?NH!6e}&^wYEF%@3V?*1O=1V} zNzY_nR~542gkw1a%MEl|meK&LgyU3{V?qmui!*;s`W9i210gnM?Ys(^OYHrhQP1{* zJ@k9c!g1&maL7-X_NhaJ> zvwlgycr_X9k{>^>^-hNO&JCKQKwEN%Vkclp!F4aK`;>c0@^9iaTz|yGSJd+>dB9vM zCBz7TOiO}x^Z8!_R?lZV!gU$dw|clJiSD)-qdUt)mzM!Td|`?fniT0oif}EX;a)=-u&r*`+6JCsi(iDHb0xE7kpNV#;1a0yMgXYEWOTr0S3NF>+M))j}!wk&N zCu{%O#5|GiMYp&!&h9y$jr^CXpV?Wn3gOT2VS4icY;GTq;lp%y;4b?xi~ACZhx~{a z_tKliI4<%GFbp0TFIJ4T(8Kz@Sl%%2@23lY86X$n^*XOwz+cp3MIGfpi}pwu61^Tk z%CH^E`ey0dIQ=U}JE2?f@ZBb*S-(ulZBuUenwOQ_Z}7WTcr7$FRd5KCO`kD`HvQoN z%CgN2QNBNNpbJ0du|IMujMwb|0p#b>M13M zG`{dd+xoe%4{%FA#;yLiWi}zZL*h#`Qd~FcVl-@z{0vA}Vzp54B4_=5m?vmr<1w1p zhFd`q!P_(ERI}+rOJp5LL()iz=D3(>$1ympMifA>sZiy%*sO=SXqCe|naUl8KEc0f zuJRgfEzYpwh|S4`7~!_oQZ3{XB>#h4RT!}SElH4#&Aujmk5u5nf3goMh$AcZj(TeGguJNP_2Q^*G7n2iyvVs{ z+yPRA{%GyMqrZ0|BYlQ0=!Kk+CIbHAAob*C0Q|0&p3 zT+k>01ZA3&Mq%b`g7hOxkck-i0w`xqJiJVCPL-lL8nI+%PF+OF&lyDUO@KuevrB7` z0ZnVoByJ{cNc`%PbQM&eY^6dWR|dtLiomd~rnnOb?zJN+*(MxQ_t|V^jsU);mCZ)8 zsy$7WG-e-~rwszGhZ&BM#v~VV`Tl<2HikB>SC%znhqWj zY6!a1i8;qqAV~kAKhv9#RYw50K>R!;AYQnB`4(VB!8Jf?xK`u%1UDc4$hui--W_=xR3Q0 zFEutti0XluFgG=82`-|W@uDoeN#||gq&5z3%v#= zZu-y?F8WESf)_xU;d%o@tp5UGr@DVPJ+ef@T}5Bj^hJ?qRZNNBGs8-&t@I`*Vd7Iv zH*#QFJMJLqKmtE5z6p|MI2w%UxBv||CuvyOh60dr5tDH;*k>ZIAiNY;S<#2u_j@K$ zD@ZG?^jc11#YoOD@s=V>V`e4_609ru#`rLLG|8cPP!K4^KM@l8AUV^hB@priC^KBc zN#unifx8$8gc%XqGf{=8GatcGV>MjEaU281h#@!$z&HMZP%Z>yE%F%t*#x4kq(*=u zz$k89WC^9mecI>J)(CGbn?({K#og-(DQHrnP;~07YpQK>}g)&qQOoi-qMAsAZ=`I5O6sf<6s5yFt%5g{fUVwe7~*}{9Jp&2)7B|7>mj$Vl>{zZLU0(P^l$k-S1P%E*b$! z!xwTnA8t=D8!E?&0i}lCq1gp_)ag*=*JvZ`NGIHSJvU zQO0#qG2e0B3%F!oCMU^RZGtSJdx(D|NSuI=mz?xC$PURlp@__@VrgK|wrmx!!?OMPMaE-${u?@zQ@d4-N>;);5g~cP|ft(J2BJ}Up{U`(#mEhg^$YVhk1&h&a0KZT~$ck8*LZRY; zyZ&I*s6iLb+#=Oa&V%h&w9|&J=1q^NirQ@If=4=(R zFJU(T7y4SJ2khzraFBTOB4RG<}X*{Gu75?}_f73`=t`Kb``wc)cJEeAac%?d>-Du|#N9JWw1VDW5l0EY-Pe7L95y@9;uqp%8v`Z~~p4)tYP z9Bq_Pudx0R;?j~m_%7z5B*{Y!FyU|Lu;H3C7leYsBSg&M&!t~n^v@_^Y0;0HMrk)? z9mtwJ2j$dLsR%R=#8X`~{(uJqnA9n*m(BiD_o&O%Tgi!z(FLkZLY0AR%kZ6{n%TpG znW)AsR+tO7K&%4gW@M4=i1u)$KToWZjdELj=TJy?= zhHw?f6*B?=P81P8yeKnV^B@!utyj@3?I~-xT}FT@P(xdLHXG?R#VBwau5I&~jy>1{ z5u;5q;)ZJ(8iFq^i#+tV2p`nzoJaLo=vva*ecYPkW3)!?*MgKnG@;;NmFLnLd}XJ7&`1OzUcgk(Y@AqkTS548#gY{qF?Zfje$wxzbWwQt&5 zi?p`}P>A>nz6xU7XnWg9W7}xGfYds_?|1KgX3hkBw4eLO?~k73%-WB&_g-u5_u7xs zLxUTJK8%uf0QTdgB?N$YSQ!CM5Ja%@B&F?Io_0qz0JQLPEqsd=NSUPZT zbml*hPBi<6AZ6Z@5#vu7z<&;hE-|F(Lkt|!1eXHxd~Y(~&d1s6O3fa$yM}@Du1^z9 z?!(MSbPbXj?DRysEyx{TZXT9KXG}+DG`%!8clVRG&)?m1`;|qRI}xrUt;oI!FgP`o zxtZ~VGg}ZoWM2&nzAvPR`U!6Oh`fhmg3QW%lJwdV;C4Q4#5arbye#fV*m+JCzE8de z<{i5Z;5U4zGhz!~&7Vs<&n7!|W9TfrXL@jIDB3Y0ay0-X+=*)`A{%kV$h;?U?rM|l zz3IlvgxZ|B2l)~y;^sfOiIqhe(TGUlK2n9U26f*a_Fmx^B9Gq)H}uH!z|FRkBvgML zTYh>2_i>=ZMZe2n=rv^0vIl-uB}-@N=_t%`xJ3o=ZU% z1#)(t!4$%2v*930&QUks=w zc7DP!HkeD-Xm%xukfuVK=3jf?36$~2BG;UWMhEvJY`AL{ zFYk%D1NTNcjRPpCd@eJlAC!~6qC=eOjfdcdPxZ{pIQ}lVh+oDrDDClE8BR&NzLnuw zh1?t2y+l@=P%^%t25BMbPU~rZE#H`EAyl;PF}VhBH})$+L8+GM*U^tev!7{{lrDJb z{p9&75h9g~E+o$dbd~a?wixo{AjvAxmnuz(CJNIm50YMa`#!{qudNlGb98=IuXAK03Hh-!eybJISgG_n3pfW+d@N2fn(c33Ics_^4ek4ds%;1TBZDRIJ+IonMA!GPwn+=%{eIOp z#(-IWFCdSk$Yl^HTBP@{UDoU&|K~_6 zmZYiDni3wN%BqzlnMbmG$@aj9ON&ux@$Fhj+8ch-i?*@oc>6`(?iBQaX#Ffzujsk< zPx$0PqCQFTVej;VzV57RyhVrF7a85fQ5x6^{1l@(>@TSU>E+W;Ea}b~gI>^!HXa}F z3-@}H4;yEWdyBBz&Q<_lO+SQIpf9pPlg`g3CHoL{^3jNG86~Y-IV&--q9zSyc{BTq z8)biiUc%g^qONB8GlglydxKv~d%QPzM%s06(4&xhBU?*9w7jbHF>BxWEa`p47tUS- zla%K%G=VIf?ODe3lMw5D7$%hxP4+29(Nm7jM`WqCUqx5hazU_Ya}nxL`#P_w+Fa;m zV|ZvWxalK;H`j4}KI_y?sj!RdXGHmGdDc?jN$^Q|?5e)md0z?0buZT8Cq_P!tmWZ%nIU{UWR|g336mGTdGz&qRLVB?_h{=;9DJ)@?lafJsjfE|_!od6crdH=4bf zafLP_5XlMMiiI|O0tIYW<0F}7FGPKUR3FQINgQ0Oc?-rr>PAgEMEmye6JEymYN=Hlh z*?6RQYYxsrmS84_xg18>7avJBbJ67H>8DF^2-jucCvAnySezg)&2=f}@b#{fur)FQ zOux6^41i(A&Vy{lquKxYA6W6CKPSINcRP;xQ6nMm)WX`q&Ga_H$%ee`&%RWK#tq}j z#k^B)!n>TjuIBg5vKwWHZu^cVCX{+)(QV?$qRq%W_?5!XIC|F(B*h1_`2Y)LXi?BO zx=2xo5C6z}2N%jqL!OP;E(QWF!Goj~5y@OySGV8&daWexFtR1zn!Rd1t>;fD;m0CH zGnra1&UZ$$dGFjVezg2(iAvHkWWo(}p<5f{gDB+t9-!;|?tFBIBKstldTh9BG+L2^ zOXPwRjc}M=&<`LJahK3YD*a_$gD6#==mx zUfR7VC74_q#?stl>ln3T_kF~qngio8Bm8p)JW_I-&E3_$Ihy_Y8b%WhE~EC<#awZ- z3a?iAgX-(1$tA1ZGDOJ34RFHxQm6#360@Wb7B?-3pahku1UYN(J<4QDu)Pl2AekyJ zW-(B`q`WGa2{#>P$-x0dWr?xO@hgU>!CWMR{nH+@?lQDE%fmysP%I6@FmD+0c!w=b ze(<))h;cHX+KupM$fMl*6wUrNiV%v!jl2vWWkPzDLaC1e8_nLOJVCoP$fuR-72fb7 zCm6?7iNVM%+2nmWxItoID-GX1MIzvEsdErFuiD>SO@uT~e9}=;*d&sDL;#Ue24J&L zNdWnaJ6&VU!l3!ZwXgW_qGC4e!6vKCxMOC(8y-@#B;Wje-f)ZW{$bj@y-{=|^E>5Q zD#1Gi>VSk~DT`*8t3cr)$1bY>jqu88>^IIF#|5r7$6b%mt$Quh z5PyHq@yYvP^M%W2`-+}wALGM@RBk;jA0NqcLF&*WpXzWi0jvUy_jbO+TP4fJcR^k& zr!f?@&)5=b@pE?-K%zIw0JsHqwhD11B}b+rX+iWls`BwzijsF!paDRW z>X4;y<#@*0?qfZq(ewl&p z^NTxVoEsVE?o+p?lG4F`#^QM7#xsLZmS#SNA!;=Hcolo&?#v%a+hJ(~7K(!rE}Jd; ze0jC6KD2SVY`i>Jam^~(i5KHw#UUQ7xI)^e_Zgk%8J!thz%dYYNu!?r6wh9Ph0idMK#pX2VaCM~XiP}TG#zX>W+ z#byrl9rM%LXLg>GwZ;?0*MCu9N-ClPzZf~84ei8Eg)+o!Q44;K8A07!J3NCe80VlK zI^2#&Fdlpc1&vfL#b^-4as{fgGYHGl9Cz0>UZ_5b4x#N2b*fg4T&H=HF+vSXafQaP zUo0t{eysE0WHMqXU0sV-fbajWCt=L0swRSzTV2hVR!xV}HEe<>~fDJ#2?= zaQnhZ-fn#9C^V|9E8i17GVl|IH5)YCZ-EwVFP#2`7R}{x{PNjYaeo}e*XVpUT66@L zlAj(}f;iED{qX=BG$XX~vLj_9^Lea%{?!O^13bHQj!)U7$hO6WOD8b7DEp&qf@JgZ z*9z6Vl%s(}^Wyl-G}&%~4y4AEV=07h{3|B+!MRLIJ0A0?#@H7=thaDHw1R>wyNscE z5@+{6z}At)n{j6`Qk{?XWA{P0NOpFga_la|8UR{Jm!tDV@{f&T(A{9-H_Kz^V=%F!1Z1ZmSU~e2-#}wx<0Yh3QBvt)vA|v-D=&SkY%ftJ_oH9 zZn-a?##U=8M%k8D%Z*mcFn7nTvwnd*KHd$Ezz};k_(&W-#m2+Z|Hk3|vi-A8a!z7q z~dOR*Y)#;Px}SD-fWGwIDF1mhmTly+oymSZ)jJVa+lup zDeLZ{OIhesX}kj)v#=U~$PAjh^;OtED`)Y;-0Ni+a2U`Q9CLt6{y<>Z$X936pQ~`) z>TYdoOQxudx7YHH{Rqq3>BF{Q<^efV$76FX&sio`=saOVO=x8vDPi zRv)i`;=?%eC#vWzhiVemCb{0YTozX>N*?S_9yT|2O27ErSV(((ZahSxn>XP9PFgTG zzJJz-DtE!idGvoiMt|%?kV(JwP*B494Crr`$u=F53p0sR`|A}&9(45bvjHAykCQ(~ z+V#{vS0VRCPN9ZyN-=Du(ygZLF4>#^ALwJgMRrYnjE4h&?uFsjsl4ImA&(KJqKX}&H+&5Dp^$n= zD5j`73q{pY>Y=1WmYMsPA(8Z}KcYl>mL^C97odf1#k3?rVfBA4j|Z*taLWPWvnU=Z zA6Ps_#EJ*y5zRic(7~lCJuQg7_KZix-Ux3PLwh@%GIKZ%1k;y+voY z78pl8PEdxC&UWaKkR!GuOd36PY~Y89JN0LA2j*Rg3V>1aNbmHs!AvZG*|26G&3?km z$f$rrmIUyCyS z5CJFBU?gM5MbWd+#cHHvg*1K07k-Y1z__^@BV3Xoy8)J!fk!3d(f8z*kayAfN3%y8 zY(eMWn$zy0T2*G)WoLQ#DwP*g8TaMP3w)**W!MO9cLp$1fNrmF z5PQxj`oZZ;8=*72-C0*b;E}J*MRc;w(Dc2Zv@x-Bvtb#gtU{e)eL`x~S5Y%3HB`7& zPnB0JIp~8u5jn^-)!2mRA(0W<|nH11BLlsqDv+ z-WfF3N49d2v;-r)X!go7%8V0MZ})#gRoF&D1I*5^lFu-)V5m7 z^PAo<W5k2pp}yn!Z;cQ?leyIK|&~Wvmm9&jQoE_L})1tYsq7gGP*sO z*JEUEg#VL6a$k7o9sqA>j2x0f8_21aDram$J28w4J=BT}TUXDU5b%`&04dlpe zjziQXQk&}7H3`$)9osjR3+^fVoZO>MD`5zRh^95(5Zv)hLeZ)xGy&5m8iNIYk> zt2nEG4gs^^ey-5V$?R9)>+Qx}`%)QsCu~$iVu_Ag$iB#f>G61DGue)QD>;91cmiG7<{QJ_b$KZPva&P2@DcJP+ z$l<}HeKzFg5GSt9mxAkV980-?u8l*E;q$%tLQqHQyePgeb9rP9zg3|0@bjX(dWYiqeq<4rS0_UwcR7T|r4pdUmx_SSDYRND%7cNvpzG&5 zkPeh(U-&)%SB$tsgtdu0nPxB2+yl}g-{L@oBO_4x_3PwPM=DXV=qlL z$0{2?h{r)HzKodoJ-jgTQ>{I%rZ8*J+PL-Qy3D`MH}c|}Jbgty+=u!ZNaks*F+2fk z%_D~kP9KsbtDSyExY$(V)=7I@A+kc+wL)Z(Lhe&*#d!m#FGJvjNB6H-|6)eEA8)-xfDC_iF%Y^}V=M3P z`XTDGgK3e+=@dR;gfX3Y3Yx<->}zD#o{#%p5KOwarWw;QcmD^j)WOBBKysR!AD@@? zIpuE@q8lq^PV561WVf9!zc3bpI*sdI9CBLvyaf3!)bgE;%%gmFOD@dWvy$&c^s~q} zS=!_B{hMS^%l8k8+n9X6aOuA&--TMfQ?FI>rB>jA@=cKlkndI~_lC*0o=z8*@8=OL zLB5!8C&)J`qUHM&t9-xv;qr~GpX&01IWaHVk>%%V%a+k4=1%Wo?E8E=vVX!b^3%D~ z`$4*r1GM@mA&V>}^cU)fB2wGXvGuyh`B&@l=fU^Xd)q^(^a6dC`aD>kNqi zLcW?aOfYGYPthrS0vpa&eUqlAa5}0KCRu%OPk1!@5rl^0Z0UVvKEd?2jDzpVT)qea zrz&*b0v;K7mnyv!9H_YDz5fDc$UcGrx>ID<=PPbQ_CL|@pXC?Z)Fr<+UZePZ-*#ew zL;$}(9M7~9+KC6~B>7!N8YI69V2bm51fjw2H0gb19)aXnwi@L35``{zP2b0R%ezzl z4t`I61DGLul%QKRB{69+<3!u6f<%5h6uRX1+wa8qT}I51{RV>J{9Z%Hf0f@)K&$qF{lGej0Dhm#X4(n- z{(?@D->-a~CYI+L${ZS*nr;$MMg?CD6B zWF>wYhYBkh0*k_DFGB3$UTL=uD14Y;R(GQ_T3DHteb4xDd;9v_Hl^1ox8`J-|CPoQbbIOFJ1IlhD%2m85;dbXx4>0W>Zv*k8)1?X%} zW7@2TBU8=cNNP0ur7I zrS8krv>fDJM77EbbW6uI4cIRhIXao~V!)|(*zpU#>BIV=W0WDLiKZIN_Nt*i&`UJ_V?2cVus)E;h!%JN_ z@7v!?f&0%gh@kI@^0GJLQEpclIjUmYGd_3`UrIz_mf_`p0 zHw%JO#1DnWo4)Rpf5>>S?UZ^|BsEIiN42y&B@LTNz4J2;wWs>95nC46lNw;;@&a z6{%B{m+)o0Le}!Zf>q6xIDgBZcN(e{e2)vN0?v9oh4~$JY_S_0)H@-abbbz zz?`iQn}Q>+z5#-cE>=4-F>(rRb-KH*mfuKj&%p=JoZikJxjm~-ZmjE`hdDl05|Q!P zS~E{Bo^UA5q8!5JB(vlx)pENtp!BWz+)V}Q-DHURlKkq!?k)sB-&G#-{1Lws!u$%` zUx4aFAM?ai-Qf-QNgad_w6{T-fm8T&#MQda@{4goyzDu;Orz;nG+jp1?`RrN(_d+t zP16~gu7;`n1(?b&qFXXd_LocLQS4X~yIsZT^ z6Y$o$9C@I1%~(g?(*R4-;MSDy_&C;TVcP7(Nqoo07v#dLIOT7S)qReh+ z__=ujd?B~hZ{z)6VzmLmSqnOy+p`>bF9HW_9R(;(apY0-O-RMLxxkV43>?WTQ#(Qg z%@Ew_^tenDfsoybqqQ-PPUsNi=pMUlGIK{dxr~M)$Y@juq1QiI>lxVsH zlsBh#Y|9nFEa+trMxtdJXjy><<<%)3Af?g^raJCTN0PWlC55QmDK>|T>T(aG-O0WI z=6vkJxQ#P5n6d3BdrvLQRBc7pkC$TxsYAQu9dyt}7@g)*(F*h@Per;$kPXFB7UNfo zU^Q%%8zC5~l`Q)U@O^Nsb#`Y?K%0-P!qk0^GD1t>WT2z^93$wqzs;Ex7X1+wIwow% z>$VRd-I$1BKRC=zx~%@khuODV?It~*JgE&lh^=`&yu+`LqXr`x>y6P~qq}mb^Om$| zM-eJ^knB9u-aIv}UpW;hS z@`OdGIBgIr$Pza~5%>@d*DazYMm&@pKbTu(pt|6?l4BR9!w3SBJ3%iB6I%JXq--ivK^;iM22@I!Nr&@BMtxmE!) zM+b*U2QvaXM&noT z0G_<8)-sMG#hy^IC&UuU5L7EoqIM>e0 zBT!QR=2&+H%$Fo9^IG-+Q{Vc6^jMF^{|R=`>^We-0RczeQ!qcJy1W5v+3Op$oA^azKIPD*2SW)y**`<%H3B zdzvli3jc~TaNO;IrOo7?@bQrE3B-iX6s>yE*}&`4jJoP3Z>uF?$xj|FeWni!fQWM$ zBnW9?-iJv=4@Ac09`5$R4qZ$#upPjt^Sosor+F&!MS8{#`n|BnefkMoaP$Eu z+jLCuUrd!@ktdRP5-5q2Qy*l8*{_{^lp!JmNeCvpd59X*zUXD5qWvn`Wd9nS zdUq~FKJ_;F+k#&+u1DFQMVcM>`59<~D(MR}AH)xoN0INsO`?R7BMx*hIDXR{d3Gc` zyE88u1;iv!9O+OY=OPfgqE(UYuty%k56H>@rl0oFjJGoz8#Zs}&-E&dkFFv7AZB~y zH0CprtKoxYR1*Fu%tFTZv@*89L9Jg!rki-sCSkr4`74T5BMVZz`>Dvq zkZt5nqyV8CorNMeg)4$h$x<#@mcZKOQ<1e8q3E-IxVz|u$XW|Ny4K#ve1(s~mLB0&6nB_|+r)T3)0c%tI6-kici)jT`Vg?2)g-H1By_&G}74 ziM7PZKrB*-X+Ug-_tQJjW(nYa0v0tUbhEUgPI00x*BB!|1YfY{4Q!F)1hYL+&~hH7 zD0A{oA>Qg3YaBf2Y&e6D%DvT*^cKE>`J{7TI`Y+8_6H|Z7*mld3fow$+ebS_MD9Xi z9JEKO0qC0vU$#FlsbXY=ANPd}oFj5W;ZWhkZc3#e^oFth1WO!m9dY*}+&Qn~hi8ps zoVeQt86IXsiiWq15smjwe~;~L4Xf#K2!)D`mj_QCDaAEONF=WQl5MqaCVjbSNXf(Y zqXKC`3+dxYk#~4IF>Y?dCavH^?{u_%w^0rFY-F1mF}|NM`0AisgGa#ld6#Nv*)XP% z<^5J{O#o$+{YE%tYCelw*hu5e{Jp`fd1%%gKg#sog@a&!naj$QcX*#|@JvjOjz{bp z;JN$BpR)nL_2pgm8(@#zG!j4WAZ8Sal({{FS&=uC1*+S~YcLM}>25t}M_jUQ;kex! zAjlm)$HNcgTmkc=H z9+SBVkWFg^l_g zi=FG46BqZKIe~sy&X@gi3Ebr$E+a6}jN zZSYl9T~P#T*i)fG5fqdb4QuR{9{41x9B*E~qo`F)8UQk$mXPaZU>3Oc2x^p@%v)g^fxu zxOj=*EE)CgUV2hbVT9zz+)=l>rK4v*G%%1@4+cf@2G_{Vsj%Z?elj}Tdy9=k6<8kR z4SDkFbmZ$q9v8wOQ||F8#Za=lY{+pBq=6mvO!R9&U}|oxTfvB~Xw>Ob7xzrI^ScAQ zL$VKIf^L@5GVhSgV?cue!4*QVwqLCq99zd*7o-iW*qVw6^wa37O|>+0h@w$7S^?I>hGA;xh*H9V_ev9b?@vy*Zhah zvq@VVSgvy4XA6#yBdol`kpa**Y>ec>U4^f6?YH~NQvib)e?UhP4WdPOF7guA@uRZ(@02|ECok-Ish*#r4(m;tG9OT(R*k1vh&F1&3WwvtWp3 zJEgVvz{nIqVYqlO>20(lEEZ^8-zZcibQ<_W;TrL15(M=OmK!q{?m)r(4#kG4S0U=t z$CzWtXt2N{Vt)h(sOb(mhwMAhDG}2R^Dg)twXcN>Q$1?0g2{%Mj@ob1ru`VK2xDUp zBFtRHkVjK&OgZiz&b(?H+(W(;m+6RocbOkI9TD(}%RnJNwFbWQ{TlN8hZ{|GG`5P7v*r zq!H{M9Ii9W9Ka@~3&#IVhz~(WzHj_d8B)godudehBh2Tu73!FQHHb!)bEWs`wr`~` zuUD|)P#91mR{=$x3WFTs^<1Qsn;O(|Q%vuJqGeEsuE<2@Piw59Mjb7Bx3;JXjh&c+@@tRu-?FZ`OIrOH0GYO2f~4 zcE0Iv2{w6lzSY#?_j+~?wFLdmW$AsT;ZwLbdfuH_K-&2?Pe-t^Ed6Boh0^qsJKtLD zZ|T6o``Oa;=VgcCco5^K((sd|q4Klg-q4M2hgQB@hG9sh(V6+94wMB?XK%{Xp@mUL zX~_QCPF&61lWG&eylZPxlSFXxwQU3w?fc$&H`;mDwe?~HV@Uo%434iKZcj&W8$x~y zAnI5N0L9cpL{_%W7{qYJWGVWOdYj8YVR8((B^(6~vg-?n7iY>U&*AdY=(miMe^1 zGdFcXYR6pM-GLCDy=h17KZ60~SDY@3xI*?H!4|xzv)6Uh{uLO}a^Qa62Vi0=SBL@) zb%L914I6a!nfJgCZSmpg)CaMG9-aEI{JmZNw#nc1@;4}dKP`WISlgyvBY)qPE>}tO zUW@wzX}(VWy5(=N{N)~{=+s5>*O0$UqGGnZt|H~k2_H_~*Y)}?KkltNyd<0j>6bV$J;9dww9}qsM{tzm5l?bw|Bx>k zgL@?MmzVM`}RL7JTz3^81_y*6;*TqnI_m{*_ zH0r?t6*_nP)fYx+lp^#@k2~Kw{B(3`V;1w^ocw)P{$7VXz#uvH#M6)V#~h`>=)SuV zDGpTGm6i_nHrh+I-TWRQ)KlF%;DK|sdofVJCtSt?!?WK7w*wXUiM$RfRf#sjckh4; zP2=*GAt(4=7+SO#=nA#I(Bj1r`7ja)z<{{#^9+K|-e$YTE<~&GC z(y<>n7P+rP(dy>cq9dP40+Mbtbb>Lwa9Lw#zYSe>cBjAbJuK^2#4e(WpiISbhIxfP z3hP!=AEJBs?$vaOPNnJ)Htp{mcnhkNcz)DB9OK_jo~q(@DW7{Ff0-YlDqfW>w_>Br ztZYPs*@)Y3BPFkZc6=fao$1n3F*f8*s};o0GpGW+u?E#UNA5~m$4kGkD+!B`+<{s& zmOWnLR@rINExtu*ytGg6#o1{( z&`^94yDs-g{-s2sQ`bX2F==CHjKjU*w8$2s={$+~?)U`MS>tdLV;c^LKpg`Kgv)Cp zGyZY`Tq9KO#*2|}8D2D2<;d=`LN)LXX5f>{5I3iLgQ<~ECMHIK+2%dDlb;*zaolq- zd=S0K`9!pV2uOG3Y8HyfxBeU_g5{Nx@4R-`El4DIO=Kp#`aDyu4W&r3V z(0gNn$T(`RVE{b%Ub5XFqHfQ1e7DgX@%fEnK}pEKiI z%>Z~9pPPt@@s-jW@pUu4E)}1LXyNQD0L=JWV)3~c01x9!BVuBF-`fs9#CI*@D^&4) z8*Yg2vjAp%m&W4jVgNjhuZ@U_@dfFP`2GytP9%aK@Vl94;q0{lD!!!1teE^N7yu9B zDUIhq!GkocTkTu?~YnL&R1Q{FA?@{b+^oxEd5}ZdWiv+q;6w z9@KEPhSz9#sfMF8JgvjOs$s8&Ki2SG4e!*jQbUvMPqnD@YBU_B;SV+`_j(Pl(D3zU z<$k}0RT}1K`1(fW|7i`c(eT-umHWLKuGVm(hR-(Xa2lp+xL@AhC&YI&{DOveYZ%b5 zLc=@_Pj681ozU=*hI=)*c{E29ILc=2({#e7WX!vOjZ_%(u z!wLdo}!yh7W5vNvD&h?fbReq&H2+w?+HkuIag7<27n{lZI6q zF454f;m0%_tKlm;o=431G~A=%77a~09vLRb6T02#)$kDwAJp(p4O=u^r=eTJd<~~) zn4#g@^(y~g)$oXhKhW?i8t&3Cpy5p#mTH)<;aCmN)v0)2*YIf#_iK2+hIeV$tf5=O zi5kAUUd8h#4WHHUfQI*LxI@F6HC(1)p@tb6zFn*0>(}sU4fkqzw}x9Ztk-afhF55q zqhXSUZ`7!GpVzQQ!*6PMyM_%K`ZT;oL-nCaA+ognFV!l(6B_`j_~4WHKVVGTc}VNk;=4c!{%X=s+yim}2iV;pT$Pa z^EIr|aF2#P8V+gb(%~x1@MeMPjv={fTun89} zc&IIiT&4KwO;T{^E(P~|Qo$XC3JT4CzoxV1YUOT>RM2!^Ut900ZEkL@s|_}_wg}Px zL3n{c?RHm-f2*t6-?Aad`Jq_ZvazLgYm3Xjto9y-2h168UsPFEMx2$)mM&XC-!7m+F!dARJs4Ox6xJx|z zil$rqn1VO82+x)cqP&(q zdwo;OY_UqL6h5(3c*HVMEy_h1-sPfFtbxq~A0KQbFjYy{YOw<5a!~uFwA$xc zQ&w5+Te-|rxwyJLSld!x8>p||Qrp~AuL~jQSk=+Iv9`rk?%&iJ*zVGzX$`nm)NX2P z_PgfH%9rHEepZPN(JVHKTG4{~QZD>rlV}wIu^l!Ke41gagKto}x4_dXsE42$$L81%r+SIZk z?%32KGt{Ns+gjV3Wb5F%LbX3#c?#~)@Lmn~YPe6s9u1o{Y}Da@P;h35g&kSqA^QlS1xB3r{$E3%^iN~2pf1>;Q`Nh*U)A=AcJZ+N(D;!$5 z;~G_--7n~I#q$dGA6KyLSq1kzqo7-l6I|Nf^Q3ZLq}_cQ_Vghy>A`=oLo-W=`U_IDND9B+KDGA<>KH}=Es0^~y<;z6$|YOc4*J!Lxp(_rrdbOG`) z@vt@}`YUKt55o0sO-37?Uj8EC(zjQY&cqV;ZgSyujj}LnhMIY>aBovPsALSQW z8pmV!v#|Hk1CNPExYvRw`Gwt0Bs?Y_)7`No9v}O&9@x#U)5K#K>qGYn;JLJ;$p_ts zk59z~@F>4~yu?-y>DUi@kG9i?`9`|mH4^c1fTu=QaX!fYH!OIJe?ROlKt5)C#4B7J zkB@i;?8EsbhLsQUw|7xIK9gV4R|t3Wp?d}F9~Pg?R^YjHKr4M|W%2kVAF#VV0$$Dq z@K`SIDNT&87xsMZL}1nrE5D>~HSk=3d`y0cKOAp|=X^ZMx!0`|fth@qfZYYi$Ba+e z;=c^BZy4TVKh3>{ea^3b@@w{uZoHmo^o~1#+Jo2f#M}0`W1^diBkoH&<5ZgNiDA;8 z_x+MSRy>-`*BS@iXs7(x23lVy?AA92cIz93-TEe2;1M16CSo(B^^twjb&wTav5@jJ zeHSN7KiYQ;d&S->_OBrxdAJepiwkVx#azX?>6ZcbEW8ER+C;(BL_aRlrh$0_e#S7r z(eSfA_gF3ehvu{$s-GPmW#7k?`w0z)`_^prCmWMED zg*MLhttbg*mT?zw%*PG$`_cd;#z~s_Sm|Io7kb!c?6dG=#WL=)iNVD!!T~V+bnUx zD#(-hSmPs~)~7i{8Ur{37L!iTe+gZGY7CBnuqPY)v7b-Uwaw9(TeZPfvrn(QW z%rD4<{9`N$m&E5R{7hFX-Ft`8ZH6N5kcDnDEd6T`R~y1%Ol_jaV3#Ef9ZmVuY)yB# z{OQk!W%Pt44r>Z5Gb|q%OG{kq4{0)8t?7_X>yw6jHtPZ1DYpt5E)XZ3iM!tthZ}f% zEb=Fg^&x(j1>XhpaG6-~N!>%1wD()g#Ixob?PmEMo^MvZto#q>%L&)s;T5z2VIx;RthJ$Hv4Pa{nc~x=Uh3ft$w{pcODFU`AoyRxTMO*fxsRxG%M)e z7R;@0qTe3uGN^giblW6#J42aj%%t!(wdCgK3+HGAL?$S=ZNfR$CVn1^YkNK85-!XM zF!-BAg;{Oxm2^V?C&wSN^7HE&HlSaEoo&*rlFF*w)g>O{M25N1aVXJ^s#9?mZf-yw zb|u{krzOO2s$yAaAHV<* z2?A7I9Lqk)2d#>7lXTPlgC1LIl1QDKly4iA zB1RRYicxb@+iWJoSV@v)2L4sd*S~A-g|W#R=drSDjZem!Z9~}Xk`D0p2@TaM~PAOBZa*n zO>p1ne{m5SJB<7MBHbhQ{@0^^NUuR@g306PGroARnB< z`Pq>}BWhABCa+BqYlEYO{r3*hKV69T@VZJU3sf?=TitBwsUp2{M8CZ)r6#!|DSvWi zipYE$VIR*BJu`)P7HLjHk8_*(nfl_beb-mCgU8^ z;g;(e_Aa+-KfVgc_-v!?V)W~yhSK^|ds5oa1~4zGGErwRBlZZgK4j~>eCAJv!EliFslNs;{bfG*Os$5LK; zmMJ>E2Kn+i1W3Mr4M<#Dx?QB-Ibz7(k37gX`wR?Ht{38Vyp4-N+xQeQ{&BRsRimZb zcj3N`?udstaXSWqz!O1)2d*^^8x%EZ#VJF~=gHZrBKyU3^dqBW`|n%<-4xzF#!317 zK1JBkH%K?QzpB$pv?JFThcztdvD$4Z=%ax)89ym#gKAa^aU0%J=xRxOq8*6CaAe&C z9~T#7ip)w`-h0gQey^7GUW=>|rKq?;$@N7*KKHLy^8P6xs5I^U8mf~V1!Qh z2^X^*j$!*E%OS@WtXdN)GR4B6BkAh;(MhulGLkMs1(2;rUrLWzCk@av#i`=y({#M5 zA=@OzZ?<Mz=}T0c*=$Jc4+r*Kt2y^yKUS^ zF)nBqqi>Q>nR!*Cyvj~&Oy70(KYFmc(23%$`K;BpjcdFTOh3M zPaklf2f@T0(mcKkNFLH0Dh~+hPl#G8vHACVzKnZ zOi|K1OI){qwwV7=z9_hNj+oswS6s1ep7>Z}ftd39Rxu^mg0^iF#vILJitT2RrQ_&H z((-!{ap4YvN%iQHsvPJ;M~k!{r?jzsehBj6M1%b2gWvv;GEcf2j(ENcNWTf!rHh%t zSH;KcPl>4ouZVHBmxb)vkQU?Ffp{JS!^CxKx%mKxboiRjtN304q}={+zp|HnNrjQ^ z1jb|>3-0~05I@AbjqRu$H>64X&aWU(e5$U~-)gspnNXZ5<^-L#ne`6a#|uW=Cfd?$ zvA#*uyBhS)Mbk!lYc##hfTV33AVep20Wxp*05WeM0A#s%2yow5RoI-paKQ7+y^62o zuc>-*A7sw3qra~5efu|5x%l~isdCW=$hf}$ZRLLikoYeEGVSv3sQCPVd~OFMt!nJ* zz}R)PaQryEZNkUwBD-L{$hOspRGsf!&y#KZwpk*rakd!xKEnSH^SW{+JF9)dMd0mD zRp#=Mj;v4Xb-#hUa6XJa74JhbEFwP74Q;dqXTsDGVtnv&F&<;$@iv#>c#GwEh;gT- zh_tz~cIL}`tr{h24x=r=o3c!|;TF4fE|ZV^;#2dulEavZDPl~e9MR=-tdItO%BK%` z!{#gPJQ)3*kV(eV>5pC%;6Y@u*&gP)+A1}tD z@4(orEd{b-UFy%b31o@nCtuTz^?Pam<}rBS`SR1su4qaXSmzfj@LI>!R<{WS>EdJc z7o&}sh`KvgsF_?4rr{O5BjomIz`-8v86162Sr};?bh;k{zln92goP0U)je@NEH(> zj?98QQv+)J=tg*!y$XaSWZ&t0P2s-=$S3o4RnF!B634bXP3*4XJTyjkn{7jy*f88} zSE|^B`S>XCMjgh4$##*mKC6Fh+n5?BDlE%>j!vKHFm6Is+T)OJd|0;W_t5pKjZ3n) z{Q_=o*v+Q~{BWA%HXQ&z12>K{T!SjDPXqEfuI&uNXA>Yz9e_+@r*{7`AZeH|DvA8a zaW3hfke)2ki$_$XT6mhAlZrVw_Q_#fV8^&%-y1?ahu0kMS?#J_9Emajc~+ub#HA>s zF(=`ehwaZpNYgggF6M3%=r_^k6$_#AG81{35zEV=*(wk7$0$Db{7vzB0Fcklw{`x> zc0uKdtb@p-zW){CO}u@TY|Fs)d2aBt;MiE^p`)=u*HP64*>6h{S2rfh_ShDWU$;Bx zT+azH4sZSn)mB>Vwuz&}#9+FZh_XKs^W=$;!^FyQ{Szj!--kX#&z;jM)m)UiOFp;# zUGei}Kt3n6eWcWp#O1jk<%}Vk{~^Sk?+I}ofeF}QfrS|&~0c9g`g;$5d@IWi{= zIvO=E&iSm;&zQJ9-)OUm(|DhRKcOuNX{^{ZoPLKlko z`Q$-IJN!YZ!D&C}38AaCK2BP&M#RUg<2f(~(-QIMZ^qwsf%sYX9)kOY(qLFVTx;cH zP3Og7>5$Ge&^VkUW9|8mIPVcZ+Nxaa z$%5|MeDWJ$cA+g^6L)iNz&i6}tjSyOzSDv=G2m^p;8nHSglh`=VvAc>06HAFon0G` z?;*JDy&P?##jO`^L%{2`g#8oT+TfOF!E?4l&kna7i(5Y2df>Lh!dp)e;lSI4o`0XK zQ*!zDxjH3>f1j&Ua`^YTy2SS4-{6%<0o4d1!_0pI7vf;KY;UA zv6C|17M|&;^J5(VZj1dv?Hbuo$(b3*m(kgR?lHYNmu!>wu*!zLCvIT5uM(SZNYpG8V z6>bP)3i6DUDbotL#7;k%{K~9ZANN^do-ATkeiH8I@uG8lg*x&ibNoNz;VYWj{LMJ6 zgacBo9YLJOWZm&Wnm_-dKd=5U>clbN}H`F!-z4a*S$#Bs-k8};8)=`x_#fDji z?u6@k5|WWEs@>Goyxo#*Vza_k2wa`l7Gc`pYVws(Tp)!T;vwSIQ3 zA?^a!c~okdX_2FDE17*t;l^r!BtuGb0x8*4UmB?0DqATW{1i>r^b>14q*(`FmC><6 zB`#61psHH31d|iwNu6wemtkL;Oq8Wyvcfh6R9qVfG_ghB4tH=Q@oI!*;(}Vsi33RBQ$Uvvs?7u;Xcj#7yQ=%oU z9WBuLEiJ2DR_2>GSH;tjRMFZVyuPC;sE@#kX*S@OG&iBCsl<^{F*V88T3cVz5om7> z+#se&o42mD^zK`L?lHE(Th1HP3hUQj%0=(Jp z<~e%+8ApkyqRg|3ftxn9x#!GfeBq>p)vIxIz8Xi2Tk90w8qaF@e1<;g=&D;Asv)io zIudwRlogjUy)PiWVo&MvMT=CpdDTmbp?^ZSs>VR;*6NaKe<08rh($^wpG;oZRPSDx zTZg8pZcETzTDr2jd_Bf*L4VNgR;Y7V1~6Pm^ACO~9~pO?r3QtILDia3WbAldf)Rsm2lE4vhYkJJQ@Jb}4gHOH(k05K?X( zEfV8LO3HP$!4F|$g{OKcNJZ+acv3oO=EYnRy{cfYn=+iu{9Ks3z7_*d98E`CwuyX| zu7rphYXjB604iI%yHxS6e!aRZ^dv9b;14p}{>ccgFDv#e^sI^r8M*zx;K~c8|JkTW z3rov1`@)Wv_Qs}$50eVhHayJ1c(y}IrE}m$2%&s4X<;4qJ_lm7G_^tkA0!dhj@3r> zGD`G`l!a(~WQqEhBFM1j$cLtE=EcN>P%0xooMw~+&#I4}d2`&7#dD}bE#U2g>d=M7 zT&%@hTuM>Ti;G;$OIe7z{(l?kCm@qKCU+ky$(6cPhpeGKVvR7vzJ$DQY~6~9+4g_B zQj{b8>Ln=3qQt&%TRUnNW&BT7Oce!1TfGEDaS`J|U-vJ^Q>^0Yv`Pg14;zUOE?cHN zx>FVcneDCO<0A!TWkShVg$^Rdiuej^m4D-ge>o}?^9J0>;I`I#w`v2E?F(_=Z}Yzp zHyW`EM~iIya?--u=B>5c+pAkz(d}<>>*`VsQRu#mm85zZaOc<-E-XgBq+7&Tk6XQz z<)M7Vs*;L11^H^^F-26Pp{}ov-9KY?C>RNC@^9MIy2XzX?r@ws4E}|>iMmZ+J2=+RrfYuGGM}om*KU)^R2xnDE z<;rEakftm~Ei?wW=43;QzrMNwN(XU=Xh4aq!`QsKy0$KWPOf!*bwfu>9dsuR4b7Oe zqX!oab;jWHggUA)tsu!(pXrkU_oTz4>s%>|RhOL1n2!-;)(4nOjtK{d!)vn{( zE-68g688#-WDB}Y@nzA0feP8{61?SxzOzL&jQ)*HR~0~?^1f7q_<%py5oi&_LL|C3 zu137jH&wTRH}KewnL%~CzmX)9W&bumG*NH~R^!^7>L4%M8A+NES37dKHrNPGcrq8l zXlZEL&=K%gw_wJD@lkDH1NMN;BTbmyLeRIwy!+@}!|k(b}*H<1g%~0Htl!C<%cLkeNMh16|0QIu$k>CKdWx zfty^^5V$(n$fkw)SKCZ6RHHZ}scKl|Wi_%CE1ngqs$7hh(!!-q)shI5G14H53TZ;w zXB6;n6cdClc4C7dL#oacBuK2aMJ5}pUC$D=z(S$+Vd<7P!CZ=%QQ{Sy$Prl~L~UEs z%uVexw>GuRgzwBoTyr$DdCtr^Gj)9*=8vl1)M`Z-<{!XhrD>D@129?3@A*zD9!iT;DS3=c`AEZS02po&^&Xa>-mNkl^gdG%yyZW z3aV8K>>+@s8I~I+&Ak8$D}FD5vYYqpB*t}NT<%LsMtsQOcZNwese_k3oJ~Isyn&Bj`z=^p_{EmB-i}|~6TH?JKvoKia@Cc)mhE}&* zT*!ZzB1%T}zIePFQVt`O(nsk=%+#?z{{rco^B{J?AF5`FNA43j!ZXwRNb^j^<_FUK zp-O}I;X5`rvl6V+`#1RiL;~2rI0J7!?*{{1i?;}9MFY$T4`zv_&!n_wS zCky-eVYZ&Z=Gp8A@Md^y5;ynwq^}Mt7EHuwVUpE2rEw)K`J9VHOV@!Unt#z)bkP0=zI= z&l2Fd~^BnE`G9m7P znP*`)U=#fbx&!`gDazj>8-#?sC`XSCv-ON9&y-GE27I^^R^c6jnefMW({P@W=R(g^ zARd^l=QOS7IeD(LsuFn(9Kw3M-+-C05%0q=w*jWDQ1!@q_R@M*vk(5Wt57C^Pw2tx zG;9*y1+xog!j&sg24J?H%j9{@SMd7aPB@6S24=$0D##pW!eXqE(VgcnSFV8!Vdh!M z8TF6@%=v&n!dt<6{{Xja1`javoZ+K*hv*LY=Ky4aHiYK|Kh}Hw%yWHT z-HNne-V12orrHN5;A(B|2mIq5;BhJ90lY2*zF>9({tE9ln0an)@NVQCaR9%F^JzUW zThFCg&#dw6+5xr!kSJPIaF6+ZS6dXjkeo%Ku zLQC$_KxKmEsf-phcIS47!_BQ-9TO5RL}XDkq+nmnLEAkyId(2Nbs5#wRxYfVf53$X zx3Zem>Vor6)T-Ze&zW7V=u3SGFFgL9z4txmyywr(j^~``eILgwV|H*i@7&`vaC{HP z8#%1P{FZWEEB4Z6%EfJHJ3ax62YA;SUzpEQmg^`=Q!ZxF5MErkk8O(&!_)gY=J7fB z`T_s^kcO)U{q1AJpOMbD=1uM|9`t<_zJzp+O~cq*{rJ?*<+|mI*wpI}~_WcR28{?!CansZ5!;W{7=9R!giw^YF@%?LR~f{P=c*oJB-7oR~bc=~fSn?CW|plsHuV;p0e_BikRpbWkjR-W*;GyTJw%}Dzt z0V_`WZC(zQ$x{0Zvsub)eTPPwk9Y}vk1xz-DYG?+exY2v`4sIZ`B0e6Qf6!Vs4*7h z;vBRDZ@~;wd&Kci{dVSH{b`O}>a@c*&am$I3_SfA>yB3j>)9;7`}l4+`*VLzR$qh$vow+PvCBHN!3~|oXRMPP2+5bhxl6tpF%di8&?0|ue%McA36U} zF3!)fkMLpm^m*1@_2I4wey{M#No0}sWB3BgLh^;VB4vlo#fUlsl?N5z56*WaAT%K7UOVuPo6t`k?B(hft>xPNFd0+)!lZA=MGr2Hu9d0^bV{AkC9;5zP6Zd;+e& zk@+#77FbryzQV77bx8G}gomba-9h;fESv6^FM!d&i@TAo!CZJT@R#AjoBXzklgL$@ zONz`BXaKML&t9bVE2DE8*&=zyVKC1j?bGL>i!#&~KMK6KV+QL)xiWGuAho$0o@0b~b!G+-^9fv$i4Pg{WVAghBCSFe?y zG37YHQfcqqIy-mro9CgY1*n;SaX;%!Zv z7xBPyX??tPac4!y;$d5J6R(x=`t=`6V^)?G+1$aqf2+CW8O&|&Kg{i}x%ugsG9GK# z!sGC*!OxU*5N2R0u|~V2xszL%j3{&E%fhc$b}^|Qd9I0d##$IVhkvU=_3_o4I@>nJ zIzrZ#=KC7-5U?uLRNoShg{<-`^NE!I{Zb<3SG6fper4sDJW@Wj-T2|!o$*&cTSdJ0 z&GVe(XmTu>OI}VIr^G3B${fpC;DnuO$97Uq+8J~*&agA$j5=9o%*i<=snS$gYJ=PE zcDV_6yW8#dxV>)5O}m3`#vOK-d)1!p)p${FgV*kLc?oa3*X{Lqy%L ztT)F0lhkp!&O51OCOMqMB={Qyvag!H1%1)Jbl>GZtKaVL>Cg0+B&${9`sZKT0>1-{ CRgNJ5 literal 88064 zcmdqKdwf*I`M|xotORA9 zPRCPYtChCeQtPi`uk<1!^^yRx0ThB-jp7BYmxJ+wmvYh8^M0Q>XR{l;w7>WB{`KaQ z-I+6&=XvItXP$ZHnK`rR^V@u7KA*3gkK_1!D+%Q;{m6h@ z2p24m1g7if>H)2U2fzPnK*B@bKUzPZec44b1;@TEYDoBe7cc7PJ7(0<^YVE+eMk8Y zC_T{UdxMlYZfesjgdrV|(rhhtZVlM114Il012{>prf3-Xn`x}>^5@`04|QY7kY z9$w-r7X)o3zHJgVmH4U;kR0!CX^C$+(Z)kdd``Z`}ikhI-v2QmE@0qu#iF21n&LZ7d6f+RuUxdgVv3x83>=R~)duQf<|BjqXx zzbg$Y{6&4f_7j`jg1U{g)i;3rF<&SbJ@L{Rmt8#Dlgn1};l1zdFDW-`_N zP?wwVW#ulp@)`;XFMUhBa_4@b+$hQZ|MQ=bdABk+X?-qUB^~5bgu&{L;Gm9=f&={* zHI(>F|0R4InYEQ`p5l-4)xk;aRG~n;D)dn>q`wFHkEd90VBlc4bhR%1)KkG~|08}f zsspPf80gK(uA^7%A9cE8^8M;6Mz?v*bk3G%CVT-#edxB5%qmu}k!eyq3rt-a;%0J%rcQC|63 zd3px+mLJ<&{)R&NQC|7;^X0d17W%Eg4P@+iP9B86mVe9CJTrKXo&!fBhI-LZs}1X% z3d$e03x1df3|nuQ`y1vNBpDS>_(wZQRcmNzSms3}!7f^2$Uc(h#5 z_Wl`Ws;a^;Csc(}X1uC0#be~0a|L0IV6vL4CO;x9b;3t|D1f`|Pd|5@gfyCR|yh;MXo#i`$owtjwW%Mpf}8#2^JfRx$~0VMda|M`y@3O}PbC5~!2 zthmIF+!8A((Q^~ok(<6z$S$#)Ju0G$Pwm%paWwe&rp%81%_A~9N}9i!*<03fR-tgI zJ=TTQEU*y60vRW|8IvUA(Bh1+n=ytAcvvyREU!yC>r(aq4Bm3Q?nqi!C|_c~`2&wB z3=_jpS5`G>)-YaT(__<$#*B==N+ee^SG~))0FM9aGtgPX)*5DFkD<sm+D_*!-w*++~((25RohVTC)f`!R+ZQ-;!*j(P8ED_^6@eo^M2SI==t)hD zCl2zQ7Y9&R+8aJCV#wF<=D~h@;p|AagdAsO#+QwS_%ba!BAkkWDQJ} zg+L#%#}%WGehvQnH1Iz1Ft-o>{`P#Y2>9>(!0nIQ@&wQsdN^cAp#6xIW_`82Vefz0 z{x9PH?3v>B_qUzz%d;MggFO64?2G^4zW8hGGyY5b_j$mJ#>3n`_#^gwT}1Q$&i8fi z{~j{-<^O}E<@sN0Z`@PV{IBExQD4OW*_M6UZx-4Qei;t~@&x4hUq(U^|D*O9#RPm^ z|9ilT+rJO~`33xUzUSh<2RibDQTP9kNGQT@*c*1c449$y5n6nd)`z>&Mf&gz(T7L& zB_NGH{Isi>0p<4ROu-@m%0C%2W;rufdXF*eDLa83WT<ZYnD!+KCA_Qg!;YM~3jj77|WaWht)GLslnH-vJrYB8FK zGJS2buV3!-4SU8=PuoxblvulA-c(g>U~fsT4)}`LvbYy>Jwi!sUeRHxs+!=yC}#XX zgXF1924kX9vzpna#GVH@LnSJVvaMdlE2}UfnyVTaUEN{>Hc0FCna?1xxhkY>1uOv9 zQmDH&(tMC1#whG^!vDKNsGkkWu^1%n&B2r!ClgT$LqZ+wbVX~g52dWJumF=+Mp!Vu z&I$j~two9$pO8@t_P!QqbYg5mY+~$O_&PJKwx`w3lzPWdJ5qJAn%M`Q(oTh$v1zfX zu_;s680I}yOEeq*TD4R{^(Xi7(W+%Urr=^@@VI8$nB5gz_>?561#JYww!&AVV|%&L zvA108sXBydAsN811|4Q>0pn_;tUVSCmAB6b&hD<5VFc&3r|Oz2XYXfN=a=-9VWAoI zFV5;RI{pEa&&rMZS7%=pSC3ZFl@Rio(eZY9a7dJBRxqa3Q^=o@TUaGe@|RS9ryGNz zQ~Mc~o9cGB6~_{h4va&7WJYj4mWe%s3Jcp?Mi>s3X)Xk^S-(D|J%zY68sL=H8Y<5O zmnp$&X?QWW)NNG=5C!w-Vxi?F!GS0DJlt#DVy{l@wGywanu#r*QV1pzlxfF|nNyLj z-!uCF-3?em3Nkif6it~IgVq{+@pi{asf(+cQo&IhQX)?XIL(@wwyt$j zWjm8k7%F)-8C`78@=660({RdV;UNwqb3^s6)4(g>qJt$aP@DK$2lR1G1+*^n%2s7xOH ztWp1L%egTXjl|TJSbfJlzv_yqj?5na^~W1aSePx}<~ZuM1@}Z?lI6dTC!?&>*=~*c z_k;h&NDO8-gLv38N$U(JS+>bg9m(Av8ACTD%>h+1@(r4O6_rw$i?sW{iqVLk z12docgLk$sY!7Bn=2;k;&9Ru-QO8rhsV!A>F@rQJ<5TxDIl#Ei?%6+Izvbn$M``v_ zsxB|-TYNsrr=c1u>|_2S-IUm-Co^;E zHZ*UVoK2#8scQ$T6WbW82{KT`_W$%lLYa@g2dV@LJ!Q6N6oOD_0H)lPHN+AL8ImWv>3r z^Sg2)+P1+^qfpH6h{BA@+K@5Z7+Wc0J9^5lob(&3{8+W2qLoHTtmc&Ua6fn)bChn0 zR*=Ff@fFnKs%SnSb7RC8T(LeDjCTe@E$E%mP12_`1w%vzOtn#$sAxXU$U4n_L!I8! z-@tuE&YWmwe#FPLx=YGe17N3Uv&h2r%{K?DWmdE%$ov5h=ksLmfepdXL_Qm!WqqUJSM)%5fi8Pc4#Uo96r@KE=vFhxda z19nONFM5t%jC$+Px9YuhXq-M#H-@xNWe1eAXg|>Q(I%OoF~7raxaQZ?H%YSa@re0% zn(|NN*^HkitD*5kS(=?TK$~YNhL3&v`(lbopRqo5?Xf3H8X@%5L4`a+MIB=oDxspre&oF(D!v;3 zL0^LZopr_d54GQ)=h6uNG8ca{_#oCJf#uL~CLN%UG$NPC|+E7wX$dz0wm z*f%t3ah0yNzyEF#yV|~*U$5Wn@}iiqRZmJHlUV=ske-j2)s4)Y%24nY zb{xcjcTId*KrlJfVkxy8o3t8Aj0GLC>`JOV_G0bkStORWR-FUO_w0C_2ha05Y-_S( zTY0i$PdSyF(^8}`g{72HDRz5)J&?2pR2o}QnH{VLSPrZ%Dq3gg<-mC?2T~Y=_1&|! z80v9NpY;PFrgw25e%6n~A5;I@>|+e|VM)f$r?Z5Dr|=>5pFHQxS0O3z*|CNnn(qmXhKb%zjw>7jn&EXFH`=e$3#3ccJ^EhZh@3k2TGN2iVXzm(E*0)VhQt{zn+KnsroG%jTF(SqUg8wmM3CP))%CCmK7mLl-yfRfCv2TSjl4*l2LMRNCyv9 zN=E%aBRHC|gEypvIyI!F>PjfTyH9$pq}TT+-RYBV=<)+e4;H77kHj1t4T-lCCDY1d zCG#qhCDTI5lIv(xAAI9$z69C$+AqO0-bmGb0nM49!Tlg;rh{faXr_ba`hC%y`DJL% z{34ok-6SJvoykJDxH&5Eu+y$O&BVjW`cxJAX%DMu@m(~@0+|JhJNb(PLn}?(Zp(Fd zrw^6+V$~mD^MbV^nT2Ib@2E$9ljgVzHee=r`&-duZgi*Y&LN?dNrq}leZ(XZTv58K ztEyx%%hXXJvHOk8jn%&3UG0lwA(EV-G zf&)iK62sm|nKLHyX^fjQW*XKHx>)QP*2ri6At>jrOj&^kh{iB;8xmq1+XGnDF}#1! zbub0dKoN>8>|109)<0td-C!>28Z3I-Fvp;?#xx9jCbqJI@kae+WvD-MmG~fHD{FZ$ ztnp3u91_S}?swB8l(jF`X^(debklrNpS|BS$snnb!GgMim@$TaRtAX2HxRFU1U{o1 zT`rj8CzBSDH2h@wv_``^x5-dH5JH{s2HDcEek{cGru~48m$dpMUbjM$gNrbanPX-D z=rKrv$y=KT>pe#Z8MLaJq_}m)i*akzHbWiYboCs#s)97@jEEEd+ZHL`-W)+uw$2Or zlEFAeLPg4Kln;yg#!AC%iu9Bv75ikX^l3FAUOx5oU-f|QvhFX`hm~oQo0zOy3 z+<%DYI5uTEQ?>q!=C@({&QZ)v;+d5-H_pzudPKmcvrcf~2B=S4*H3YE>4!;Yb29UO z=y73VQhkp7zR9rW?=jRbA_H8;o~;?H4)_lf!|X0ClH*WTy;Ktk(Ok|{bX2{i~>e!TTP86V~NZ~eKIR`CUY1wSKfcE zr?0+Sv6{bzS{pZwNXm@rU0Ll*#(QMV{1dZVJLrsD!qI#8=HyU9ZP(uAy`$8^?aM`g zGwtKB^Na-}Iw%^sem~~oMtlJ(M&BeRB~)!}SB~C4g~3Lwv}J&?#Ww0ngSSQ`rQ?Ir zSmtf!fq2znhE?9Gjia*MuwClaXzj`9m2Rf0e!|K_kY*=*>(f3Tf~OA+{EB$nIBUSA z_-8lEUWQHR~3hww(TAq^Esx)pC6T54SZjCs2p$MW$=eFw)Vs zdZb4w%}DQHD0b?z4!tA2mh%p9Af?)+99OL=u3CH% z4=11!FU1xG-ki6TkP)L^E;-n-rA1~<`tY9~}2>G9(&NG{}WhRm4Swc``SQzKDh4e8DS zDV4YzfzcMScqor_%|NmaU9}9}#q=4Xgca*2esd;(ke&b@kvu27a-)FT38dh{TS>|J z=shPFEf*z-@@MX93ayJ)N<3mVRTIDy-DL({mNduI^1&_`W3*C(ctg^>ZZe-nJ~R0= z@tKn}SLp2Gnu`|e3M<;&K$cV#TS_1T$mSU1Ou7JpSq#^BhO^B1wEpgd%F*T|`+GSi z`;@x*wWONQ;F?sDq+g!BSrj;XzviiXlGYd&bw({R$52ggW=Wb04otjFC+(KOrEb1H z&ic@r__jcMkr69Q9Ma3g@Tw>j!u%Y{$hw|M)XXrEUt8z7+;Kulu%EJ6dURw z`#9Prd$ARuzz}TbQBSJu0m=?kuPLY_*nr+ z_$STL%lWh?%(?AJ^PH8##Fncttn=7}T2mX7X! zIhQW_KM6=KrS7jn`T5djIbPm7y)DSwTnZ2e8JbD6Hf_$VkPS!R^B4_@UQU>EkRGEt zws>X@5>z|RDt|)Hl%9oL#?p!23JqQGA&>myxZlmL^GMqE)J+Y15t=+R}R^PPUEZ=5>SX{arUUjK-Iq=-Nh5J1=a)KYeV2Gm^p1t5k7(!ADf3J{t$4bUD+4*jI7?e-gtT z`}-`^&7s=m{6Z3Jsq2}_`j1|>P{S{AH~i9SVklkTb=~n|D~?kWI`)iH%&zD2GV9&@ia4$|+iyNcXpPs4 z4_2b?@z?#6qPlQ+_4KEV6YhfhUo2#(=iNfo+vJ3QOQ9aP@Vu=}Ja6^I{C@>2u-UyU zFqtf5s$l(9kgmRKutq&>WDSOd>oX_Tec}ol_6+7~gIKiz79vSGfGi_UBDFeVO|5+* zBBN(W6ytncr(3~(4Xe<|jb^rn@3}G^SQl_6IxV7nu0d5^8gtqZ)EpC1`KfyBxTQGVhnPyjke-KeiXTd@J!bXI;HZ-7Qzr z>=-`3a^!WO>J-&m#tR1=2HE{#PpO#s%=l8EqECU!J_Vxq2IF;iNqeuGnzqvAF;znG zL*fd*65jcwiVZNzIyjoKd#4LDRuxk*!C72p&^~2GB%RgCOh<#tx{LU(Sr2eC*;u{9Z*DB6f9A}(wjS|I2ZUG$xJ)&1z@VabJky6ohK?#Jh4sr zRjfF++(TG+@R_+*izbUjR0vIWRwQPIB)Nrs>ln(fLAI>bSz>)v40}fBpnhUn{j8Qr zmIu~A(aN|=R>tH6^`WuM-jWtJ9?rrLZU96Kvn^VUff2(}H6m#=R6h-}Y)hX4jF%$M zTHoO^Q-(_u$HUOYnHwPwl1D`pWT?v{>Yq%sEA*_sphNq;$~y6!#nj8LqSX_7V?ly$ zW`71g>WTd)#M?6Sfr$DZ8&O7iPyd*Ds`kv9;9VWYMJ#9sG^N#}o5Z`7#)6gux#O)t ztR5Mi>bay^FYP(fTOV8H1ND4ALeKXN`T2f}%=hKEsi;%Gy7|N;$C8Zt=Vu>=diSM< zZa`r^T~L^BK@uuPG}qy?WE$w5A^!SUf$(j_UBc<9v9P&G_tf>wl4BagOgk_oN8!{u z^>^%er;F3P(mMx8=bS)|+8%uWQqPFUoWWtlvDJvI)C_3sVk!CWR53OXhSuQMz^*%LkKEWTx#$Vl`4D|8ASY~^D^uK?Bk!3;{$mO>Fo!g=o(#@ z$vsuuV3?u&*5Td;!NRImoq`nLXHQi{6Q}HoFJ|$`Krkm%RoC6&YN-{BUBmj5^qFYb z8n)|PcPNu#@qXK|t+Cq5yya(zu`{{A>`9Lnm_3_#n>b_9cmQmWR#kKUOP!R=+#K=+ z7d}ublwQwh*qm6ntNB>WyZ&}yx+I=iA-k04VW7z|hm+Z*oJf3e{N#$B(7>=4hi4}Dr@UJilD zI~KfO(K@19AK-tl8QU2OtwATsQoX5wHKm>ugJ7m@V9&)e|0-!Y)h&50<8-q<#w)4( zYnjSTn((myaboOBpIlCJ{z{zYpcVxBxu}cH_CdXMmbi7;v`^Qy;9N?J1#JyJ5+l7$ z{3qkph@BqiWDounARK5vJGH>M=>X82Q~oIe3&ugfM502#fS!}65wL$*RAXeJ#%cK) zEBwQVLgoNQupB~ZV=gM<+V+h_Tp@I3woRPN4I&0>57R{-W$f-TsRP8b&K7>yq2Nmj_;bJwcK@9a+ue(e{K;U zZnCUBKa%lzN;~?FWNT%iF%J@FB*VYA^dJ^6z3UCV#>&i*0sS`-Q|S*G(C>KzdW1){ z4CuwkPkulzB(4YaaAfftb~xX-kpCreWYkk5@TyxQb+R0qd65SZg38!K-Y|9^?<0^6 z>-z|#C_YYxebs2b?|he^#qxOLeKMMzeMa*fucY$tYR`}6oy78^`7Pr6j^=o8oegfC zv^uqF*n7#kt3%gaQM{k+GK`}e?1-WX>;rl$rrnA?N0VM=e|%nV_m3b$q#H#t%tc)) z(~F);2JEnZ@*-?-o=nzX9h$YYSk#vnKwMS;(ddDQIk{wJZ3WFozK>zN>?7ZouPBJb zkBKAS$eiKmX+_I7679-23oKbIWmY5KMEc5iXYiKm89gE*LyG15paON9pU@3tg1xP( z)knM8W<2MI(tdA(^R^phYTU=mByt>a-&Aa19?TB%=D|23PVF|JVbm&Hdj;j}3id7J zCed$tJ*Y?GUMKv?DzUrSz6PA9t{AnO>~}v9#-xA-O;6n6(zIA;Ao-btJ(9EC2a4B+o&Ax|A6bG?F}9K^A;HQ0 zBwkY?oNbims%rVFk*`{rl2{0BwpV;48=U7JpbtRVD<=t=be0UlpD|__ge459B}6%S zRe>DcPIPf>f5W<~0iy_KkYSj^+#qIdYF)8%B3eo+=@wmnsYW{Du68(h%k zl{g2PE~fsC(RqNd#dBu60DMo^74x<2~qu>Rk$)b3MZK z>vA|)@6vFd`?(~$bnwcT(DC}8c)ub6dL0N8e<-fY);DF zmb53fYhi9B?B<|C_;|8gI0X>iA2RKyG-;mcZavn}9}>&8H#X0bfQxSRM2~rb z?w?p=qIzM;M?^JCR``2}^6n0&QgDk18#k?!)!?pJ<_Vcd=<_wqByqKCY~B64Wd$0{ zE+rw4h~pAc>AYe@_tWVb5kj?;xV#z^$Dd{S$1^A9#bLQ{`Il?7I7r;cZ0tRsx~6qR zrIv@SA`fzo$Qm@n#q_}huSxoN>Kf!h16q4efhNvS2u)AsOValKv{IT5MF+YK4vkii z?*1L^iLzVRd|JLe9^=e>U18iC`SvLPcA_5RUUFMDfcm9hsGlsZ|JOqOKNaf#rcl4U zxjv8IljZ)!^%oZEFUZ$_*ncBYEz1y4(R@-KfG233=Xh+A2Aop>(BJ`}QV5TiH-bEl z{U~3FIww)kBCUIG!iNNy`;DF3V&FNlpi)Ozo4|l%=&VfJTYHQDBF%adouYd zQIEAR5bYzAr>zx@>yCDOyn_Ahv%Ll~tba>+CPV4n``oIf&bnCr!`c$MB`PzGyd>q; z=XC_B|3Fdwix5kxkByh8e?V)xTMpH4k8%uJbm{lerTO|E{!4Xz_F2Jyl8b)`_`jv` z55ZbPA#=_~d(R5HtKKFW>oD!e7C%~0znjA`krF37Yq>P4TYTZHL)YoqywkRS;`6O5 z|0d3PyhP}5z4B!F=y~LHDS;&BvlIRoz-@RC9~H zZu@M|aI&-b{dswxOus|_74N`5{YOEF_dCDMBNHaGS^oVyeVhhgv6?&}Df>DN71^x` zHMv~9!&?I{#WFo5%}2|(`x3i;4#qN{m9&InnGL0xjRSi6xsej>eSb;)N%}RG*;rat z9?O-8&n^KVx|m|9uICfhm=ZYHlCn%0;BE2jlg$_4j@sQ7&vv(*b@Uq5xn;+K?GyPM z+Mzm+8O9&mwF|aOq^whQblBx(9mfRuQ=J_|{`pxy%AD@36sJTin4tfrR55BeEi2*>5lvRjc|OS8Z$}Mf@TR|v zzjGgc<9-J4;f5-48bY`+^qYI`8(f@%i8-VdG25*_$54U)J*v!7aDuxYp#IUK!`y)bnzVQEh|GZJtB|!Dn5k@X6h7+RcV}zOywGxRg zZfRm9y}E*HF}l--x5qNu7_;F`JabN>$^Q5E=u_ExvnDoSRzQf)eh?QqLf<80J8ECh zpr5j`qh)0Fgl3&1r(2aSYK-)Sc%5m)u8twi{haXR-}6orLurpu-!Y44Lq8AgG%|ZD zX7gODpEG-py^e>WZsAT_zc-!J@P%{;Zzoun6{^#-qLDXuF)=r4&L?KNG_^FYOEj?E>A&Zp0w=|YxQ%Bx z*3Mq5KYr)Nd zS2i;%n{U(yfj8T}F_1H<9Y&dn#^69r4HI3Wky$P)1Y)pg7PufxKjif$;tV?{D!@BgUj&WQn&|aQ$NETi<0i`7XRf;XpH0m(cVPsjs_HtIbpyE7kc&i zE}qFwIIK~ocRJx=5>r-}kjc#P#zJ|#r%I3C1#4wo>(x#Ez8vg;pDfo+QjGvNE^0s| z{O{u`FwBMD5PXKp=zyHKb&Evf`U9;f9}6}WFd)RGmxo|O8)wS0?MSKv#jFoL7G4=8*gX{ti`(^8^?Gs-IP zJ?hj43=&Y6>_4+xB890Gl9I>fOOlIWO_5&Q(k5?ylowC&Ch`8H?4xp>&Qg@89>U7M zg%>H@%SjpxHF7JsXeL34A&nLR;pODN^kw zx}0v~?H%4ps2$5>#AOdqW#E#|<@~<^@HeFgNn)Lj{zq@8P}f@byOYM$R;i4*Cm05D zm?9??=!M#pl^}9vg%);TBmkY@Wq^wxlnxMS#0zAdSt@`dG8q}r>a%~(!NF|%oar(r zMmf!z*~8v!N{we%BCR&d6NbzfR@_ls^C>k3zk_ZVQH@OF3D*l;`CN@mfss{aO5EA zy1*R-bj2+i&&0NrIdZ;am?LKr#B1k5Dw_ekd8hk@K9Ek6U3=*;q5SU-M6iuY+1h~rhR1)N;D5B2 z|55lK-a=79=!7?;BRE_(g2dnH#Etg5fTSt+^4L>VXiWGXuae98SE)#;TXo{d3IY-7 z8WHJE5ou(Hsb*p)VYis;yzT3W^q!=84ME;ELQ3Tlc3z-s_7&)_7GqxSZIxk%7|+uU zD2Q>*SBdexT94$#c>KO%{1)5ft{4NBtsGB`$Mq58(YjEWf7u#l=Q)pB-TP@8R9e1w@=gkr&g`{76p zQ-tXF5XBF3<6|PsO@SkL_(*bUl)q9kIE;tZxkO+k>`@{{=1Av2Ew61P#%t$$L+eZ} zubi{e?dcWKXQk{{D2K7)?zj8~aK2nbH`}+p+$*9-QtZDJ(PIg&DOu6DvFiFB#Nw%&R_4+#C@GgX0?ByJ|q zmmE~l+S9zbmU@WqZun>K|A}ZlU3TQ-QkHkQs|(G_cgq!eYembG0Z-}EJ3J({Qsc-k z)~M4pv?4u3*GM;rCGgTsqDtZ zxonkJMAUCCTkSnzqe;bF_5?2>rasSQ<@n?=K*(iJ^^$Yh24WSYVV2kSCf0bdxcQI> zi@0SQYDY8o@?y2ju?U-X-8Rb8OlI6E-i$lhopD7~wJhetyxzhb-?Eg?G66Hk2~WSz zoelmBR;4Ks9gjN4?Yp~MrrkNH`NZLD2ch+2nGV1GF2`HYHq&JNIY*7nYpOG~rrP99 zxMO=Y)mW{m8eDb1@>g0@9mGIrmFd-ouII$w(w^bk?Xc1QfSm?ja_Heu`lIeU)9JJB99*2*EuBaBe9lD&4>rD2^^2@XOulJ@nfzg9-Q=^#KW_Q&j?;KT=FI1+t<1YBaibE z{5~%-_*f#rA%ct=SFcYg?k#9q&sm%2X9Tx$3B@%XGlJJ`;+~4@+IiT{r<;eZZ_Wt5 z_KHN-5_x(?u&0~Pb`q{$9kWK21EZZ+MUoLxJ_o`zn}C+%;hJXw*BQ&KuHZpHZtViV zlT_}KTwZD2O7WiOJg5?)a009=frqhFQ-KLLY!TD!xBgusZUEq{~u^~UB9lR?OUYx5Ej zmgPiASwgTMnWQukO_{@Q06W1){YrdaKLRPvYN7NO?%?_(Pbzz-z;FjwxCTHny!;OU zYTp5--t3*Q7L&d2gtbX5tS{PBuV_eOQ=PN4cfvAU@b;4DWx^U>$D>SG`JokF=2dbB z*Z0_Apwy3aV&HxWFpMX|Z&ZRiWw{z=rnFBqLZf^&IpIC`@~i^aQbn8Z+q7dSVXyK< zrC=fyTPc1LV%@TUVR67Ui53Y!kG(#2cbao$y~YasEeHWeHg-uz^@FL0_apyjNdFOPniR zpr9Wph;9@3vHY2ew!12BCTdD_P44Bng?+Kyz94A3P>xpDJWNN0j2-UP{V*Xf08KZ3MZjAZ^vL zoqP*GC;T$1(op`pSi!h#`W3NNUbpRut(5Z8XY_?8=0A7cyMC%jZ@?zF72%2dX*HU_ z7=Ys+bLHj+9}A0{aOR6$`TYIBLG?z2SPRV<0gn!a43(M{H4#|f{~)LHE$ib<54 zP48e5i8+Hw6v8B`#3YKOF^yuu>e)N_n!7C#tRCIP*SL1R&RNda1xxw5sEw~HS}~u* zbi)73{2h3uOongxt^0)OhRc}sZrVqr8K%F7Cnx-EH|q~>)(5)yVWbUPkN8FQeDc$J zidND%)}3cbAvqTU^BgXzEbv04=^%o%DIlMkuo-!G;514a}LcyTQ%D64ATni0u({M z;1h1aWxC+;6ck7odyp;$($C!jx9S2W#ukt+wwIowAsy#Il7cO6!D+f+BL%^~*O7qZ z<%8dr*w8iHCsBrjwWK8+uj6!;ObL@u5H`WM$vqreTEo?Q&4*=bPiWaMUN@(_xj*l1 zR^VjsdB;$^t81vk4fEa*{4mVH8L-2r`!$ll|4LBb(R^U0wnSH>^Ez5SO5n|^?}!EO zTo-fJ*|qZWqS}L`{|}Yb@TEfSI_1BIkR200Z~Knj>Dnh;*jTTRdpELRkil&4xJd~0 z-zI3yte6z+JjnOG3;9;rL6Bu?^TRM76Wb%FZ2=<0#K=HB`jmeHxXi5B9=Ysa`IdZ9 z^E%;CF9AI7N`u_xiwb<_BVeaFn|PE&5`*+1ZdyAym%YcUg7HfX%^jkaD9fIHaq}_B zWeHe`^~@QNcF%gEZ{7qk>&YKWo5HPvi?v-yim1SPoj3Px$(vYdnBhk9MKT&Z$v_bX z?$?E`1^^<_U|XkZk$CIh1V|wz!p93#6_~3FCN!qOhse52`TI#Qw43`C%j6u~w>Xb( zI2$$gL6{DKvAWKAx{m)PU{b2(75eRY;w;5vH3;8CLuadbAgFNz{r0?>jk-^U8;H84 z$YeKYbc2}$n)GLo=9qddUPl9ii@pzBQ5AB#x&Kf>tp;GgSOPew;lfh-v79ED**K{w z^D)6gpz-819x24&7F-g|Wd)s1RQ~=zO<4oF+zye((Buhl%L)IOCOd2Dr$EUqH=;G| zhtL(O+w>o&8y;oW6ou=)h=61oB-m%(CZyJCF&N3IS1C-A6jGwHj#S&qs|)Uf=mQkb0W3;ekAxYp0`Ooqc z9|6UbdXfZ~M$>x*a*oTCRySyKgC%aT)D2cjp!|yk8ajg~RwjJw{_~F{+05Rli|V&6 z=$7SPZZ0czr5tQrNkLbLgaCvn2;cE{-wC|LPBG=Bv#`O<1b7k^InR)`Chny0Z>a5r zt9~fRyO{ID0@t0}%>!g@a|7EAe43nWhyX#5L@Zdoc^0wRNmLu_%Kzt2b@e@@XahMj z@(NPsv9LF@Qk?4L0xXxUAdsfyG+2p|CunooH~996cvzaYeO3_cONc(=@EV)r5{saj z#B9Xg7tkv3G$@_$FYeF}?fPN0w3KxWP<1=S>0u^X`4n59%XnU%ONhb-xj(BLlUV9$ zz9wy(s=yyK#I80C{+IgUGmR_!)LzoirH8t@^k&WEA)rQBmXoJB>`q~N%D_d$#1RD> zTp*_LD*sg)FY{wA(f&=3U~VrFcQ;o`<`U1j^ihE4Zme(jS%V+fY>h4>Fy3KeRsQq$NKFBj%chFT|JWc;ml}pr0aimKCks1{zm}ED6g?>X^pA*O! zZ8@!kMA{)lv>u!Kl7(R9vQJ42*o>)sab)gWf(#8d2Mn!z+W0IrTNXk<#3f)cn{+No zn(`&N#8QvKcHmHa8C|-0x7Vc;NZl_iE4s(N^j({pAbZV zgv>|^!5+zg5n2GUO9J(uyg2#7!%((_BrUHK=o!=~y)b*rV14u;?@-2V3 z<8`d{bU3rkWtUCBa{UmpH6i&i9?d0Ag=!&L<18%FNpb5YWW~{Rs8_Qv8W-!~W!}6H z)o`2agZo-fHyDq?R;=eGWG{8y%EDCIw$u$MC{N2J$Yp;=AOv97^DQtSdLe?CUx0719S{h~7OmNOkN(A@>=8Uu`2<}~^u#w@FrsS#Q{x7;ZXn8(#0COD zol3y=0^ib&x$FpH9(Oz`<_-nGs$66}9pqnkn0ZO}?DMk^onqTpCdl)*`n zBQ%8A%EdyTW!WUzj{w7b=o6wMM6eR#_P9?Xlst*p8dhQ_(F!*jAu2Zvh%#;>9?|i< zlfT-15|YWQai3iEnXTmsRz)RI`Nw{wix}?HPu!;lo}hZNTe6V=Lzr)Ewpf0pOW}nO zqXJ9Ql7?xvaCt!Bp^ru3J-~*Mp4a6#i3%*$iSm08I`OU$EG600$%O)LpIY^e1gJjMkc^bvuk5Itdx-|V zwfS6tYjuz2&L2`+qhbHWEj}99YXtVMT-eop2L z_JM%+|3L!r4sd2+;c}#?jlAuWs0N^vEj@mfKPLdptX2U?$T1>z?O&Kz4JJ#f-=_;F z7R#1cuLfA6{HKx5fL8wDI&y#Bw}?FsLKbG6Bxn~elT37veJCy44SVE@e&r9i^nU)n zM=#g4-*Ofb#MSGP>ry3<->?X_BYHxA!g5sEgd_7MV>R3rvKrzt^m+;f|9A z0w(`&P(V|5mlT4wBX#v*&_?d5`{tJE+^|rttfeBExT(fWm%`k`x8Hx-la0#V;Sp&5 z6)hh2o}7cgl4|B7t`zR1n_B6qq)EL~*qw}gMJ zuO%o5*GWbGIXvv$dvFSXrY;X-nZQ6U-V^v+TV95`hXewC0iA@^P4WHn#SfFVo8SJS z#uYwX0=g%PY@eiS-6V&NM*fAkzSmQZLUCRw#r>X6cx0B+;se-hk9thHR|=co)_7ES zx(7?p{frjTEl&8S3pKjYlBdE|MMZ=!y2#xE)%8x zzaT+@PsfVCq`;rU@^z~<(f$Xew96klP3|`5&{Qp4T0(~Wv{d-^BFtMMMicja7xQe% z%dCxjaeuz}eWYa^SH_1~Q$b&Rn(SklsrL@%e1F?6G`vLpDfcKJ7U z??AV3-~4K$!#>?8>oP9;v_cA%ea7v)&EL|X0s#f0SFx<{_Z?^~SffkSzt*z7$4{vi zuB8sYBWlbF9DExkpKo)Vb^brT%81N+so9ghYO%q2De^`2)QYO$D^)eeJ{J3ctcowm z8bFo{A`wZee*&bYb>z7KE@|Pd+VJnCij|JoZ(yDP;uYxJ7y5d#QmUmUZeBH0n?5o{ zwn7>~n*^5sT%nyUZ>&ZV5Nt_&P{Yy&H|C<3j(4&*wiA8AziY1QIog|?vEqb-`H7h0 zvEEdz60P!9?3^TX8fx1u!2DY+#Z{s@r7npihpu(=1V+mIUzynPV-M|-X_Y<~tWNl; zjHo?>yz2LGxweBrCTaqisFD3)@9HO+uX%ct+T_jT38m+}qh%LYk^o>ZZX$tumKTVu z?0s37>$jibw4VrGTr1BBRdLGwT9;c&IRr2+CJXiLm0>M_+PlGBIMT6F_y%2gHiZrD zOEd?|bK#~=JvZ4KXOL^%<<3*w0BBKKkk)ueYk(8eaQ@~Z<#>s2a z7Xbph%AYR@CN`RZe{FRM;ov1_K{;x)aR-e?u?7B2gnNXAoWS#x;j(O@Or} z!T1`}|7V`GKeA@l4?gAy*!S)b%aq?2m32vpyYSF`wwMR{rOgeNxWO_vSndWZ-JqSo zwC?2Fb&C^$b8^np%r4cbQVnL{U>F8^Gb>(k*4%va-&?km2k8YEK$<0XuAR6vaRQpwfr2JB9QNn_$>_<)8bK&sXQajBjqaa5_v&RtJWq*#~%8An+t;dD_Mk zay^!=(Smc=Qpr|<{cP#0$*lqsbEpaS5G;0sHaA$}21_MS{^#D-0Oz<*yQmNNR-WX7 z*8H`N-Ggb3l`M%{r05HJy!-hi!wFa3B(+4vu@NDFP?{m6T%%W=_T*)9WvJq>|CbBIf1YIUvob(<3sUi42n4;5 zsPKU3bp)-AQUzm-Z&&07ze!!#Uf4h6J_xsmYA6K6J2k{b!m-R6?vmvqotHr^)lqui zEug+a7tG!)l(h}<@Or1R#c-8PZ`1Wq(y=%U2|$D(Wm+m%_V;46$`9#i>04~m#@F+2^UAeRp4bPbHcmkGJ0<4X2apmE%J(y z^m?Owq46=aSvEInO5#-)d*MtkyfP%1^e$$E7`*EyhhQ54eq6q#q>OL?W^Epz;Rksd z?&=}|GA3#=z61MQl(%ajmufyAuq2Gts_|45e*!H;^bAmASWgL-39P12xFs#g*h!#<+D+ZvI6!!VEo z66Y*LDB>KA=cX^c1DkV)&virMT{~ejPyh-fpE8%}$-*+YarhfKT!rl78T?wbo1+(0md2o8@C&`Y_ zc^!aTt)8>*`5&F`$j*IYae8QujeF>R!OlnqbT5Nc`#R{xm=yC=mPf~)E!}lbm6)~O zYmZ!THaD8T?wLCS@@8@2eGGlffo}aFo#%w(H~4&iDzP8aFMoy)e4S3_$9CP7NfRXy zHYa>6eQ%h#ugL$frg=># z4Xz<5{9s@H$NIhptG%VorOUD6&Ebn(2xk>RxYvd7{Fg#71hKm~I>v=?hzmjEg-oI0 z0vE!gUkYKr0)&73QPXh03n3wa*U%Vk*X&wJe?#VRl4lLRTn8i1A?UMSJO{T47sgTf zV0xoGUwN)H*BmC+{Z^xH_=IgtfhWV#!-=2(;sg}`JJ4b)OOZ?`Tmo@2&i&Pr$KI<1 zb;EBYXs)K_b`~Mjyj+r5|LK|~KB<}0xP~|AobV&E{PEh-LMh#rN8KX#N}d}0KT?t% z@H2uTd6i>hFf^`O%&QuH7qp7^jeo_j)5#A*_WqhjyUfA79+VOz@mcr!C(O-x&A>5- zzpbGLDnKT$g9hcDx{qDveWGdd57!9WC(4b>AE&iPnNUNCt1aa-KXZWtsT`7`B4F>fF8tjCd zuZB=4+$F^9{5W{t@Us^RmUZ^R^CjFP(fLH(mP5I;G*}JI19>A$`k(SJK$AKte5Kod zX`VD$S}Q;L>b5^PZ~|2ZYFkF<9A~*L4EU+0AvU-iaN7k38l-s-j}VleGa$PS0ua^H zSVYCG(Ri`lmQq-9D7RFCdD4uaK!_yaN0c<)yPPdx{T~BX)|5>e zu13HRXp*J9HXG$GIuasIc*iVF$44)TxN|aBlhN*ke+eLbY5R~zd#m|SufMLRh3>6u z1%!@%rc?a`s3!2|KuWES%fV8^wV;<1c0%zKA44yfSjsou&WoNLKV8b8{Qy$<8C^s3sGMu(P&7el%-c@*UoKin)``i?p2LIL*${}#qvbg`!dtHIU(rk4f=|oo zqFI#-KII@XH|7`=UGV8CDBj@0qlpU?UTgjxHHzNj|U_ z6Jw?QTQ36zd;JeL&_t}4k7>lJviX8nv|QqEeEz|wCztr!0lLV|+((~)S&5B&p5?&&))b6TTb@R`~>gRDaD0UrPcpuKpQ3k6F%8{@aC7mi+iYVzXiSbsoaY0n^gApQv{2GdjH7wv(yiaf+#Ecp zxk^&GPeq8(vB?sv8~Ml!$oLcXY5YEU&r0g5^jHz&PZL7rJyp^T?qTk^9Ol-Cv+1xM z7aZYsoOB+4B8wHWf#UuCHefz7Oq4P=rs8}t`LUwxGr@(e&}Xdtjr+Ghm-^uYw*rS^ zTH!CJXJr9(wqeQ5loVXx_(I6g(fq{Nu}1yo>!Cg+h3s3P&9IDPjm-9Kz~0ltZ;oxY zuas1*DwC(@4SrpT8$`JgiCBo3eHoLZQNrc;db5C;)w0z44vgD(sh7Vz<8Yls`6=FF_$fwGF-IR_47wcc%gnqyXnMULVway_um3cd`8|}RRR2`{=HRWr zlulO3P;%&Aqr)jP>bJC9u77u$Jr4IM_3ZPLL$@RgjbnE;$xp2eT`lLVXN&A?WtMfq zerZN>=z6(UvSV+Vo`*ZQKbI@SxrtVOOO5+YgQGk7x$k6f^cMS-V+Aum`6oj83r$8D zHsv0}+~3HIu0TdyMTcw%fkN3;`%pGDr4A@iCS*bw^c55LT}|A~8UHupqMA6wmo?6Q z`X-B#eTnjZKhaR!KlGw3!UpqGdZE-%ek^fMnLYno|3R8${2m5Tg}?u5{5H=uR?gej zPyXdXPCa%nz+Q2}*-L?Ol>@Z;X-qg<<=Uq<;0VeZ7wCuNt$v*OWI`Y63lzrrNDE5vMo_KMwNR<9Z zd9`Rkue|=ul~*-jr7NxFUn{LHU45kW^j>K#Etb}{CLwLrMgKR_J~~E7bEUOh^+Pc+ zfQm`deN^CYN`6Z~y_ix=4xe)AbrRiJzh(BpNvoVW5bakq^y)XyzB#2|B+<2WEpwqyFRgb@8)mS5FJQFQruY24b3@8&H8FoU;cR zLtligKfz48Sc>U6#i-vH%&gE-?LSviwdR7c=v->8o;@X*cWs`YR2?$&c}#9%jmSBG z3#AEVThdzDl=54e2)Jlk*v!u>baLqn9m;Q5qGneh8_@XnQxb%`34oYVbmMB~67WH9 zYG^yZn_6nmJVP21_#1_^ru9E4F`aD8Vzg)LJ@Lfrj_ID(iC}hsj?eCp$^w=%5|0>=7#(euXB*7!?_hy=8Sx1_g zrl|eD*6sIi-l*H(P9PA~v$P*dHNv4?cEfOKa^OL--Im`iF`-7coXK9VL;9Em`Su6? zrfW(2rG9V!7q?#k+$qJr(Ep1hwW$BS_UHS5$UgmlrEdT9eEaoe^R@l|W4Heg1q}pN zXabA-pUK?o|1x`~ESprIXM+TYo&QCN_2I&1N$K1FfnVwJ(*8S$eP#P^>aAKV|5r=u zf8YKqbo;0M_w8Tgw%=K=+rN{bul)CE|5#~w;8_Z}?eCCSpY}g2Df_hl0$pC(pCtB` z?LW1*YH|C&C8_^?`!Cn+pZedo|2nt*4|nSJFC+Mx_K%T<2NZ?e_TM0}KJC9oQu6KR zem#AH3k}Tgq$Y7M?`6zice1R1qZKkmTIESmOxoX(*@IavyYQSpFbwrM(*QpNU6Txs zs^v$Nf(w^O1r9Xudk2o(T&jPDa!7tF`4CnGu4guZXsQ=vfo%7L-CHW{*@yq`A4%v4X?|D*etr3q6Uh8_Dh?r$HH!kW8_-#+dk>~z9+2rTmo zDhWB}+)Bfot=9w7rxot&%FjRCcZmTgx4NAvaJbU1?*w$4?ybTEbd!EAOh8xaXKwH8Pl6M(Lvl3%1hLwEyy0UyP6z0S@FVRK}zADbq9m5;o@v4YE zG^IX;M4n~}0G7{N7)G*wS9j zSGsmN&LN$Wzr*dYf63w#KQce9)2VA%{%+*eF!x(G<|f~|G1t;RCd))PK zK|(j!7Yr3_oxGpvgzrAj=UXU0Wt$uAqAW6bxbe-LJF{Q};Un2eaj(BKkF>gEzbIQZ zCEV@lG;nu`)BFuXDamref1aXW`RXrDdp%ZQr8{N{z2&Ya_3zc!&^c+L%arO!rt6u~ zyLv7Ck>I#Bro6d~AJcKdLp11M_9s$O+!WYThWeXSTsQirlIY^u*E-=*P%DFIym(I| z(koFJg7^WYe;9sIiA7kjJK;ZEAWYHzNcM-!(WL5P7DQcQ;vUd;C{WRN9w@VxK*(Ln85REuYl^bc137r+kDUZN9;WrQnGNHY=%KL8W z@%k4p<@WTk&GzL0=RhW{0U6Dz;J21QLJP-*dbsJ-f7WCvpzkv~07b9dW_~ znpx{|fd1tU)!xPu@!HEZtMWr_uBMD}Na|YdgV~YfM=ABFOx4xMjk5YGj?4}GkY<_u zF0aRx=K1!ngEa%Lqp2@BFlmzXD*kW7(|UiZShK<4etBuQHM95U;d6#;Wr zR)yLg4_<1KRvSbC6*XQPrM9uP$EC3~UW)P3=J$Tj%xCwrOQ@c{&L7{``T5Cf^2|K* z%rnnCbH8rs#BI>miWv;W)ngp#H2FEV6+okx$pCnDb*>6=FezKcB4q)4&8lJ|HYp~e z`^CTRbiAy8X%NgtvC0*~G9>FBEFaT?CmQg>^RPgR7dPVfCaYN7Y}-D98n6I#eO6@A z9P!Su1oDslnQOBMjZ+bT6%*YHN=LP?1!iJYGR!!_v(kh(_85snE0mWX8hZjuDeY%$ zE5)+z(UsT>5fc#6a8?`aq_aIz`Y~RV1~c0u0#wMM0W=Pp0rd1tsC0&FW=|r1?x_zO z33H<7sc_VLvF-76bf5pRS@L|AJa0UDJzg0s9Ukac>@WM6ziN?jdM#s}hs0u)F=MQB zogrO_3&p`5DxE9)eEE{AzI1WCbfUZ%@x2uc7Fvuc=M%gbag%h<>~(j3DlbM{E!~}; z%8L;#interCH&TWEM%iqsW_L*zn{%TA?f}>4$CfhNM#W)8Ipv`G}1Rc|C=!H@)#b^%2ayhE9JqX9rZ0EM~%CK~S z-`+w`(y3ZCraGl85hKiyR9Dwn_KU@ZGoR_)f0bJ{Z=S9cCM*&?d*`SN?X_&Ls?@Wm z<~Gat(Qm}&L!zwD?t##<&6-idf$t|sbvsr&@H3Jnw$e{sp+m>sV`C9=Iq^%`OkN&F znyV`6Qt%65SaE+b;Ba-94ZGM5UFs|kC6sl$(VC@}Zp-7Zv_8bJMuUd;DbS+r0P0_8 z(Ht(@`~221sX^+FSdW50gIFau`%XI457>3!{j`e&!{x= znR3+_mxm7MBN!>mskqY9DXSYdX>R;Gl6COHtMEoJVqJjtk-vX8p1z;X?#~Y%E4ta2-?ciWh*{vTYBQLZIU*l3!}f` ztnXi}TBU2GtD7$TGRDAkw8ZV%w#O^piDdb_N?C`P-_(GiKmR3V94tjztx>YodK;=Y zw_2}AceK@dQo41k#l1fe%T{Y9+9@0>4^_-yt5rG^s))8)PPAHX<8<3v)pDsOB|6>> zj=~W8Pe@1X_$e|Tmi~{$`^)yvJjr<-Gn@5T8or&u(VGr4|Q&r z`TA|yYII?#nnNIa33fBkSe0~%&DUHj@VW~wsr@}&PI~zIa~@xRzNZ2yHh;1`)M*4@Y0(;rQO?zDebyBg>Oz{7FGigo<)1NetLQ$ zVq;^Rbq*}J%x4LX-N!9^5HM``*R$!*OE=x>Eo*a2ruEDHKso=BZ`y}h-i{xz1T*%@ znKpLou9i&9ZK64SM*o7oh5K=~t+z|nEw{V({#5NAehmv3?vMY>C3e%Ln3%mq^nN!l zV^QIz;{N``xVf=U`bFo)HPRiO8+)W%&y9-}@!{~kSzov~i-epY|92wtM{e{O@|!R9 z#O$v?{v*f%Y^*#XH&$Y&_76y}sQ5djJ1YLSq+5%>RS_Q!e})<|!cL=8P1_x^H%HgX z^_2HVTu)I`IAc9U9+7f~KIVG5d*OL*+v0rp!uM@kf5POd^_1hmVyvfN+5T_9afTU0 zEZDom#x$C7y{=Lgda1PAHOg36VFyqadIsl`%30J*3oAodSUJRnl}zr3EHlpAhMo(* zdNxVZP?zMm;N%M|dE^;gV8Mf3UDse3{-nb+ga6;w$6aQ9IOTxodH4RelBk8L<@o)N z3iQ<3YK_Sq!qgyz>TnHX$2B}KZXfkAoV9-G$&`U#0>KL0!8Rb zX&!%VLTs;>Egafd;6CKCLo)R2o3Lhr9I+f^)aaQa1076Mxn2}SFz(D&b*A~P>CR$J(iVY zkIIA(|3hwrHeK_)&D?gcKpBn?W^ zJ0&R(eaM1A9UsJTAa@A$Y~epPo@Oj3%JkcAxQ&`bR%1%`D_UGBROPb^!BLh=4=!7oe0s@kn(OjP2FAG60m?}le^9z^Aa#Dgc%Mvp4DYOSD z(^v&GtL6?#b|1-NRdcghHGWC8RZ_9goxuY4T!zw%_n#Fwlr>d9i~@JA))2;xQbU+Y zFSVo4&4HG}$LIV{RD_noZ7z8%QtoaS=CK%wxycmR=+q^b#Wwm zz*s*Ey^G@M!<_6e25?x4S!Jx96{AN}^Uy)AopJThwsi`ob(=R`1=~2-%nQ`*qw;%> z4Py*x%=p3wOMBIfuNU0=*}-A5(i=15JB}Gm7(@Hs2Ha>_(aX|~dEJYcwDp!_RtVqz z^2BEltVgX8MJ95OEapTGzfc{v*lrvK>w%<_Z5ywW5r7@pYG&xFF6-e`?reG*$hfJT z+?F{Hh2M}z&gLEl-;%X&M7xbOs1Vi+oBmd~YX{h-7FrdZw(#6umL(gJg0(%oz zA*J^@St2w3cs~IqjO-P=mIjTTgqTg214o>U*2zt*LRIi_>wzsx4d)u%aZ?*Hwi`b| ze(u9IC*S$Tsy+7aw#bNvFtW34JqizQjZg$ggZJp*$sh6va%+#;^e9 zf|TQnRQZuhWnA4&ntya)MN~30{FLl(7clnj^j8Hw_m)rO zxzjVzVB*~Aex5shMG`=VqfP-q_bMDc{bN5=Jv5LK{%A7Q+UE{oI`b@6%2IF+k6nA- zaF!S9l=!w3_e{)H|BfeM@DM48?8afemt=iT`cQ4_bYVxMJ86L3cAoqNt1c`bai7N+ zxam~J=(kAg_eY40`n^%6!k8^8{o}FCci{1Q-D?kaaCi1p^Ngiay(%V4FE^8Y>w`xiv>8&L4 zUrq0u3%($|zmfzAj1{e>J^-KlcmL`&&tX^x95L@8{zY zvrO-6chSc3{3LAA{B3}?fc!7%efA!L*!13`$Ypw;#sw5zp06imm-U+j@#(#S$P?4c z@dldb(YSnrO_L{c54dsN3gaFBS?+GO(a*3KAH&pyODMPL7&;fMWC(16=E^i3mTq&8 zSC0LoZ0;nX(<_&Uoa7WU`kTNwL&9+(wk11Xu*rEdX0e|RoXuliJnQ-5vB=8xeeRFn zaliOy$h2XC`_LgdzQi&P&L1!A*_d=u_abZ%U2H*DfX?PPrpv1xJr(0QsvMEaqxNq4j#nk?PAAIea~hr?Uh78obW^kK4y z9D4Z#6u8Ka18?Q>z#G>NAm8s6A55Ch953V<590(J#y^-)ZE|Bu_PTBBqZn3{hc0nq z5!JT!HaPU~56dg=$JXN-JwJEXqMXR@BXHUwkBFa#NmX!JS!k&Pw^Aw=zUUuY#+AG7 z%*|NWda-lFpvioHDsTRAEJfh=l`ysDEWuTn z*KWlTtbws*m|b$;6EZC>Ea30Khk9}7GyJo^pz<8LP@UMrNTX|`-MQ^t`J1(YZ2WAb zy{xlG9^dJevy0~nFek=N6BLD`7xR(jDAznp!?^|yuNXO{mSh5UzgnNn<7peD#mU0N z^@Y6Z1I}#-ej-+0<9mR=1HmMQtmAJ5ko(pbVrpwE!|ol=b(BMPA142}WlbcWPPJE@ zf*Zi4FAy_>w%^cpI&FWXZ6a;&(KeU1k7+v>wu+Zws~AsQB5c;%u?9)>?}?sCbQ+!S z(0K-JAJR6Twqe?EJM}2D1MFTnR60QGq0-L?@H2oGV8Li?#V9yj__e^&_h@mzf?u^B z>w;p`t+^-1Aemd9WTK6yG-NZv@j7Ce1XJs<*<> zK?L@z3cqA>QcTHV-mGStD87_WCP#9Lries3MX(vUGeUA8$CpSBQfaS+qV1ce#3cu~ zCMd{$)uNN71(K?QkdB;G30Z?YrcxH897%OU+<%%JB=O572coFtFi)ac6wMu5F6#%g z2m+&7WruC*prh+G>d-No(;80y(e1s@JNxB19-LprS%K|ni{AMzDk)kYEL>z9ox^3Y zzt3hPHH4WLZYg$T2gbWYZo4})v2qVcd@T0IK!(z`- zUJ&FW5o~olyNoFDPDTkyd6H-zH(li!H>H=^y3<7IvI;%NneOol8OCbrU%( zlkB=qT|jB&394aF3PxGstvDaKT_>rE{MF`84IAY~D2B^0ru}F9UNiOvyE9g!&BxtX zt`(-byDkFCoq~5l$e`OgOUi-nmPeU+WUJeH7UGSl7;a0Pn{t``$HuuIy~E^h$m7bD zt7rS=_Aj1l>viw{Ly|iY`*XvG-QAUgomZy}w=cwkQ~O`T*;e-Ic=05q%Z(#kWnDO; zyP(+Jo?q4_*gbMB>`<}Wg4}`KsRR!#M#9oJyF&@(-ARW-Nr$@@rws6bg%hC&%E5-~ z{KLiWXehOw5t;AGeHh!;UEqwx&Rq2Vm@i{S>JH^|Eo^#uh%UNW{RT+PT}xsP5}#&ZMi&Ock)-;R_VRf*;JCszN#X&jo_`3Rys{eY5R!bymJ z)wJ_HOZ)E_Umi0)y7H*`A^uFpZ&mR#2O00XI$re0P&=OYKodCoLX^}GZEL><`>Dms zzJ`6k^nB%5hsOUYcF^oOV8H=$Tkf;4KdZXDSMbxU__^~BOEBy>9ZP3yIpv#-t6sz} zo*n+scF7ZX+68d+F__m?hyE6Z6HpeGcvEbdPkvw<>&otOq0hBng+|kU3A!J)`qr6U z7?R}v8P=>RumYZKS8%q1_SFcq_5)k)XYf1qqivGr{6PuV4%l*^2O4JWkc##x)smx^ zjmRkFE3sYPgXVSh5dRIb4+jiG#-JhNQ;^Yuat!I3DB>n4;cr@0_V}(O{JRR?s~5nW zg($%W3158)Js587P>C&fkVsR(e(%7+5GT<9n5S0kK#nk^SY zywz8!{&%&}k6GY#1>;(+yP`5URv|)mPqpkt$#>QJs-G%jxBEGeP0$rOtV^M;gX~d| zp|bOT-FE4x5aU|p#ieh^EXidiXC9z{IbE($ z&)oapLH$^Q9k;I!a&eFEJZFw#oj8qU(vC(2=d62hv`oHAiGetdx_tL66mIk$n22%5 zvwIneexzDoe(Z`jQLQ7NtS?+-#<)G^QbS%j?}WdP7*spB|E)wGEjhJ}4I`q=jGQqk z3#FMklm}yQcoXvr?wq^pT%Y64?6H*BN&6&Q37ouhhSkOlHrBctp#@)d2pi}~ z_$IVMP%L&T(PHeCeI@Riyu6+LI~0ID4G>^H^KAJ0^o(5MdU?C^_*0hPn0^~pw zOP0e0q$Vo6l#tUNp2h(-~uSJNf?*5T>TPsgBR zdXN17ApRS?9%Ww^_Sx|7myit;{O{5J6#ikMC4Av%AtXsxVt70{7u+Fg4!;B>diRN` zAWVkD;Y}*!IS7PqXr*+AcLNEr(m`oKfbcl-J#-EZ^F6_Tf58+Cv3ZE?#y@bjhCdwx z*URuh6DkFN6igKVBg)zegxc!~&xBRdVdVvLo$!YU7rtGxMc~|HpMe>4rpv89x0t6A z{24U2wWmEzF`Y!F&iv!fZJ7Cne+<8y`a&5?Z^CI(UQ|&;A0ee!Ai3y}bq+kJHVmMK zza~}D4-mc>@gc#fdxzn7>|I6hxFV<~K`samSqHn+3z`Ed&PQUK`_i+2D@HbL1Xtsu{0qC0n zUsk_YRZ&7cF7I;Vo{19+4jwyBAFB1VvJj4IU`ybGgU-VU_i35!zlPk2xRKQXJsw~~ ziiX!mMia|s{)6pp4cDz+gkrd`@!G)a2TSk_4kC#z7s<9-H04{&*Eh+9BK+qE}MzA?^-UKJs<8c7~}U527Wsr_pA{xdLN=1t5RqTWSPGvO_sT- zUUw*IdLD~h$eqHM*AE9X=c8G(J&9B@ z=Dpw-{yp*mTAPUu1{ZZQ(#iTpB`>*N4V<00t0c~LThG9HG+~!*#(EW&oI+1L>I9e0 z{-T+O_gKnaK3qQkiE_Lh1{1~_2;txdSVS4LjkP}T4UWg-{l;ee1bBny9%&z&@t=Tf zUeWA7>AdW~qo`9NoDH3`a3Tpsb1m zpS2zVJKrjDzl>dO-110eJ$e)I4cO%2k&FxBA3jI{p5}pv5{xp~X=3`O%Hu9}t{?ZE z((`d2W>+k6M~?j%`><~7DQC)fF;+kRge91|FPk~>8W|W}bqeR+b096tv_qF^hr838E!@A%zw7t2zdMs$t&d&@u<$^88;;Xt z4`+=#jo4C%S&A3Q(&0iAJCF7|PL+2241sbF(T$%Y>#nflbhyJkANKjBcK7`K#^b)= zbOs#rjmQ=rfR^{8GcRSF5ZcX?bY0d9=!D`gJFu6P7X@Uma7u`3^;Rc5ho}1y8_IOI zm2TBXesd1g+nJIS#uNUu;Heho(fpnT*8XeJ2He)ed3YI)n-MOM5V}Z_?~FP>Lzpe^ z_E?dK*B`}#fe1f0O4jkD(6i5-)j54L;z6%iF~}KIFS|4={D~OOdXwA2p&y1N#drGH zFUSYKhqHc39~9?B?#_BVKIZQ3?w7ujs2sCdY=2;k34yx9^neqiLmcXX#lZ*K{F3#J zhv~^>7k8KZp>EWxZi#g5#iR<_>cOB$zRnqLO@CO;a4Y@nE}vub;_hv4^uJ zfF~~G@I68BE5S$-zpG@s4P^*d`Wfk;PEoNraeoq4bVI4fkDk&q)p{*DF}@Gj3pHU~ zmfDi*vh#aX9bw>Y42+Z^6l?pry1}uue}IwaBD^29rBJpoe|%K6Fz)R3zRtZ{k%}=J zUg+FA1PdO&?~$2@{6+<1epiC4>KAZ`W=mmRpryZkN@quQQu~CE+tIzH|I_CG?i@<0J>(qPuGm*mj@&eSiUsL9j+L97K!oLiknedk_CcKFhxF zcN}eVN{qTx$Py|0CYBlYfBLRS_o)qvSM>?Jy1s~4WXwy&wN0ktaH^>lmf4zp@5Bg6wS4G4%>4XU*Celc^}n=saGNDQv-7inT;f{ zm{^a405$z$qPwg;n-!%K_9x+U$of5C80#VHZLqNl9;M|9j zEQ~pd$NRsVfbS|#au`UQQ_E;FkO*j<3JLjBT1Z!+YX9Xk2TnBA!&(163Bf}Tu^`OH zdx$+edR2pVZ#b)kK1l2Wqc)XuMH0TznuMRk;47o7M_igiQ#YJ72f>C9bq*(7`v4+z z>TR!x?zwE#ozdl>&}m%Vi&@hPC!_yFd$C3ON0;=_uDE$5yW6@^`f!41T`R3f?*8~S z!;G_}*9iRYh58V*?4%lmb%ijtDhGbN#yTwC7vwg#JA zTRv!N^_ID|47LWn&82C*C848u7dU`c*tO+DS9`FrH0|}!%Oz>AZ~5RtZ)-a?s)tI_ zUXmS#?JkU;N_GN>z+?OAMPA-Y&^w{F(m(xOxuqS1kw=Pf{^drfGGNc^b6`WGF1=0(EGNa3KslIMxZdkAoAKeWXMhNu4lUE=WckL3Ty<^MhM|6TI`!}9<0@;`qeaCrLFC}L=@_)xB0TntXI!PUfqd`O5X5 z;pt;0FmHCq|98p%{2eJIJMN3lUx)i+y^P;`!xB-250@T$QPA zfg53^y-@pUeh&KAdn5dPsPD(#{)MF9LwldHJ4v_pN{ROn-=px`X@3}Y%`ca958oeg z_h3ABEb{>wj!#1$OM#uV{2GR$W2e=A)#$N`67e4LpL!y z(T8APX}X*EkklbuYm`g-+mGcx$y6adHBA5Ak3`b%r2Dq0Tk0pG@2)C7RV-hiLYZ38 zhz6|@uaAZUMMe8g$~OIShp z7^eug%k)c)3{Ovhej?h&&=t24Ln-0cNTw4L@!fnKmYgQwW98AYg-a_4+jtwM---(Tk3bV|AdP^ai{~p_I`$PND z+Mg(d4?;k@D^{^kgh!c#V0oqHJI~L#7Lf$c3qLm%i=Kz^uRLU(3LKU?QUQ#kh8Mhu z=OQlcaUZe@2Eg~B?d#U?HL`rk_Ez(oqWN7%ejSS6B*ibm#P6@Y2EQTjF!=SK$nR}> zL%!MMm#g@_MY2%VGXREscSiX2FaSQvx0{5zKhW}((;NKW#LrBG`|uCxEhAYdD<8n% zw=BXhmjUpRpPhuU{4(hce%<7^P4OE`vQXBiR~!5i!k0(%HwY|0^6P`$%#TOZENc0^ z%ldCYG{2{XQDj3I7sHNy5?q2|?25rx|Nl>EU_vQ+34BfsZ?07CS=zmOxpLp8VTXom zG`v8=uW2|z!_PIJu!b*bxL3owH1um|DD`K2^8?>z4Zo)0^W_S^Rl|!kOwe%OQsp1i zuu#JU4e!55`LEINuVu=;SHlhs&(rX)@;HMK_iK2jhJ6>RaL;J?h=%uRc&mm!4c!_} z&~Rvp3jYTUdo;XPL%)VAHC&|OIvu~Er_(e(MZ+PTU;nM)OBz0|;r$wZPs7a``ZT;; z!&@{o;#ssz$$O@TnHpAU_$!^?eL9~$(C~8Y|G9P>{BkZ<{4zB(@Y$Cr{0G*QByHC3fd2=oxRfn6Kev4HGnccd_E%r{O*g@73^T4Qn(!Ps1r1 zrfT?Cm*Vq=hWj+!q2Wy$wraRa!&w?;YA7^(*QxmRXn3E7w`;gn!z(nrNW(%6r)ik3 z;Ri*EU%!S2HM~>9Z)n)8;TjD~G%V0?jD|xOD1I+#*rVb78s4tqwHh{SxKhIs4f8b2 z)=+5p?)i%UD;oYx!`n0rYFMM8TSKFqdd3Nv@|t%QH1NMYs@$8$YP?;+JPm6!+^%7d zhJzY9bU2R@-e9buUGqPpVVw6#oztw}p2QR{ny%7KLd?%ZPrgMs=)lL-md0$0OxL!z zl-z?2%DsKPf?~3UI{p1x&VC&(C9J|3_;t1Qj@sttwz}G2Q(LPLHIr2M5$XK?+D(pD z??y+nw{?B6kut4ly|T4!W2?j4QRnq37DNm3t*Z@S;3&j&tN~4LZpMEh)~e*9J}Te|GxOP7hoE0!!Nt;A+$h3mr7Y6EMy%gdIPy5MWLFZNV{5UYsd z8pkwOBsqa;=Q&7(ZZuV`%#cK91`g*w<3uHL0UVkDeA4W|nj> z@q4{d94Qt4#H2^*PxL!Uxbmhz5CZ?36hMX_jbv7B@-e{@iGLKsj7F+-QWOQ;7dJEn zyyQfTks^k>4F8xw@}?gF>@o@m5%^}MPP=$xYit%n7ZDmqMXk^0trv!%HV zFOd|aj=oH!Y7HZvlScwMu1TF%)fTL6Hj)l{x3|{kpp5*0r&y7AnT{pRwd;)%R#LjM zthltg{NiF+Y-Ku^2Wx|Y^AtxFJ{ks9Mt+XCfDZS?%d4tamMt$^T+T|Q+{G8WODjv4 zRYlycVs~i?892&aYs$)3SG&p?b(KVtH}y143ZUXoMNXt%mn!DWWhf!$%$d8csdcVc zDOQMbu~fLkGEprmL@CV0qEf7e%LSiuxQbz`lGtjo9QF#~0bW({l}oG3U8_qgtIJm` zb5&kg9SGL8*4O&$t2fj(H`VJ#2=T3KZ@#j&)luPXY4dM#=&We-JC@hB_?o?rd2{ll zSdo7#@uP{&;!05~TG3us2(M@nZNe`$!R3NaGhB7>4N9y7a@B&u2fP;)^TZsX7)}p3 z_Bo~pa1;RL4`#!a#eQ#X&|BhN*S@~e>ud7|i_wa#tF61TbQScw;YyQFg*OnWUGJp_ z@qwVfZIj8{9B_GKZN1k&XM@)toD=YE5Zc|~uU#+QEwxQ(A%!Tf-Q@KzRydzf6Y=@2anDfl@a% z)m_=-C~aMjEt43&8*6=ajan9}WrMda*ydkUb%7(;*5+t#Yh5qN1=8Box;~0*YLyx4 z(0E^4ph@-)j&#+(T$`fcHVwCHc!!3&HQcLVqYl@j;dTwPHO$peNi&EutNdc2MInU& zwQ~fZ!kOu4Hyahm;6DP37}RFSLo@Pc=4*~KD*s5~6u(${CF67Lo__}8<2Gb4Lxi8A%5ncgYFxGB zK?OH!X#X$e&bwd1&)-n6=RXy!xlh3aJzh-Fu;wQU@7t~5=3NTrY1n_Sa(C$XMcjCg zQtpAd;}j*|EFG^g&U|HFR7)IZ_QUM}E(LUj+HKGg_L%4x?_s#(>2$CNU^qMI92#Q=tZ zKu5WE&yMorj_F~z9f14{e%zBCni-|T`CO4V%OVa_0DJ}=_2M?sQSMc6_tFEOK_@wx z=*T7oZYPQG8FY*{e3qn^c=k_@uUT}@6{VA&rhg2v7vUTlXVzc*e4}v$I*OZPikEsS zgxjh48S+u??YU9;DDQ5#YvSn0uh2wCer~uOadhMqFP|8Jj{NSP5sjDp_QLG|zFg2bNQBSOGvhsCn%7ajez@b~9f_YC zbPmw-GxKv#BenX-Zx!4Q?KbGNGXfYQFOKflKhHUjZU1i`dXs&W6UH?>{PnMcvz6%shyrThxI0_X}x6j6VM?}drdkM7NJMoeG4V>RiRhRa~8PP2%rWR-HmFCF+y zn1|u_aGW0x8KfZbhv9cP&TkC-%-?Nh$N!=|rAGB9@hp3j6~0fycz+{R{iHI0mOLplKeHT+XQYp9M&2n= zzo-puaf3yK={G`pd%PA||H#uEPdF}~J3wdfG}F~gF|e^xvCTSsYSeEJ(nA^fnZtL) z$>cyh{MZigV;CpQkck(ZGEId1%=~0J6{j zm79@p499#gMEy2C+V;ROUDOHFk1?I$9f3peGqC3LIpX9t0+7~c;$wuRe+_sIGM`4s zs}xfh(i-}w-5hVc{^`$;WptH^hdExB8I})>X(g@sA7wJI=6Ix>Cc+eyU85clO}%+w zHk;xk-uOscXwp#+=G2V{-ci9t(9nR89)DKy#XZ_YQmjq)3xZ{~EF z(;uI%?I<5#wrny!67s3o4r3Qsi`S=M5$NqegWA7GokwE=H+e%`#Z=lJT^8FJ^s zLc?zc%!S2O5qABzFa8wRdLuB~|9MfruFqHAf*$;%Ty;Ft*t z*CBj^Li{qqYg0XZ@nH;)$-x{hSk>m7O$_=!IsTZFmsi)Y9{m#B)ZLuo%Bq}I#V*o> z<8VX{#^ShyErx(|0_w0Uh$|cw!*307&KO*ojKdv=BhnLxD+X>=EKc&PiNSd!zs6Xc zr1Qn#sw7=UEKbsG1}==enB%9Od&#r@tDTVC$0lHQ~nna$Ni^$I0>og&LK|JpzUYcRFrb8vKMpK(p?phyiOsqktZ!(l59Dl;G$`0Lu*~rX{B=2b^N6bf4x|Q7%1!(~`z$!2 zoSdJKXGu*GsRhX*H9y&BIVDM)QaRB#p(ZQOGR`W-tsCRZs7d#vsZ7WtemC&MHN(!2aEGRQ5Rl)W0U@dwhe0lBE#p(f_^MPfzF@Q% zpKlw?=uh)awI&E_eTr~^rV*wCbC#(HV|GuPif{#XG2Wj(nA)G>8 +$s0gD0i7i? zMP$~ei_EGtky(%`GW~X+#g;5=^`nFhX|?62`7G&Hk^V$Ve{ye9Pl88?sb{5%vl^`; z=c9CSdN51a>oY}4!FbU-MTpN~2Bi$C5C{brGy*gNG$M}1f@=?>^R0u4H3=S>j)D~7 zISmKtVBBXQ9g~y9czD3WklAuNbz&E%*W1MD1!KhN{_MdC{h7Vvdh8y{ zgk&+Hk!5I%NWC@LGj&apSQE@Z{-q1&G$B^Q6y{KF!~?t;H)(RRNb`^CxB8N555NoyMlemBR-Y;+RgFfON@1DrXPMs|SLW@w1IKdxDImY; zvvfU~Y8B%x<3vhta!(S<8`D|?|EZI}^P@2$>usAD3mvD{j}=J;6ND4tHo??PS9xxB zkEJ|SW04mu|FX{XjP|9}P@Yv<9@&q9|IT;uk017NDA!N)e$Ej<*27!Sf)KCU1J9&s zsbU)P?o`yBY}B39%H+YM{sbR+=YjVgO+Q|@G8X_c{MgxPVs^n)G1ZbSCJbiwkL$Jf zsMbTZ87W$>e#k?5iu7e5+Yp0}VeIpj`*}cq-Kg-$J+T{*bYm%h!9+3P!EuB3{xQAj zJ*mFYH7Op--=C@4r$H?PdD|DLaKq@C8npH&`Vwl8Uo1lf z>7obz@55xzK>lPViOeUHg|#X};<6TDOhPVtuF3zkUBc>wGDTs4SW$K`A z^(5cz;L9%$D;A``#wnfRc0d{4kqVi{h*JtOL`LQ4LDbeBYJ&P~)H2ykGSw_r;rami zZ75dqd<&5LCY_Teatfx%x*%I_nL@UfLi`MM!U>)?ARK8OCcbpfg4(L93Wh zpCl$!C5j0J31UJ&cqO6Vou(mlYVL;*gP)^H#Uttd0ROW`LBETk-$f$H7uD+y(7uGM zrc55Bf!|s{=6<7g6VjhB3HTr&=jEFL`*l23ax{uCfDF3<5Iu`{50GK*T&co52*`T* zsdfi0RsIdDv|i?<2}dwrIO^vKM?s!&SZ0aL!Eya|=$vh%2h1m}Jw+_-n=OhD&k+~w zohugX&JzW9%oB6B<%_R%%ok@i7KmwYw~1-NR3ZpRT$ZaVSK_qZ1eR( z{5y;|6K#m>vr?qH_Up(C>6Xcs9Nc{>Qmy!{-I~_es)OlD+T0tIUseb#&I@`G|m;hw;=p>>fg~2R~Vl)*0=5 zvKYJ#za;P-q#64_a;$*-iN!%bCYnjpM~R8SGsMJ#X=0+qA=0!=gXEo(BvSHat<018 zigB;=ZXs@ish9jkjC(A`ieoD?eCai5d2DY{sih9`pnras12VsS8eR|BdymqS{a#5T z<^qzY77%I?-vgwcj{r)2%JIMK=WA418!fQcDE7k)X_?G!OIC`=!Wc3Oei{c1c{h_EaJdk~IL&ewXB{i^*sJRr-@ zen8S#ZcGt3R&g#Bk=bWipCZ=B-ZTH zc@r`⪚%7Rc49n$y4ilEXyp%#JvKX6VUj2kRHwhodyE%J>b~i8{eL1RGg19mfQ)xiY68|~K0#Can0 z>sIvZ&Lcup!&G6Xjq(8fR-%2Ik|@SlCJ2s?*e~t|y(Ql&@;lHyG@|WV zEQHG24CHNkByYoGRo-rY4*~FbUQ+418jzpsW#yLhD4j>Lt|8C1zaqo~Fhf#4B;W)b z+J6&`MUIKFK3g?PRBc0EbR>v#8xv)_YT~ck7u4lGAzp)7m9Nroc3UQ=ipiL3O-4PK zjJeii=wWi@g#JlW*ng*@|5o#_lu9)&<{A^f!Z%cURs-_;j&_fh^ZuwlJJ60YMDA~e zxZq9nThI?7abxkorMZk6wDW(5{DS$F*5&Bz6p_ubM1f6Y-zrBfK7)2U==Q>dK}$GV zkKYXcnf*dM0&~`&f|dl7XXLFVBS~Zwq>2oGvS+MJs~WFQsg!f=2>(C5g;y^Afc!&v zek$B(k%6*k$zor)4}IZY*%w+yTg2#fpkcq`SN)DGlQ#f2E?JBVqTNQ`kIPTva7##8 z@&Gv}MH@c}b$k-?bW#D@j{K~C#>elYcNBCNFjVY^(jsbr_naN;XD3;`!Wnfv8>j-! zi{8d(@ZUWGf7ZRjz>gG%5uSTt{LJy(78eiY%!Q2cax$C)YgGIOM~I(z?o;qH^RGdD zivPHe#?MT*V{DXa&w0w9n3PhiPok`FZPEBVz#%t8USSlMw*>o57Oah)8^t|Sfqhia z-C?2&F0%*+%9dCVrQ3Z8_8o!CHpP1wxE+bugEi5Gfg1!}*u*cj5^H|I;g@p_@kAbQ zy}%Wka30`#fZJh;qklQpHIcTFzOU|`DZPAk?@a08t9xfk4`1Coi)}Byx_74Z@zuRE zrHB7l@13oPp8Q|puWi9``-TRuUx>w#n*?R8yz@}!6=EZB7kYzbEx0qFcEKyXSG70! zy)AgZD1f_7kud6ZKt)@9do#k_sKR*~HwBvNYMV>E8=C68%he5lC2jr|+#w6#7)RoH1_4Dt?%p;Heqk=r3gdS%wEi~1}#ZeB1ezX|*a zY0){pTwMu~IsR|a@SY~0w;8uNaDk+)J%}4IkvO#&-v?#|cbajx#SlU&T`Yl*lDN`~ zvf%gBa)0{l2-aMfOgCMYcx9=+BpOo;V=ahCCm)sDE0GCCQ9}&hhR-GK-gfV$wN1gY zdKC3UVD!llT|>BPs7jtnhEaxY0QLk8NoGrGTbi0TnX*k>rD#<-ku*Yl3pi=@_`LyC zKdSR;MQ2o4u&8s@*ZalRk>b+1QC#b*UDwo%%PwA72`>Y!&TEr0Ex;P-iYYG8!lls= z(dsEmfK_=Cm$70Cd`?KYDYgjGt>M2K7i8 z+6dp$*7g>!A4SQ~mAtlO5<+|fR3}KOb*5F0x~+6}v!adE04avl<|IneQeWb)-6&fr zT=Wx7=J;c4JG5B`UzO33LM2XBTu@a_S%S$4@}y3-zsqoLNJh$1FjY|-3R+z2_cyUc z-vm5TBiVu~o%OCt2VD+@dRPYj4F`-_p{`Wu@iw^A(@=1W#Kacu9LxP+t)gGc2GlZf-(T zQ;92eVtPV(TWx)DyFbw8zf{bSuClte)=R~OiBF zfUh=K*BHfhCsZQ)13|C9xET$2X$R=85~}>6{M5H?Bq}H+i-t-RuKFw513^VW(Mqw> zxKosP8x)GIAULg9URr6S?jBK8G_Sh64Oa+*&Oor<>Aa-W<#MiCgPy*qDDOlcw-Z}# zuUd+#p@a3!ZTebvv)9Rc=m3&OvCC8HTFJmoEk5VGeDVt=6jiUn)$(dw!ECEjbgNyf z;PdbFK}T2J)=&*~t=CN8=_y@Y!uW1Se2ZNr7cW_&!p*O~XfgIU5w5Dy-?p*3xZ3OY zxA`MXDdf|MMNRe2qMSN3O?4ZB&XST9)fMY7ehYeoPNyQxUzw-H&Ud=RPZ7(KCJZlH z4Ms@Cit47eb?psKCuMQV)ZapR)iwIKcnx{CSgR|79k{V6uaQA?Q-L_Et&pfYNeZZSV_{vD{U?6rv*bmApwBH1Z;!La!{ycT$IQnV&^D z>uNFZ#MNE2Wi3os$qJ~bvDRN5^rNx`oFyvl>VKIo3q6TN>%Bo{+y9WkC8dj9MXr?* zB_p^0hE$ay(f?Ccl!c{bhP9}@HPG19@I_i-+-?_hFq-YqQpr5v2%(gF6N>6^Hq{@I zrKt@X_-7hn?O5fmUPg^Rl~jbrN0z96i9xzKN4_v-BQGY$gi;y##WJHLxK@7o%$w(w zN&FObs1<4ZXLV>~H80j`UR**|&yT8H%ugypUH{)@dL48!&q&=DYI21x)m`RLU$RCR zVP8ewH@0oW#B9_5xKdOg{_2ZRltr<%s3U-yMIHZ-DyCvV)mC4GqPT>7(AWLTd=@J{ zoo0=o|6wEX&&!sfkM5)*5VO5i>G)ECS(#8WR-%K5Bt_iGTIIcR{lCl##k>J`)}7{h z?^JDIqO}O`2{-=>ywQk_%of@B!-S&R=8d(R0@bZ;==L`_b#aTHwDT-Q zMT^ld=@v24<5n+ad8k;vve+}PAWw}vrip4a)b-Vo_gRb%1tXyrZ%a$t1}{dq@icW9 z{0ni0#$yQ4($tDRPmBSkxT8Z3`_$0KKOcJ+#8wA<-nym+^b3u0OoS2a1&jf$8}~36 zO!l*-(T8GI7FVuVhKE~9Bhq5U0M9_IZ}rw!H(=30d|fo4MAl(!UR_;V=SL^kwywIN zy|oT&Ck+kFn6#q@7Y%jIZ2_z+AQmHZiyR<`4$**`QHL1}x<~mSLA9@z%Q8q36y4O? z(1tmtOnp;ZbsZK-+FGkOH2CnAL9jvOh-%e~RyQKsUf`I z8*KNtVuMFigNg2qtHBrRP1Qc64QUKuW>6jQHd16J+1ug8N)#}`YCMxt9mLx$SbahI z!7G4Vt_?OKCA>KaWwbUlt#9{xt6MSS!T6}wzaD3o=2IrjZlUO_BVOLt4Nd;GRxVp% zS53*Isf^HOl5-Rf7Awq&BtP1#y?9&1+kwdfOP;h;DOnp@F#h5>q3UX1HA;eiJ#=P` zx*!+wrcQ;;f=z`!U*M$+H3Y5>HnM49{?#^94b>>lh^iV+`Iv_+#VXB;R8=nWrMB?s zQMD8TWsEY&qC%Na_Q?YNjd-?N7dx?DFhQ!$6vSw(tyM-FtX;a4maLp2HM{??fskNPRDNKd@C{LVj z;8m5fdXZ_uq9s9Y0x8}kcDa!7Hk_u`e?cyA?^*T}Ry=amS4Nno2SM!}YX zzA9Cu!IqAm7Uw@J7jW|a$)jsns%L>t|+IvtSzw4>5fku5}gs>mzqbb zWd2jRzi?6HBsk4{A25^FqT-eR` zWuH1zi0g8pE8q$D!|Z_Fd_R`=W`yr92F~`R=9pekJc&{t{*p?p_i;3sVI<;R`VJu=Bp;>>~8Ku=CF25f}6id(C1Y zE-FDe!~0^qW7xP926i9d`!M~m;}KKw<4U9#cHZMV2D1v~e-QB0D)3}o09*;P8}=H& zH(`2U=l!`CRzZH)djZd0t;)F*aGSQn{I=Er(J=v1iJ(97chOW^M2a1 zS3(}xdADri^{6Ab6K1~uWxgB6J7V91xdSwWYc@mAuoHH|{0jE%fZq#27qIUDeC`&= zkGoRlds4hVm2(@$`@j=k1`~$82Jmf|A=u6Lpm=BMr!aQhAtLO7$$_0P{dUx4*zJJ7 z{122{*n0tQ!I*`0b35Rf-&Oj_13U`DcB3D#==;j<1PsEkt=JA&`U4f#1Gp7N;sFPK zsOSd)AG}l9djJc5r1cM2v_sng7wpt_z@odf9kA%f+74KBx3&Wscee;<-GjVCS_lha zw!-cNd88APC9dA;KG5@7x;Jv7g4jgw3^?r%=zqay7GNLDUf9JSg*XL<^o4-e zy^SXVfZqz(2Sa=q(D5f_C%giN<+&N~5pCZK_$~}(BYf*b*g6W;P3#sIMMu9m!m1sX!$ zu_8So?^DrE$h%P;kdv@QZ61DU2UxJj}T;%V0bjUaH|`fN3yaegBs--rFwao$V49DCEpG9*|9?PP+F)}Th8%Z33FPdxU4UX7MJc&!ao85bF-@V~` z2k#xaSL{mJW#5&(%dsnWSKco7F3+yryY}qbyQ^o{;a$DE`gZm23hx@+m9pEuJ9~HI z{l5Dw)?l=f5-hh?%#d?p8NOS-*Z2rvZG%EDBS7X>E7wtxoT(4&c>a-ogF(j z@7%U?`_4Od?%26|=boK=clPW&yt8j-|IXaq&fV_ad^^p7aQ#0Pcjw++bN8ORhwgUV YR diff --git a/docs/html/annotated.html b/docs/html/annotated.html index 20a707d..1872afc 100644 --- a/docs/html/annotated.html +++ b/docs/html/annotated.html @@ -64,14 +64,16 @@ $(function() {

Here are the classes, structs, unions and interfaces with brief descriptions:
- - - - - - - - + + + + + + + + + +
 CVmaAllocationInfoParameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo()
 CVmaAllocatorCreateInfoDescription of a Allocator to be created
 CVmaDefragmentationInfoOptional configuration parameters to be passed to function vmaDefragment()
 CVmaDefragmentationStatsStatistics returned by function vmaDefragment()
 CVmaDeviceMemoryCallbacksSet of callbacks that the library will call for vkAllocateMemory and vkFreeMemory
 CVmaMemoryRequirements
 CVmaStatInfo
 CVmaStatsGeneral statistics from current state of Allocator
 CVmaAllocationCreateInfo
 CVmaAllocationInfoParameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo()
 CVmaAllocatorCreateInfoDescription of a Allocator to be created
 CVmaDefragmentationInfoOptional configuration parameters to be passed to function vmaDefragment()
 CVmaDefragmentationStatsStatistics returned by function vmaDefragment()
 CVmaDeviceMemoryCallbacksSet of callbacks that the library will call for vkAllocateMemory and vkFreeMemory
 CVmaPoolCreateInfoDescribes parameter of created VmaPool
 CVmaPoolStatsDescribes parameter of existing VmaPool
 CVmaStatInfo
 CVmaStatsGeneral statistics from current state of Allocator
diff --git a/docs/html/classes.html b/docs/html/classes.html index 67a1270..ac33cf4 100644 --- a/docs/html/classes.html +++ b/docs/html/classes.html @@ -65,9 +65,9 @@ $(function() {
v
- - + + +
  v  
-		VmaAllocatorCreateInfo   VmaDefragmentationStats   VmaMemoryRequirements   VmaStats   
VmaDefragmentationInfo   VmaDeviceMemoryCallbacks   VmaStatInfo   
VmaAllocationInfo   
VmaAllocationInfo   VmaDefragmentationStats   VmaPoolStats   
VmaAllocatorCreateInfo   VmaDeviceMemoryCallbacks   VmaStatInfo   
VmaAllocationCreateInfo   VmaDefragmentationInfo   VmaPoolCreateInfo   VmaStats   
v
diff --git a/docs/html/functions.html b/docs/html/functions.html index a924d1f..0c91a0c 100644 --- a/docs/html/functions.html +++ b/docs/html/functions.html @@ -61,9 +61,21 @@ $(function() {
Here is a list of all class members with links to the classes they belong to:

- a -

- r -

@@ -170,18 +205,7 @@ $(function() {

- s -

@@ -197,6 +221,9 @@ $(function() {
  • UnusedBytes : VmaStatInfo
    • +
  • unusedRangeCount +: VmaPoolStats +
  • UnusedRangeCount : VmaStatInfo
    • @@ -209,8 +236,11 @@ $(function() {
  • UnusedRangeSizeMin : VmaStatInfo
    • +
  • unusedSize +: VmaPoolStats +
  • usage -: VmaMemoryRequirements +: VmaAllocationCreateInfo
  • UsedBytes : VmaStatInfo diff --git a/docs/html/functions_vars.html b/docs/html/functions_vars.html index e09f776..5bdd9e8 100644 --- a/docs/html/functions_vars.html +++ b/docs/html/functions_vars.html @@ -61,9 +61,21 @@ $(function() {  

    - a -

    - r -

    @@ -170,18 +205,7 @@ $(function() {

    - s -

    @@ -197,6 +221,9 @@ $(function() {
  • UnusedBytes : VmaStatInfo
    • +
  • unusedRangeCount +: VmaPoolStats +
  • UnusedRangeCount : VmaStatInfo
    • @@ -209,8 +236,11 @@ $(function() {
  • UnusedRangeSizeMin : VmaStatInfo
    • +
  • unusedSize +: VmaPoolStats +
  • usage -: VmaMemoryRequirements +: VmaAllocationCreateInfo
  • UsedBytes : VmaStatInfo diff --git a/docs/html/globals.html b/docs/html/globals.html index 7eaee9e..8479115 100644 --- a/docs/html/globals.html +++ b/docs/html/globals.html @@ -71,24 +71,30 @@ $(function() {

    - v -

      +
    • VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT +: vk_mem_alloc.h +
      • +
    • VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT +: vk_mem_alloc.h +
      • +
    • VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM +: vk_mem_alloc.h +
      • +
    • VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT +: vk_mem_alloc.h +
      • +
    • VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT +: vk_mem_alloc.h +
      • +
    • VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT +: vk_mem_alloc.h +
    • VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT : vk_mem_alloc.h
    • VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM : vk_mem_alloc.h
      • -
    • VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM -: vk_mem_alloc.h -
      • -
    • VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT -: vk_mem_alloc.h -
      • -
    • VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT -: vk_mem_alloc.h -
      • -
    • VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT -: vk_mem_alloc.h -
    • VMA_MEMORY_USAGE_CPU_ONLY : vk_mem_alloc.h
      • @@ -107,17 +113,35 @@ $(function() {
    • VMA_MEMORY_USAGE_UNKNOWN : vk_mem_alloc.h
      • +
    • VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM +: vk_mem_alloc.h +
      • +
    • VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT +: vk_mem_alloc.h +
      • +
    • VMA_POOL_CREATE_PERSISTENT_MAP_BIT +: vk_mem_alloc.h +
    • VMA_STATS_STRING_ENABLED : vk_mem_alloc.h
    • vmaAllocateMemory() -: vk_mem_alloc.h +: vk_mem_alloc.h
    • vmaAllocateMemoryForBuffer() -: vk_mem_alloc.h +: vk_mem_alloc.h
    • vmaAllocateMemoryForImage() -: vk_mem_alloc.h +: vk_mem_alloc.h +
      • +
    • VmaAllocationCreateFlagBits +: vk_mem_alloc.h +
      • +
    • VmaAllocationCreateFlags +: vk_mem_alloc.h +
      • +
    • VmaAllocationCreateInfo +: vk_mem_alloc.h
    • VmaAllocationInfo : vk_mem_alloc.h @@ -141,10 +165,16 @@ $(function() { : vk_mem_alloc.h
    • vmaCreateBuffer() -: vk_mem_alloc.h +: vk_mem_alloc.h
    • vmaCreateImage() -: vk_mem_alloc.h +: vk_mem_alloc.h +
      • +
    • vmaCreateLostAllocation() +: vk_mem_alloc.h +
      • +
    • vmaCreatePool() +: vk_mem_alloc.h
    • vmaDefragment() : vk_mem_alloc.h @@ -164,11 +194,14 @@ $(function() {
    • vmaDestroyImage() : vk_mem_alloc.h
      • +
    • vmaDestroyPool() +: vk_mem_alloc.h +
    • VmaDeviceMemoryCallbacks : vk_mem_alloc.h
    • vmaFindMemoryTypeIndex() -: vk_mem_alloc.h +: vk_mem_alloc.h
    • vmaFreeMemory() : vk_mem_alloc.h @@ -188,27 +221,39 @@ $(function() {
    • vmaGetPhysicalDeviceProperties() : vk_mem_alloc.h
      • +
    • vmaGetPoolStats() +: vk_mem_alloc.h +
      • +
    • vmaMakePoolAllocationsLost() +: vk_mem_alloc.h +
    • vmaMapMemory() : vk_mem_alloc.h
    • vmaMapPersistentlyMappedMemory() : vk_mem_alloc.h
      • -
    • VmaMemoryRequirementFlagBits -: vk_mem_alloc.h -
      • -
    • VmaMemoryRequirementFlags -: vk_mem_alloc.h -
      • -
    • VmaMemoryRequirements -: vk_mem_alloc.h -
    • VmaMemoryUsage : vk_mem_alloc.h
      • +
    • VmaPoolCreateFlagBits +: vk_mem_alloc.h +
      • +
    • VmaPoolCreateFlags +: vk_mem_alloc.h +
      • +
    • VmaPoolCreateInfo +: vk_mem_alloc.h +
      • +
    • VmaPoolStats +: vk_mem_alloc.h +
    • vmaSetAllocationUserData() : vk_mem_alloc.h
      • +
    • vmaSetCurrentFrameIndex() +: vk_mem_alloc.h +
    • VmaStatInfo : vk_mem_alloc.h
      • diff --git a/docs/html/globals_enum.html b/docs/html/globals_enum.html index e8d4941..0589833 100644 --- a/docs/html/globals_enum.html +++ b/docs/html/globals_enum.html @@ -59,15 +59,18 @@ $(function() {
       
      diff --git a/docs/html/globals_eval.html b/docs/html/globals_eval.html index 4fad40c..8c6790b 100644 --- a/docs/html/globals_eval.html +++ b/docs/html/globals_eval.html @@ -59,24 +59,30 @@ $(function() {
       
      diff --git a/docs/html/globals_func.html b/docs/html/globals_func.html index fca5d30..06ec566 100644 --- a/docs/html/globals_func.html +++ b/docs/html/globals_func.html @@ -60,13 +60,13 @@ $(function() {
       

      PhysicalDeviceProperties are fetched from physicalDevice by the allocator. You can access it here, without fetching it again on your own.

      +
      + + +

      ◆ vmaSetCurrentFrameIndex()

      + +
      +
      + + + + + + + + + + + + + + + + + + +
      void vmaSetCurrentFrameIndex (VmaAllocator allocator,
      uint32_t frameIndex 
      )
      +
      + +

      Sets index of the current frame.

      +

      This function must be used if you make allocations with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT and VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flags to inform the allocator when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot become lost in the current frame.

      +
      diff --git a/docs/html/group__layer1.html b/docs/html/group__layer1.html index 80652d9..c58e442 100644 --- a/docs/html/group__layer1.html +++ b/docs/html/group__layer1.html @@ -70,20 +70,20 @@ $(function() { - +

      Classes

      struct  VmaMemoryRequirements
      struct  VmaAllocationCreateInfo
       
      - - - - - - - + + + + + + +

      Typedefs

      typedef enum VmaMemoryUsage VmaMemoryUsage
       
      typedef enum VmaMemoryRequirementFlagBits VmaMemoryRequirementFlagBits
       Flags to be passed as VmaMemoryRequirements::flags. More...
       
      typedef VkFlags VmaMemoryRequirementFlags
       
      typedef struct VmaMemoryRequirements VmaMemoryRequirements
       
      typedef enum VmaAllocationCreateFlagBits VmaAllocationCreateFlagBits
       Flags to be passed as VmaAllocationCreateInfo::flags. More...
       
      typedef VkFlags VmaAllocationCreateFlags
       
      typedef struct VmaAllocationCreateInfo VmaAllocationCreateInfo
       
      @@ -98,59 +98,64 @@ Enumerations
      } - + - +

      Enumerations

       
      enum  VmaMemoryRequirementFlagBits { VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001, -VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002, -VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004, -VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF - }
       Flags to be passed as VmaMemoryRequirements::flags. More...
      enum  VmaAllocationCreateFlagBits {
      +  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001, +VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002, +VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004, +VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008, +
      +  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010, +VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +
      + }
       Flags to be passed as VmaAllocationCreateInfo::flags. More...
       
       
      - - + +

      Functions

      VkResult vmaFindMemoryTypeIndex (VmaAllocator allocator, uint32_t memoryTypeBits, const VmaMemoryRequirements *pMemoryRequirements, uint32_t *pMemoryTypeIndex)
       
      VkResult vmaFindMemoryTypeIndex (VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
       

      Detailed Description

      Typedef Documentation

      - -

      ◆ VmaMemoryRequirementFlagBits

      + +

      ◆ VmaAllocationCreateFlagBits

      - +
      typedef enum VmaMemoryRequirementFlagBits VmaMemoryRequirementFlagBitstypedef enum VmaAllocationCreateFlagBits VmaAllocationCreateFlagBits
      -

      Flags to be passed as VmaMemoryRequirements::flags.

      +

      Flags to be passed as VmaAllocationCreateInfo::flags.

      - -

      ◆ VmaMemoryRequirementFlags

      + +

      ◆ VmaAllocationCreateFlags

      - +
      typedef VkFlags VmaMemoryRequirementFlagstypedef VkFlags VmaAllocationCreateFlags
      - -

      ◆ VmaMemoryRequirements

      + +

      ◆ VmaAllocationCreateInfo

      - +
      typedef struct VmaMemoryRequirements VmaMemoryRequirementstypedef struct VmaAllocationCreateInfo VmaAllocationCreateInfo
      @@ -172,32 +177,42 @@ Functions

      Enumeration Type Documentation

      - -

      ◆ VmaMemoryRequirementFlagBits

      + +

      ◆ VmaAllocationCreateFlagBits

      - +
      enum VmaMemoryRequirementFlagBitsenum VmaAllocationCreateFlagBits
      -

      Flags to be passed as VmaMemoryRequirements::flags.

      +

      Flags to be passed as VmaAllocationCreateInfo::flags.

      - - - - + + +
      Enumerator
      VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT 

      Set this flag if the allocation should have its own memory block.

      +
      Enumerator
      VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT 

      Set this flag if the allocation should have its own memory block.

      Use it for special, big resources, like fullscreen images used as attachments.

      -

      This flag must also be used for host visible resources that you want to map simultaneously because otherwise they might end up as regions of the same VkDeviceMemory, while mapping same VkDeviceMemory multiple times is illegal.

      +

      This flag must also be used for host visible resources that you want to map simultaneously because otherwise they might end up as regions of the same VkDeviceMemory, while mapping same VkDeviceMemory multiple times simultaneously is illegal.

      +

      You should not use this flag if VmaAllocationCreateInfo::pool is not null.

      VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT 

      Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such block.

      -

      If new allocation cannot be placed in any of the existing blocks, allocation fails with VK_ERROR_OUT_OF_DEVICE_MEMORY error.

      -

      It makes no sense to set VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT and VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT at the same time.

      +
      VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT 

      Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such block.

      +

      If new allocation cannot be placed in any of the existing blocks, allocation fails with VK_ERROR_OUT_OF_DEVICE_MEMORY error.

      +

      You should not use VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT and VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT at the same time. It makes no sense.

      +

      If VmaAllocationCreateInfo::pool is not null, this flag is implied and ignored.

      VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT 

      Set to use a memory that will be persistently mapped and retrieve pointer to it.

      -

      Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. You cannot map the memory on your own as multiple maps of a single VkDeviceMemory are illegal.

      +
      VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT 

      Set this flag to use a memory that will be persistently mapped and retrieve pointer to it.

      +

      Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. You cannot map the memory on your own as multiple mappings of a single VkDeviceMemory are illegal.

      +

      If VmaAllocationCreateInfo::pool is not null, usage of this flag must match usage of flag VMA_POOL_CREATE_PERSISTENT_MAP_BIT used during pool creation.

      VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM 
      VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT 

      Allocation created with this flag can become lost as a result of another allocation with VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag, so you must check it before use.

      +

      To check if allocation is not lost, call vmaGetAllocationInfo() and check if VmaAllocationInfo::deviceMemory is not VK_NULL_HANDLE.

      +

      For details about supporting lost allocations, see Lost Allocations chapter of User Guide on Main Page.

      +
      VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT 

      While creating allocation using this flag, other allocations that were created with flag VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT can become lost.

      +

      For details about supporting lost allocations, see Lost Allocations chapter of User Guide on Main Page.

      +
      VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM 
      @@ -216,16 +231,16 @@ Functions - - - -
      Enumerator
      VMA_MEMORY_USAGE_UNKNOWN 

      No intended memory usage specified.

      VMA_MEMORY_USAGE_GPU_ONLY 

      Memory will be used on device only, no need to be mapped on host.

      +
      VMA_MEMORY_USAGE_GPU_ONLY 

      Memory will be used on device only, so faster access from the device is preferred. No need to be mappable on host.

      VMA_MEMORY_USAGE_CPU_ONLY 

      Memory will be mapped on host. Could be used for transfer to device.

      -

      Guarantees to be HOST_VISIBLE and HOST_COHERENT.

      +
      VMA_MEMORY_USAGE_CPU_ONLY 

      Memory will be mapped on host. Could be used for transfer to/from device.

      +

      Guarantees to be HOST_VISIBLE and HOST_COHERENT.

      VMA_MEMORY_USAGE_CPU_TO_GPU 

      Memory will be used for frequent (dynamic) updates from host and reads on device.

      -

      Guarantees to be HOST_VISIBLE.

      +
      VMA_MEMORY_USAGE_CPU_TO_GPU 

      Memory will be used for frequent (dynamic) updates from host and reads on device (upload).

      +

      Guarantees to be HOST_VISIBLE.

      VMA_MEMORY_USAGE_GPU_TO_CPU 

      Memory will be used for writing on device and readback on host.

      -

      Guarantees to be HOST_VISIBLE.

      +
      VMA_MEMORY_USAGE_GPU_TO_CPU 

      Memory will be used for frequent writing on device and readback on host (download).

      +

      Guarantees to be HOST_VISIBLE.

      VMA_MEMORY_USAGE_MAX_ENUM 
      @@ -233,8 +248,8 @@ Functions

      Function Documentation

      - -

      ◆ vmaFindMemoryTypeIndex()

      + +

      ◆ vmaFindMemoryTypeIndex()

      @@ -254,8 +269,8 @@ Functions - const VmaMemoryRequirements *  - pMemoryRequirements, + const VmaAllocationCreateInfo *  + pAllocationCreateInfo, @@ -273,9 +288,9 @@ Functions

      This algorithm tries to find a memory type that:

      • Is allowed by memoryTypeBits.
        • -
      • Contains all the flags from pMemoryRequirements->requiredFlags.
        • +
      • Contains all the flags from pAllocationCreateInfo->requiredFlags.
      • Matches intended usage.
        • -
      • Has as many flags from pMemoryRequirements->preferredFlags as possible.
        • +
      • Has as many flags from pAllocationCreateInfo->preferredFlags as possible.
      Returns
      Returns VK_ERROR_FEATURE_NOT_PRESENT if not found. Receiving such result from this function or any other allocating function probably means that your device doesn't support any memory type with requested features for the specific type of resource you want to use it for. Please check parameters of your resource, like image layout (OPTIMAL versus LINEAR) or mip level count.
      diff --git a/docs/html/group__layer2.html b/docs/html/group__layer2.html index fb24f50..271bd8f 100644 --- a/docs/html/group__layer2.html +++ b/docs/html/group__layer2.html @@ -61,6 +61,7 @@ $(function() {
      Classes | Typedefs | +Enumerations | Functions
      Layer 2 Allocating Memory
      @@ -69,8 +70,14 @@ $(function() { + + + + + + - + @@ -81,8 +88,19 @@ Classes

      Classes

      struct  VmaPoolCreateInfo
       Describes parameter of created VmaPool. More...
       
      struct  VmaPoolStats
       Describes parameter of existing VmaPool. More...
       
      struct  VmaAllocationInfo
       Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
       Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
       
      struct  VmaDefragmentationInfo
       Optional configuration parameters to be passed to function vmaDefragment(). More...
      + + + + + + + + + + + - + @@ -91,18 +109,39 @@ Typedefs

      Typedefs

      typedef enum VmaPoolCreateFlagBits VmaPoolCreateFlagBits
       Flags to be passed as VmaPoolCreateInfo::flags. More...
       
      typedef VkFlags VmaPoolCreateFlags
       
      typedef struct VmaPoolCreateInfo VmaPoolCreateInfo
       Describes parameter of created VmaPool. More...
       
      typedef struct VmaPoolStats VmaPoolStats
       Describes parameter of existing VmaPool. More...
       
      typedef struct VmaAllocationInfo VmaAllocationInfo
       Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
       Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
       
      typedef struct VmaDefragmentationInfo VmaDefragmentationInfo
       Optional configuration parameters to be passed to function vmaDefragment(). More...
       Statistics returned by function vmaDefragment(). More...
       
      + + + + +

      +Enumerations

      enum  VmaPoolCreateFlagBits { VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001, +VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002, +VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF + }
       Flags to be passed as VmaPoolCreateInfo::flags. More...
       
      - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + @@ -110,15 +149,18 @@ Functions + + + - + - + @@ -138,7 +180,7 @@ Functions

      Functions

      VkResult vmaAllocateMemory (VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaMemoryRequirements *pVmaMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       General purpose memory allocation. More...
       
      VkResult vmaAllocateMemoryForBuffer (VmaAllocator allocator, VkBuffer buffer, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       
      VkResult vmaAllocateMemoryForImage (VmaAllocator allocator, VkImage image, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       Function similar to vmaAllocateMemoryForBuffer(). More...
       
      VkResult vmaCreatePool (VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
       Allocates Vulkan device memory and creates VmaPool object. More...
       
      void vmaDestroyPool (VmaAllocator allocator, VmaPool pool)
       Destroys VmaPool object and frees Vulkan device memory. More...
       
      void vmaGetPoolStats (VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
       Retrieves statistics of existing VmaPool object. More...
       
      void vmaMakePoolAllocationsLost (VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
       Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now. More...
       
      VkResult vmaAllocateMemory (VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       General purpose memory allocation. More...
       
      VkResult vmaAllocateMemoryForBuffer (VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       
      VkResult vmaAllocateMemoryForImage (VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       Function similar to vmaAllocateMemoryForBuffer(). More...
       
      void vmaFreeMemory (VmaAllocator allocator, VmaAllocation allocation)
       Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). More...
       Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). More...
       
      void vmaGetAllocationInfo (VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
       Returns current information about specified allocation. More...
      void vmaSetAllocationUserData (VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
       Sets pUserData in given allocation to new value. More...
       
      void vmaCreateLostAllocation (VmaAllocator allocator, VmaAllocation *pAllocation)
       Creates new allocation that is in lost state from the beginning. More...
       
      VkResult vmaMapMemory (VmaAllocator allocator, VmaAllocation allocation, void **ppData)
       
      void vmaUnmapMemory (VmaAllocator allocator, VmaAllocation allocation)
       
      void vmaUnmapPersistentlyMappedMemory (VmaAllocator allocator)
       Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. More...
       Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL. More...
       
      VkResult vmaMapPersistentlyMappedMemory (VmaAllocator allocator)
       Maps back persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. More...
       Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL. More...
       
      VkResult vmaDefragment (VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
       Compacts memory by moving allocations. More...
      -

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().

      +

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().

      @@ -172,11 +214,102 @@ Functions

      Statistics returned by function vmaDefragment().

      + + + +

      ◆ VmaPoolCreateFlagBits

      + +
      +
      + + + + +
      typedef enum VmaPoolCreateFlagBits VmaPoolCreateFlagBits
      +
      + +

      Flags to be passed as VmaPoolCreateInfo::flags.

      + +
      +
      + +

      ◆ VmaPoolCreateFlags

      + +
      +
      + + + + +
      typedef VkFlags VmaPoolCreateFlags
      +
      + +
      +
      + +

      ◆ VmaPoolCreateInfo

      + +
      +
      + + + + +
      typedef struct VmaPoolCreateInfo VmaPoolCreateInfo
      +
      + +

      Describes parameter of created VmaPool.

      + +
      +
      + +

      ◆ VmaPoolStats

      + +
      +
      + + + + +
      typedef struct VmaPoolStats VmaPoolStats
      +
      + +

      Describes parameter of existing VmaPool.

      + +
      +
      +

      Enumeration Type Documentation

      + +

      ◆ VmaPoolCreateFlagBits

      + +
      +
      + + + + +
      enum VmaPoolCreateFlagBits
      +
      + +

      Flags to be passed as VmaPoolCreateInfo::flags.

      + + + + +
      Enumerator
      VMA_POOL_CREATE_PERSISTENT_MAP_BIT 

      Set this flag to use a memory that will be persistently mapped.

      +

      Each allocation made from this pool will have VmaAllocationInfo::pMappedData available.

      +

      Usage of this flag must match usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT flag for every allocation made from this pool.

      +
      VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT 

      Use this flag if you always allocate only buffers and linear images or only optimal images out of this pool and so Buffer-Image Granularity can be ignored.

      +

      This is na optional optimization flag.

      +

      If you always allocate using vmaCreateBuffer(), vmaCreateImage(), vmaAllocateMemoryForBuffer(), then you don't need to use it because allocator knows exact type of your allocations so it can handle Buffer-Image Granularity in the optimal way.

      +

      If you also allocate using vmaAllocateMemoryForImage() or vmaAllocateMemory(), exact type of such allocations is not known, so allocator must be conservative in handling Buffer-Image Granularity, which can lead to suboptimal allocation (wasted memory). In that case, if you can make sure you always allocate only buffers and linear images or only optimal images out of this pool, use this flag to make allocator disregard Buffer-Image Granularity and so make allocations more optimal.

      +
      VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM 
      +

      Function Documentation

      - -

      ◆ vmaAllocateMemory()

      + +

      ◆ vmaAllocateMemory()

      @@ -196,8 +329,8 @@ Functions - const VmaMemoryRequirements *  - pVmaMemoryRequirements, + const VmaAllocationCreateInfo *  + pCreateInfo, @@ -228,12 +361,12 @@ Functions

      You should free the memory using vmaFreeMemory().

      -

      It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(), vmaCreateBuffer(), vmaCreateImage() instead whenever possible.

      +

      It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(), vmaCreateBuffer(), vmaCreateImage() instead whenever possible.

      - -

      ◆ vmaAllocateMemoryForBuffer()

      + +

      ◆ vmaAllocateMemoryForBuffer()

      @@ -253,8 +386,8 @@ Functions - const VmaMemoryRequirements *  - pMemoryRequirements, + const VmaAllocationCreateInfo *  + pCreateInfo, @@ -286,8 +419,8 @@ Functions
      - -

      ◆ vmaAllocateMemoryForImage()

      + +

      ◆ vmaAllocateMemoryForImage()

      @@ -307,8 +440,8 @@ Functions - const VmaMemoryRequirements *  - pMemoryRequirements, + const VmaAllocationCreateInfo *  + pCreateInfo, @@ -330,7 +463,84 @@ Functions
      -

      Function similar to vmaAllocateMemoryForBuffer().

      +

      Function similar to vmaAllocateMemoryForBuffer().

      + +
      +
      + +

      ◆ vmaCreateLostAllocation()

      + +
      +
      + + + + + + + + + + + + + + + + + + +
      void vmaCreateLostAllocation (VmaAllocator allocator,
      VmaAllocation * pAllocation 
      )
      +
      + +

      Creates new allocation that is in lost state from the beginning.

      +

      It can be useful if you need a dummy, non-null allocation.

      +

      You still need to destroy created object using vmaFreeMemory().

      +

      Returned allocation is not tied to any specific memory pool or memory type and not bound to any image or buffer. It has size = 0. It cannot be turned into a real, non-empty allocation.

      + +
      +
      + +

      ◆ vmaCreatePool()

      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + +
      VkResult vmaCreatePool (VmaAllocator allocator,
      const VmaPoolCreateInfo * pCreateInfo,
      VmaPool * pPool 
      )
      +
      + +

      Allocates Vulkan device memory and creates VmaPool object.

      +
      Parameters
      + + + + +
      allocatorAllocator object.
      pCreateInfoParameters of pool to create.
      [out]pPoolHandle to created pool.
      +
      +
      @@ -396,16 +606,16 @@ Functions
      Returns
      VK_SUCCESS if completed, VK_INCOMPLETE if succeeded but didn't make all possible optimizations because limits specified in pDefragmentationInfo have been reached, negative error code in case of error.
      -

      This function works by moving allocations to different places (different VkDeviceMemory objects and/or different offsets) in order to optimize memory usage. Only allocations that are in pAllocations array can be moved. All other allocations are considered nonmovable in this call. Basic rules:

      +

      This function works by moving allocations to different places (different VkDeviceMemory objects and/or different offsets) in order to optimize memory usage. Only allocations that are in pAllocations array can be moved. All other allocations are considered nonmovable in this call. Basic rules:

        -
      • Only allocations made in memory types that have VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT flag can be compacted. You may pass other allocations but it makes no sense - these will never be moved.
        • -
      • You may pass allocations made with VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT but it makes no sense - they will never be moved.
        • -
      • Both allocations made with or without VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT flag can be compacted. If not persistently mapped, memory will be mapped temporarily inside this function if needed, so it shouldn't be mapped by you for the time of this call.
        • -
      • You must not pass same VmaAllocation object multiple times in pAllocations array.
        • +
      • Only allocations made in memory types that have VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT flag can be compacted. You may pass other allocations but it makes no sense - these will never be moved.
        • +
      • You may pass allocations made with VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT but it makes no sense - they will never be moved.
        • +
      • Both allocations made with or without VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT flag can be compacted. If not persistently mapped, memory will be mapped temporarily inside this function if needed, so it shouldn't be mapped by you for the time of this call.
        • +
      • You must not pass same VmaAllocation object multiple times in pAllocations array.
      -

      The function also frees empty VkDeviceMemory blocks.

      +

      The function also frees empty VkDeviceMemory blocks.

      After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or VmaAllocationInfo::offset changes. You must query them again using vmaGetAllocationInfo() if you need them.

      -

      If an allocation has been moved, data in memory is copied to new place automatically, but if it was bound to a buffer or an image, you must destroy that object yourself, create new one and bind it to the new memory pointed by the allocation. You must use vkDestroyBuffer(), vkDestroyImage(), vkCreateBuffer(), vkCreateImage() for that purpose and NOT vmaDestroyBuffer(), vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example:

      +

      If an allocation has been moved, data in memory is copied to new place automatically, but if it was bound to a buffer or an image, you must destroy that object yourself, create new one and bind it to the new memory pointed by the allocation. You must use vkDestroyBuffer(), vkDestroyImage(), vkCreateBuffer(), vkCreateImage() for that purpose and NOT vmaDestroyBuffer(), vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example:

      VkDevice device = ...;
 VmaAllocator allocator = ...;
 std::vector<VkBuffer> buffers = ...;
@@ -433,6 +643,36 @@ for(size_t i = 0; i < allocations.size(); ++i)
 }

      This function may be time-consuming, so you shouldn't call it too often (like every frame or after every resource creation/destruction), but rater you can call it on special occasions (like when reloading a game level, when you just destroyed a lot of objects).

      + + + +

      ◆ vmaDestroyPool()

      + +
      +
      + + + + + + + + + + + + + + + + + + +
      void vmaDestroyPool (VmaAllocator allocator,
      VmaPool pool 
      )
      +
      + +

      Destroys VmaPool object and frees Vulkan device memory.

      +
      @@ -461,7 +701,7 @@ for(size_t i = 0; i < allocations.size(); ++i)
      -

      Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().

      +

      Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().

      @@ -499,6 +739,94 @@ for(size_t i = 0; i < allocations.size(); ++i)

      Returns current information about specified allocation.

      + + + +

      ◆ vmaGetPoolStats()

      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + +
      void vmaGetPoolStats (VmaAllocator allocator,
      VmaPool pool,
      VmaPoolStats * pPoolStats 
      )
      +
      + +

      Retrieves statistics of existing VmaPool object.

      +
      Parameters
      + + + + +
      allocatorAllocator object.
      poolPool object.
      [out]pPoolStatsStatistics of specified pool.
      +
      +
      + +
      +
      + +

      ◆ vmaMakePoolAllocationsLost()

      + +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + +
      void vmaMakePoolAllocationsLost (VmaAllocator allocator,
      VmaPool pool,
      size_t * pLostAllocationCount 
      )
      +
      + +

      Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now.

      +
      Parameters
      + + + + +
      allocatorAllocator object.
      poolPool.
      [out]pLostAllocationCountNumber of allocations marked as lost. Optional - pass null if you don't need this information.
      +
      +
      +
      @@ -533,7 +861,7 @@ for(size_t i = 0; i < allocations.size(); ++i)

      Feel free to use vkMapMemory on these memory blocks on you own if you want, but just for convenience and to make sure correct offset and size is always specified, usage of vmaMapMemory() / vmaUnmapMemory() is recommended.

      -

      Do not use it on memory allocated with VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT as multiple maps to same VkDeviceMemory is illegal.

      +

      Do not use it on memory allocated with VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT as multiple maps to same VkDeviceMemory is illegal.

      @@ -553,9 +881,9 @@ for(size_t i = 0; i < allocations.size(); ++i)
      -

      Maps back persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.

      -

      See vmaUnmapPersistentlyMappedMemory().

      -

      After this call VmaAllocationInfo::pMappedData of some allocation may have value different than before calling vmaUnmapPersistentlyMappedMemory().

      +

      Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.

      +

      See vmaUnmapPersistentlyMappedMemory().

      +

      After this call VmaAllocationInfo::pMappedData of some allocation may have value different than before calling vmaUnmapPersistentlyMappedMemory().

      @@ -639,13 +967,13 @@ for(size_t i = 0; i < allocations.size(); ++i)
      -

      Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.

      -

      This is optional performance optimization. On Windows you should call it before every call to vkQueueSubmit and vkQueuePresent. After which you can remap the allocations again using vmaMapPersistentlyMappedMemory(). This is because of the internal behavior of WDDM. Example:

      +

      Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.

      +

      This is optional performance optimization. On Windows you should call it before every call to vkQueueSubmit and vkQueuePresent. After which you can remap the allocations again using vmaMapPersistentlyMappedMemory(). This is because of the internal behavior of WDDM. Example:

      vmaUnmapPersistentlyMappedMemory(allocator);
 vkQueueSubmit(...)
 vmaMapPersistentlyMappedMemory(allocator);

      After this call VmaAllocationInfo::pMappedData of some allocations may become null.

      -

      This call is reference-counted. Memory is mapped again after you call vmaMapPersistentlyMappedMemory() same number of times that you called vmaUnmapPersistentlyMappedMemory().

      +

      This call is reference-counted. Memory is mapped again after you call vmaMapPersistentlyMappedMemory() same number of times that you called vmaUnmapPersistentlyMappedMemory().

      diff --git a/docs/html/group__layer3.html b/docs/html/group__layer3.html index 9e929f6..1da6a66 100644 --- a/docs/html/group__layer3.html +++ b/docs/html/group__layer3.html @@ -67,20 +67,20 @@ $(function() { - - + + - - - + + +

      Functions

      VkResult vmaCreateBuffer (VmaAllocator allocator, const VkBufferCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       
      VkResult vmaCreateBuffer (VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       
      void vmaDestroyBuffer (VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
       
      VkResult vmaCreateImage (VmaAllocator allocator, const VkImageCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       Function similar to vmaCreateBuffer(). More...
       
      VkResult vmaCreateImage (VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
       Function similar to vmaCreateBuffer(). More...
       
      void vmaDestroyImage (VmaAllocator allocator, VkImage image, VmaAllocation allocation)
       

      Detailed Description

      Function Documentation

      - -

      ◆ vmaCreateBuffer()

      + +

      ◆ vmaCreateBuffer()

      @@ -95,13 +95,13 @@ Functions const VkBufferCreateInfo *  - pCreateInfo, + pBufferCreateInfo, - const VmaMemoryRequirements *  - pMemoryRequirements, + const VmaAllocationCreateInfo *  + pAllocationCreateInfo, @@ -147,8 +147,8 @@ Functions
      - -

      ◆ vmaCreateImage()

      + +

      ◆ vmaCreateImage()

      @@ -163,13 +163,13 @@ Functions const VkImageCreateInfo *  - pCreateInfo, + pImageCreateInfo, - const VmaMemoryRequirements *  - pMemoryRequirements, + const VmaAllocationCreateInfo *  + pAllocationCreateInfo, @@ -197,7 +197,7 @@ Functions
      -

      Function similar to vmaCreateBuffer().

      +

      Function similar to vmaCreateBuffer().

      diff --git a/docs/html/index.html b/docs/html/index.html index 20a1a2b..9e8b311 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -62,15 +62,32 @@ $(function() {
      Vulkan Memory Allocator
      -

      Version 2.0.0-alpha.2 (2017-07-11)

      +

      Table of Contents

      + +
      +

      Version 2.0.0-alpha.3 (2017-09-12)

      Members grouped: see Modules.

      All members: see vk_mem_alloc.h.

      -Problem Statement

      +Problem statement

      Memory allocation and resource (buffer and image) creation in Vulkan is difficult (comparing to older graphics API-s, like D3D11 or OpenGL) for several reasons:

      • It requires a lot of boilerplate code, just like everything else in Vulkan, because it is a low-level and high-performance API.
        • -
      • There is additional level of indirection: VkDeviceMemory is allocated separately from creating VkBuffer/VkImage and they must be bound together. The binding cannot be changed later - resource must be recreated.
        • +
      • There is additional level of indirection: VkDeviceMemory is allocated separately from creating VkBuffer/VkImage and they must be bound together. The binding cannot be changed later - resource must be recreated.
      • Driver must be queried for supported memory heaps and memory types. Different IHV-s provide different types of it.
      • It is recommended practice to allocate bigger chunks of memory and assign parts of them to particular resources.
      @@ -82,7 +99,7 @@ Features
    • Required or preferred traits of the memory are expressed using higher-level description comparing to Vulkan flags.
    • -
  • Functions that allocate memory blocks, reserve and return parts of them (VkDeviceMemory + offset + size) to the user.
      +
    • Functions that allocate memory blocks, reserve and return parts of them (VkDeviceMemory + offset + size) to the user.
      • Library keeps track of allocated memory blocks, used and unused ranges inside them, finds best matching unused ranges for new allocations, takes all the rules of alignment into consideration.
      • @@ -95,10 +112,12 @@ Prequisites
    • Public interface in C, in same convention as Vulkan API. Implementation in C++.
    • Interface documented using Doxygen-style comments.
    • Platform-independent, but developed and tested on Windows using Visual Studio.
      • -
    • Error handling implemented by returning VkResult error codes - same way as in Vulkan.
      • +
    • Error handling implemented by returning VkResult error codes - same way as in Vulkan.
    -

    -Quick Start

    +

    +User guide

    +

    +Quick start

    In your project code:

    1. Include "vk_mem_alloc.h" file wherever you want to use the library.
      2. @@ -108,8 +127,8 @@ Quick Start #include "vk_mem_alloc.h"

      At program startup:

        -
      1. Initialize Vulkan to have VkPhysicalDevice and VkDevice object.
        2. -
      2. Fill VmaAllocatorCreateInfo structure and create VmaAllocator object by calling vmaCreateAllocator().
        3. +
      3. Initialize Vulkan to have VkPhysicalDevice and VkDevice object.
        4. +
      4. Fill VmaAllocatorCreateInfo structure and create VmaAllocator object by calling vmaCreateAllocator().
      VmaAllocatorCreateInfo allocatorInfo = {};
 allocatorInfo.physicalDevice = physicalDevice;
@@ -119,36 +138,174 @@ VmaAllocator allocator;
 vmaCreateAllocator(&allocatorInfo, &allocator);

      When you want to create a buffer or image:

        -
      1. Fill VkBufferCreateInfo / VkImageCreateInfo structure.
        2. -
      2. Fill VmaMemoryRequirements structure.
        3. -
      3. Call vmaCreateBuffer() / vmaCreateImage() to get VkBuffer/VkImage with memory already allocated and bound to it.
        4. +
      4. Fill VkBufferCreateInfo / VkImageCreateInfo structure.
        5. +
      5. Fill VmaAllocationCreateInfo structure.
        6. +
      6. Call vmaCreateBuffer() / vmaCreateImage() to get VkBuffer/VkImage with memory already allocated and bound to it.
      VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
 bufferInfo.size = myBufferSize;
 bufferInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
 
-VmaMemoryRequirements memReq = {};
+VmaAllocationCreateInfo memReq = {};
 memReq.usage = VMA_MEMORY_USAGE_GPU_ONLY;
 
 VkBuffer buffer;
 VmaAllocation allocation;
 vmaCreateBuffer(allocator, &bufferInfo, &memReq, &buffer, &allocation, nullptr);
-

      Don't forget to destroy your objects:

      +

      Don't forget to destroy your objects when no longer needed:

      vmaDestroyBuffer(allocator, buffer, allocation);
 vmaDestroyAllocator(allocator);
-

      +

      +Persistently mapped memory

      +

      If you need to map memory on host, it may happen that two allocations are assigned to the same VkDeviceMemory block, so if you map them both at the same time, it will cause error because mapping single memory block multiple times is illegal in Vulkan.

      +

      It is safer, more convenient and more efficient to use special feature designed for that: persistently mapped memory. Allocations made with VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT flag set in VmaAllocationCreateInfo::flags are returned from device memory blocks that stay mapped all the time, so you can just access CPU pointer to it. VmaAllocationInfo::pMappedData pointer is already offseted to the beginning of particular allocation. Example:

      +
      VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+bufCreateInfo.size = 1024;
+bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
+
+VmaAllocationCreateInfo allocCreateInfo = {};
+allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY;
+allocCreateInfo.flags = VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT;
+
+VkBuffer buf;
+VmaAllocation alloc;
+VmaAllocationInfo allocInfo;
+vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
+
+.// Buffer is immediately mapped. You can access its memory.
+memcpy(allocInfo.pMappedData, myData, 1024);
+

      Memory in Vulkan doesn't need to be unmapped before using it e.g. for transfers, but if you are not sure whether it's HOST_COHERENT (here is surely is because it's created with VMA_MEMORY_USAGE_CPU_ONLY), you should check it. If it's not, you should call vkInvalidateMappedMemoryRanges() before reading and vkFlushMappedMemoryRanges() after writing to mapped memory on CPU. Example:

      +
      VkMemoryPropertyFlags memFlags;
+vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags);
+if((memFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0)
+{
+    VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
+    memRange.memory = allocInfo.deviceMemory;
+    memRange.offset = allocInfo.offset;
+    memRange.size   = allocInfo.size;
+    vkFlushMappedMemoryRanges(device, 1, &memRange);
+}
+

      For performance reasons it is also recommended to unmap Vulkan memory for the time of call to vkQueueSubmit() or vkQueuePresent(). You can do it for all persistently mapped memory using just one function call. For details, see function vmaUnmapPersistentlyMappedMemory(), vmaMapPersistentlyMappedMemory().

      +

      +Custom memory pools

      +

      The library automatically creates and manages default memory pool for each memory type available on the device. A pool contains a number of VkDeviceMemory blocks. You can create custom pool and allocate memory out of it. It can be useful if you want to:

      +
        +
      • Keep certain kind of allocations separate from others.
        • +
      • Enforce particular size of Vulkan memory blocks.
        • +
      • Limit maximum amount of Vulkan memory allocated for that pool.
        • +
      +

      To use custom memory pools:

      +
        +
      1. Fill VmaPoolCreateInfo structure.
        2. +
      2. Call vmaCreatePool() to obtain VmaPool handle.
        3. +
      3. When making an allocation, set VmaAllocationCreateInfo::pool to this handle. You don't need to specify any other parameters of this structure, like usage.
        4. +
      +

      Example:

      +
      .// Create a pool that could have at most 2 blocks, 128 MB each.
+VmaPoolCreateInfo poolCreateInfo = {};
+poolCreateInfo.memoryTypeIndex = ...
+poolCreateInfo.blockSize = 128ull * 1024 * 1024;
+poolCreateInfo.maxBlockCount = 2;
+
+VmaPool pool;
+vmaCreatePool(allocator, &poolCreateInfo, &pool);
+
+.// Allocate a buffer out of it.
+VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+bufCreateInfo.size = 1024;
+bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+
+VmaAllocationCreateInfo allocCreateInfo = {};
+allocCreateInfo.pool = pool;
+
+VkBuffer buf;
+VmaAllocation alloc;
+VmaAllocationInfo allocInfo;
+vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
+

      You have to free all allocations made from this pool before destroying it.

      +
      vmaDestroyBuffer(allocator, buf, alloc);
+vmaDestroyPool(allocator, pool);
+

      +Defragmentation

      +

      Interleaved allocations and deallocations of many objects of varying size can cause fragmentation, which can lead to a situation where the library is unable to find a continuous range of free memory for a new allocation despite there is enough free space, just scattered across many small free ranges between existing allocations.

      +

      To mitigate this problem, you can use vmaDefragment(). Given set of allocations, this function can move them to compact used memory, ensure more continuous free space and possibly also free some VkDeviceMemory. It can work only on allocations made from memory type that is HOST_VISIBLE. Allocations are modified to point to the new VkDeviceMemory and offset. Data in this memory is also memmove-ed to the new place. However, if you have images or buffers bound to these allocations (and you certainly do), you need to destroy, recreate, and bind them to the new place in memory.

      +

      For further details and example code, see documentation of function vmaDefragment().

      +

      +Lost allocations

      +

      If your game oversubscribes video memory, if may work OK in previous-generation graphics APIs (DirectX 9, 10, 11, OpenGL) because resources are automatically paged to system RAM. In Vulkan you can't do it because when you run out of memory, an allocation just fails. If you have more data (e.g. textures) that can fit into VRAM and you don't need it all at once, you may want to upload them to GPU on demand and "push out" ones that are not used for a long time to make room for the new ones, effectively using VRAM (or a cartain memory pool) as a form of cache. Vulkan Memory Allocator can help you with that by supporting a concept of "lost allocations".

      +

      To create an allocation that can become lost, include VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag in VmaAllocationCreateInfo::flags. Before using a buffer or image bound to such allocation in every new frame, you need to query it if it's not lost. To check it: call vmaGetAllocationInfo() and see if VmaAllocationInfo::deviceMemory is not VK_NULL_HANDLE. If the allocation is lost, you should not use it or buffer/image bound to it. You mustn't forget to destroy this allocation and this buffer/image.

      +

      To create an allocation that can make some other allocations lost to make room for it, use VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag. You will usually use both flags VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT and VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT at the same time.

      +

      Warning! Current implementation uses quite naive, brute force algorithm, which can make allocation calls that use VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag quite slow. A new, more optimal algorithm and data structure to speed this up is planned for the future.

      +

      When interleaving creation of new allocations with usage of existing ones, how do you make sure that an allocation won't become lost while it's used in the current frame?

      +

      It is ensured because vmaGetAllocationInfo() not only returns allocation parameters and checks whether it's not lost, but when it's not, it also atomically marks it as used in the current frame, which makes it impossible to become lost in that frame. It uses lockless algorithm, so it works fast and doesn't involve locking any internal mutex.

      +

      What if my allocation may still be in use by the GPU when it's rendering a previous frame while I already submit new frame on the CPU?

      +

      You can make sure that allocations "touched" by vmaGetAllocationInfo() will not become lost for a number of additional frames back from the current one by specifying this number as VmaAllocatorCreateInfo::frameInUseCount (for default memory pool) and VmaPoolCreateInfo::frameInUseCount (for custom pool).

      +

      How do you inform the library when new frame starts?

      +

      You need to call function vmaSetCurrentFrameIndex().

      +

      Example code:

      +
      struct MyBuffer
+{
+    VkBuffer m_Buf = nullptr;
+    VmaAllocation m_Alloc = nullptr;
+
+    .// Called when the buffer is really needed in the current frame.
+    void EnsureBuffer();
+};
+
+void MyBuffer::EnsureBuffer()
+{
+    .// Buffer has been created.
+    if(m_Buf != VK_NULL_HANDLE)
+    {
+        .// Check if its allocation is not lost + mark it as used in current frame.
+        VmaAllocationInfo allocInfo;
+        vmaGetAllocationInfo(allocator, m_Alloc, &allocInfo);
+        if(allocInfo.deviceMemory != VK_NULL_HANDLE)
+        {
+            .// It's all OK - safe to use m_Buf.
+            return;
+        }
+    }
+
+    .// Buffer not yet exists or lost - destroy and recreate it.
+
+    vmaDestroyBuffer(allocator, m_Buf, m_Alloc);
+
+    VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+    bufCreateInfo.size = 1024;
+    bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+
+    VmaAllocationCreateInfo allocCreateInfo = {};
+    allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
+    allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT |
+        VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
+
+    vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr);
+}
+

      When using lost allocations, you may see some Vulkan validation layer warnings about overlapping regions of memory bound to different kinds of buffers and images. This is still valid as long as you implement proper handling of lost allocations (like in the example above) and don't use them.

      +

      The library uses following algorithm for allocation, in order:

      +
        +
      1. Try to find free range of memory in existing blocks.
        2. +
      2. If failed, try to create a new block of VkDeviceMemory, with preferred block size.
        3. +
      3. If failed, try to create such block with size/2 and size/4.
        4. +
      4. If failed and VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag was specified, try to find space in existing blocks, possilby making some other allocations lost.
        5. +
      5. If failed, try to allocate separate VkDeviceMemory for this allocation, just like when you use VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT.
        6. +
      6. If failed, choose other memory type that meets the requirements specified in VmaAllocationCreateInfo and go to point 1.
        7. +
      7. If failed, return VK_ERROR_OUT_OF_DEVICE_MEMORY.
        8. +
      +

      Configuration

      -

      Please check "CONFIGURATION SECTION" in the code to find macros that you can define before each #include of this file or change directly in this file to provide your own implementation of basic facilities like assert, min and max functions, mutex etc. C++ STL is used by default, but changing these allows you to get rid of any STL usage if you want, as many game developers tend to do.

      -

      -Custom memory allocator

      -

      You can use custom memory allocator by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These functions will be passed to Vulkan, as well as used by the library itself to make any CPU-side allocations.

      +

      Please check "CONFIGURATION SECTION" in the code to find macros that you can define before each #include of this file or change directly in this file to provide your own implementation of basic facilities like assert, min and max functions, mutex etc. C++ STL is used by default, but changing these allows you to get rid of any STL usage if you want, as many game developers tend to do.

      +

      +Custom memory allocator

      +

      You can use custom memory allocator by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These functions will be passed to Vulkan, as well as used by the library itself to make any CPU-side allocations.

      Thread safety

        -
      • The library has no global state, so separate VmaAllocator objects can be used independently.
        • -
      • By default, all calls to functions that take VmaAllocator as first parameter are safe to call from multiple threads simultaneously because they are synchronized internally when needed.
        • -
      • When the allocator is created with VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT flag, calls to functions that take such VmaAllocator object must be synchronized externally.
        • -
      • Access to a VmaAllocation object must be externally synchronized. For example, you must not call vmaGetAllocationInfo() and vmaDefragment() from different threads at the same time if you pass the same VmaAllocation object to these functions.
        • +
      • The library has no global state, so separate VmaAllocator objects can be used independently.
        • +
      • By default, all calls to functions that take VmaAllocator as first parameter are safe to call from multiple threads simultaneously because they are synchronized internally when needed.
        • +
      • When the allocator is created with VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT flag, calls to functions that take such VmaAllocator object must be synchronized externally.
        • +
      • Access to a VmaAllocation object must be externally synchronized. For example, you must not call vmaGetAllocationInfo() and vmaDefragment() from different threads at the same time if you pass the same VmaAllocation object to these functions.
      diff --git a/docs/html/search/all_0.js b/docs/html/search/all_0.js index d65975f..36e26b6 100644 --- a/docs/html/search/all_0.js +++ b/docs/html/search/all_0.js @@ -1,5 +1,8 @@ var searchData= [ - ['allocationcount',['AllocationCount',['../struct_vma_stat_info.html#a240402222ac6777e4079653c5d542cb0',1,'VmaStatInfo']]], + ['allocationcount',['AllocationCount',['../struct_vma_stat_info.html#a240402222ac6777e4079653c5d542cb0',1,'VmaStatInfo::AllocationCount()'],['../struct_vma_pool_stats.html#ad1924eb54fffa45e9e0e65670c8fe5eb',1,'VmaPoolStats::allocationCount()']]], + ['allocationsizeavg',['AllocationSizeAvg',['../struct_vma_stat_info.html#a0019702e4f4ea4ed824aebf250841f21',1,'VmaStatInfo']]], + ['allocationsizemax',['AllocationSizeMax',['../struct_vma_stat_info.html#aba2ef16e2ac09b97305c58a168a3805c',1,'VmaStatInfo']]], + ['allocationsizemin',['AllocationSizeMin',['../struct_vma_stat_info.html#a61a834fa78bcf41fb5a53b286b2e1580',1,'VmaStatInfo']]], ['allocationsmoved',['allocationsMoved',['../struct_vma_defragmentation_stats.html#aefeabf130022008eadd75999478af3f9',1,'VmaDefragmentationStats']]] ]; diff --git a/docs/html/search/all_1.js b/docs/html/search/all_1.js index cb221fc..e73151c 100644 --- a/docs/html/search/all_1.js +++ b/docs/html/search/all_1.js @@ -1,5 +1,7 @@ var searchData= [ + ['blockcount',['BlockCount',['../struct_vma_stat_info.html#a9af0d9809007cf4a0009d324b2b8bbc9',1,'VmaStatInfo']]], + ['blocksize',['blockSize',['../struct_vma_pool_create_info.html#aa4265160536cdb9be821b7686c16c676',1,'VmaPoolCreateInfo']]], ['bytesfreed',['bytesFreed',['../struct_vma_defragmentation_stats.html#ab0cb9ac0dbc106c77e384ea676422f28',1,'VmaDefragmentationStats']]], ['bytesmoved',['bytesMoved',['../struct_vma_defragmentation_stats.html#a36f9d5df2a10ba2a36b16e126d60572d',1,'VmaDefragmentationStats']]] ]; diff --git a/docs/html/search/all_3.js b/docs/html/search/all_3.js index 60408e6..06fd7c5 100644 --- a/docs/html/search/all_3.js +++ b/docs/html/search/all_3.js @@ -1,4 +1,5 @@ var searchData= [ - ['flags',['flags',['../struct_vma_allocator_create_info.html#a35d3b42ac5a3be3758926819dc053859',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_memory_requirements.html#a53c9ea8c3cbcc12dddbe9ae23bb85eb6',1,'VmaMemoryRequirements::flags()']]] + ['flags',['flags',['../struct_vma_allocator_create_info.html#a35d3b42ac5a3be3758926819dc053859',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], + ['frameinusecount',['frameInUseCount',['../struct_vma_allocator_create_info.html#a21ea188dd212b8171cb9ecbed4a2a3a7',1,'VmaAllocatorCreateInfo::frameInUseCount()'],['../struct_vma_pool_create_info.html#a9437e43ffbb644dbbf7fc4e50cfad6aa',1,'VmaPoolCreateInfo::frameInUseCount()']]] ]; diff --git a/docs/html/search/all_6.js b/docs/html/search/all_6.js index b23ec17..1002ad2 100644 --- a/docs/html/search/all_6.js +++ b/docs/html/search/all_6.js @@ -1,7 +1,10 @@ var searchData= [ ['maxallocationstomove',['maxAllocationsToMove',['../struct_vma_defragmentation_info.html#aa7c7304e13c71f604c907196c4e28fbc',1,'VmaDefragmentationInfo']]], + ['maxblockcount',['maxBlockCount',['../struct_vma_pool_create_info.html#ae41142f2834fcdc82baa4883c187b75c',1,'VmaPoolCreateInfo']]], ['maxbytestomove',['maxBytesToMove',['../struct_vma_defragmentation_info.html#acb311c940a777270e67e1b81c5ab6a1d',1,'VmaDefragmentationInfo']]], ['memoryheap',['memoryHeap',['../struct_vma_stats.html#a0e6611508c29a187f0fd14ff1a0329c0',1,'VmaStats']]], - ['memorytype',['memoryType',['../struct_vma_stats.html#a13e3caf754be79352c42408756309331',1,'VmaStats::memoryType()'],['../struct_vma_allocation_info.html#a7f6b0aa58c135e488e6b40a388dad9d5',1,'VmaAllocationInfo::memoryType()']]] + ['memorytype',['memoryType',['../struct_vma_stats.html#a13e3caf754be79352c42408756309331',1,'VmaStats::memoryType()'],['../struct_vma_allocation_info.html#a7f6b0aa58c135e488e6b40a388dad9d5',1,'VmaAllocationInfo::memoryType()']]], + ['memorytypeindex',['memoryTypeIndex',['../struct_vma_pool_create_info.html#a596fa76b685d3f1f688f84a709a5b319',1,'VmaPoolCreateInfo']]], + ['minblockcount',['minBlockCount',['../struct_vma_pool_create_info.html#ad8006fb803185c0a699d30f3e9a865ae',1,'VmaPoolCreateInfo']]] ]; diff --git a/docs/html/search/all_8.js b/docs/html/search/all_8.js index 4093397..48da8dd 100644 --- a/docs/html/search/all_8.js +++ b/docs/html/search/all_8.js @@ -8,8 +8,9 @@ var searchData= ['pfnfree',['pfnFree',['../struct_vma_device_memory_callbacks.html#abe8a3328bbc916f6f712fdb6b299444c',1,'VmaDeviceMemoryCallbacks']]], ['physicaldevice',['physicalDevice',['../struct_vma_allocator_create_info.html#a08230f04ae6ccf8a78150a9e829a7156',1,'VmaAllocatorCreateInfo']]], ['pmappeddata',['pMappedData',['../struct_vma_allocation_info.html#a5eeffbe2d2f30f53370ff14aefbadbe2',1,'VmaAllocationInfo']]], - ['preferredflags',['preferredFlags',['../struct_vma_memory_requirements.html#a6e105f836c2288034c711815b18226dc',1,'VmaMemoryRequirements']]], + ['pool',['pool',['../struct_vma_allocation_create_info.html#a6272c0555cfd1fe28bff1afeb6190150',1,'VmaAllocationCreateInfo']]], + ['preferredflags',['preferredFlags',['../struct_vma_allocation_create_info.html#a7fe8d81a1ad10b2a2faacacee5b15d6d',1,'VmaAllocationCreateInfo']]], ['preferredlargeheapblocksize',['preferredLargeHeapBlockSize',['../struct_vma_allocator_create_info.html#a8e4714298e3121cdd8b214a1ae7a637a',1,'VmaAllocatorCreateInfo']]], ['preferredsmallheapblocksize',['preferredSmallHeapBlockSize',['../struct_vma_allocator_create_info.html#ab435423d84d5ab26e2c347c51771f90a',1,'VmaAllocatorCreateInfo']]], - ['puserdata',['pUserData',['../struct_vma_memory_requirements.html#a8470093e93ed07ed2557490cdc67566a',1,'VmaMemoryRequirements::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]] + ['puserdata',['pUserData',['../struct_vma_allocation_create_info.html#a8259e85c272683434f4abb4ddddffe19',1,'VmaAllocationCreateInfo::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]] ]; diff --git a/docs/html/search/all_9.js b/docs/html/search/all_9.js index 0a576b4..9505699 100644 --- a/docs/html/search/all_9.js +++ b/docs/html/search/all_9.js @@ -1,4 +1,4 @@ var searchData= [ - ['requiredflags',['requiredFlags',['../struct_vma_memory_requirements.html#a8876c1b0f112e13a277f16967064cfe0',1,'VmaMemoryRequirements']]] + ['requiredflags',['requiredFlags',['../struct_vma_allocation_create_info.html#a9166390303ff42d783305bc31c2b6b90',1,'VmaAllocationCreateInfo']]] ]; diff --git a/docs/html/search/all_a.js b/docs/html/search/all_a.js index 5261aad..740c955 100644 --- a/docs/html/search/all_a.js +++ b/docs/html/search/all_a.js @@ -1,8 +1,4 @@ var searchData= [ - ['size',['size',['../struct_vma_allocation_info.html#aac76d113a6a5ccbb09fea00fb25fd18f',1,'VmaAllocationInfo']]], - ['suballocationcount',['SuballocationCount',['../struct_vma_stat_info.html#a09fb04b5491661c2e838d098d51bcead',1,'VmaStatInfo']]], - ['suballocationsizeavg',['SuballocationSizeAvg',['../struct_vma_stat_info.html#abb6c3e160a136787f474a18a8264d83b',1,'VmaStatInfo']]], - ['suballocationsizemax',['SuballocationSizeMax',['../struct_vma_stat_info.html#a6be7faf2b7fcff5a9bc017d90aed9271',1,'VmaStatInfo']]], - ['suballocationsizemin',['SuballocationSizeMin',['../struct_vma_stat_info.html#a9dc0b50fab2f10ab99366b79424bf14b',1,'VmaStatInfo']]] + ['size',['size',['../struct_vma_pool_stats.html#a326807b2de2b0931cee4ed9a5f2e420c',1,'VmaPoolStats::size()'],['../struct_vma_allocation_info.html#aac76d113a6a5ccbb09fea00fb25fd18f',1,'VmaAllocationInfo::size()']]] ]; diff --git a/docs/html/search/all_c.js b/docs/html/search/all_c.js index cc1a3e7..91686af 100644 --- a/docs/html/search/all_c.js +++ b/docs/html/search/all_c.js @@ -1,10 +1,11 @@ var searchData= [ ['unusedbytes',['UnusedBytes',['../struct_vma_stat_info.html#a394d2aef4348cb58abf73764804b4f2d',1,'VmaStatInfo']]], - ['unusedrangecount',['UnusedRangeCount',['../struct_vma_stat_info.html#a56c4fb4dba646479180b601854cde2a6',1,'VmaStatInfo']]], + ['unusedrangecount',['UnusedRangeCount',['../struct_vma_stat_info.html#a56c4fb4dba646479180b601854cde2a6',1,'VmaStatInfo::UnusedRangeCount()'],['../struct_vma_pool_stats.html#ae4f3546ffa4d1e598b64d8e6134854f4',1,'VmaPoolStats::unusedRangeCount()']]], ['unusedrangesizeavg',['UnusedRangeSizeAvg',['../struct_vma_stat_info.html#a88ad9bdc2b3a98964a4d0c338c0c9060',1,'VmaStatInfo']]], ['unusedrangesizemax',['UnusedRangeSizeMax',['../struct_vma_stat_info.html#a10c52c0841f01ca704e8ddb1ea6a635d',1,'VmaStatInfo']]], ['unusedrangesizemin',['UnusedRangeSizeMin',['../struct_vma_stat_info.html#a07c508f42a4d3424bd0c259784a7f2d6',1,'VmaStatInfo']]], - ['usage',['usage',['../struct_vma_memory_requirements.html#ab588497177a57847ed04e0a1aef54bbe',1,'VmaMemoryRequirements']]], + ['unusedsize',['unusedSize',['../struct_vma_pool_stats.html#ad7c54874724fce7b06aba526202d82a8',1,'VmaPoolStats']]], + ['usage',['usage',['../struct_vma_allocation_create_info.html#accb8b06b1f677d858cb9af20705fa910',1,'VmaAllocationCreateInfo']]], ['usedbytes',['UsedBytes',['../struct_vma_stat_info.html#a86f82cb9cffd456b9da63eaf26c9ff04',1,'VmaStatInfo']]] ]; diff --git a/docs/html/search/all_d.js b/docs/html/search/all_d.js index dbd1fed..fb398e6 100644 --- a/docs/html/search/all_d.js +++ b/docs/html/search/all_d.js @@ -2,22 +2,30 @@ var searchData= [ ['vulkan_20memory_20allocator',['Vulkan Memory Allocator',['../index.html',1,'']]], ['vk_5fmem_5falloc_2eh',['vk_mem_alloc.h',['../vk__mem__alloc_8h.html',1,'']]], + ['vma_5fallocation_5fcreate_5fcan_5fbecome_5flost_5fbit',['VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a5f436af6c8fe8540573a6d22627a6fd2',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fcan_5fmake_5fother_5flost_5fbit',['VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a68686d0ce9beb0d4d1b9f2b8b1389a7e',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597ae5633ec569f4899cf8f29e7385b2f882',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fnever_5fallocate_5fbit',['VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a89759603401014eb325eb22a3839f2ff',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fown_5fmemory_5fbit',['VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a965e035b69f8728b317803ef2d523aa4',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fpersistent_5fmap_5fbit',['VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597ae443691ef3d077c0dc3de5576ac4c312',1,'vk_mem_alloc.h']]], ['vma_5fallocator_5fexternally_5fsynchronized_5fbit',['VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT',['../group__general.html#gga34fff29c218d23a7ff7dff44b77b6b6fabe92b706180652ceb320da5bc383aef4',1,'vk_mem_alloc.h']]], ['vma_5fallocator_5fflag_5fbits_5fmax_5fenum',['VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM',['../group__general.html#gga34fff29c218d23a7ff7dff44b77b6b6fa914e905a08c2e578f76b9d6c418626cc',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fflag_5fbits_5fmax_5fenum',['VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619eda11f849416519c61bebd90e2e7ecbab21',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fnever_5fallocate_5fbit',['VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619edae1a503c29619c0c9885521d371e9b4c4',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fown_5fmemory_5fbit',['VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619eda9831d98a698347b0cd1594db6a29e8ed',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fpersistent_5fmap_5fbit',['VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619edae75c9dde227c9d4f64693b1e004106dc',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fcpu_5fonly',['VMA_MEMORY_USAGE_CPU_ONLY',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca40bdf4cddeffeb12f43d45ca1286e0a5',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fcpu_5fto_5fgpu',['VMA_MEMORY_USAGE_CPU_TO_GPU',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca9066b52c5a7079bb74a69aaf8b92ff67',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fgpu_5fonly',['VMA_MEMORY_USAGE_GPU_ONLY',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305ccac6b5dc1432d88647aa4cd456246eadf7',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fgpu_5fto_5fcpu',['VMA_MEMORY_USAGE_GPU_TO_CPU',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca7b586d2fdaf82a463b58f581ed72be27',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fmax_5fenum',['VMA_MEMORY_USAGE_MAX_ENUM',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca091e69437ef693e8d0d287f1c719ba6e',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5funknown',['VMA_MEMORY_USAGE_UNKNOWN',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305ccaf50d27e34e0925cf3a63db8c839121dd',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a1c7312bea9ea246846b9054fd6bd6aec',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fignore_5fbuffer_5fimage_5fgranularity_5fbit',['VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a9f1a499508a8edb4e8ba40aa0290a3d2',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fpersistent_5fmap_5fbit',['VMA_POOL_CREATE_PERSISTENT_MAP_BIT',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a918441f7b40dca90481b114f5d224fe9',1,'vk_mem_alloc.h']]], ['vma_5fstats_5fstring_5fenabled',['VMA_STATS_STRING_ENABLED',['../group__general.html#gae25f0d55fd91cb166f002b63244800e1',1,'vk_mem_alloc.h']]], - ['vmaallocatememory',['vmaAllocateMemory',['../group__layer2.html#ga1a7d45920877a53e9a8fbefd6a536119',1,'vk_mem_alloc.h']]], - ['vmaallocatememoryforbuffer',['vmaAllocateMemoryForBuffer',['../group__layer2.html#ga3536b4da2a2744dc3f3f138be555af02',1,'vk_mem_alloc.h']]], - ['vmaallocatememoryforimage',['vmaAllocateMemoryForImage',['../group__layer2.html#ga801f98c24eb928642742bb1e5615e86e',1,'vk_mem_alloc.h']]], + ['vmaallocatememory',['vmaAllocateMemory',['../group__layer2.html#gabf28077dbf82d0908b8acbe8ee8dd9b8',1,'vk_mem_alloc.h']]], + ['vmaallocatememoryforbuffer',['vmaAllocateMemoryForBuffer',['../group__layer2.html#ga7fdf64415b6c3d83c454f28d2c53df7b',1,'vk_mem_alloc.h']]], + ['vmaallocatememoryforimage',['vmaAllocateMemoryForImage',['../group__layer2.html#ga0faa3f9e5fb233d29d1e00390650febb',1,'vk_mem_alloc.h']]], + ['vmaallocationcreateflagbits',['VmaAllocationCreateFlagBits',['../group__layer1.html#gad9889c10c798b040d59c92f257cae597',1,'VmaAllocationCreateFlagBits(): vk_mem_alloc.h'],['../group__layer1.html#gabf6bf6748c7a9fe7ce5b7835c0f56af4',1,'VmaAllocationCreateFlagBits(): vk_mem_alloc.h']]], + ['vmaallocationcreateflags',['VmaAllocationCreateFlags',['../group__layer1.html#ga5225e5e11f8376f6a31a1791f3d6e817',1,'vk_mem_alloc.h']]], + ['vmaallocationcreateinfo',['VmaAllocationCreateInfo',['../struct_vma_allocation_create_info.html',1,'VmaAllocationCreateInfo'],['../group__layer1.html#ga86c44f9950b40d50088ed93a17870a7a',1,'VmaAllocationCreateInfo(): vk_mem_alloc.h']]], ['vmaallocationinfo',['VmaAllocationInfo',['../struct_vma_allocation_info.html',1,'VmaAllocationInfo'],['../group__layer2.html#ga795e6ff02a21d5486c0565f403dd9255',1,'VmaAllocationInfo(): vk_mem_alloc.h']]], ['vmaallocatorcreateinfo',['VmaAllocatorCreateInfo',['../struct_vma_allocator_create_info.html',1,'VmaAllocatorCreateInfo'],['../group__general.html#gae0f6d1d733dded220d28134da46b4283',1,'VmaAllocatorCreateInfo(): vk_mem_alloc.h']]], ['vmaallocatorflagbits',['VmaAllocatorFlagBits',['../group__general.html#ga34fff29c218d23a7ff7dff44b77b6b6f',1,'VmaAllocatorFlagBits(): vk_mem_alloc.h'],['../group__general.html#gac51ae2c316ee760ec5aebbae06d31253',1,'VmaAllocatorFlagBits(): vk_mem_alloc.h']]], @@ -25,29 +33,36 @@ var searchData= ['vmabuildstatsstring',['vmaBuildStatsString',['../group__general.html#gaa4fee7eb5253377599ef4fd38c93c2a0',1,'vk_mem_alloc.h']]], ['vmacalculatestats',['vmaCalculateStats',['../group__general.html#ga333b61c1788cb23559177531e6a93ca3',1,'vk_mem_alloc.h']]], ['vmacreateallocator',['vmaCreateAllocator',['../group__general.html#ga200692051ddb34240248234f5f4c17bb',1,'vk_mem_alloc.h']]], - ['vmacreatebuffer',['vmaCreateBuffer',['../group__layer3.html#ga2f711e32e95cf9bf8dff4917230c2e9b',1,'vk_mem_alloc.h']]], - ['vmacreateimage',['vmaCreateImage',['../group__layer3.html#ga9e34bc318ff4b25d1958e79b9db3f1aa',1,'vk_mem_alloc.h']]], + ['vmacreatebuffer',['vmaCreateBuffer',['../group__layer3.html#gac72ee55598617e8eecca384e746bab51',1,'vk_mem_alloc.h']]], + ['vmacreateimage',['vmaCreateImage',['../group__layer3.html#ga02a94f25679275851a53e82eacbcfc73',1,'vk_mem_alloc.h']]], + ['vmacreatelostallocation',['vmaCreateLostAllocation',['../group__layer2.html#gae5c9657d9e94756269145b01c05d16f1',1,'vk_mem_alloc.h']]], + ['vmacreatepool',['vmaCreatePool',['../group__layer2.html#ga5c8770ded7c59c8caac6de0c2cb00b50',1,'vk_mem_alloc.h']]], ['vmadefragment',['vmaDefragment',['../group__layer2.html#ga6aced90fcc7b39882b6654a740a0b9bb',1,'vk_mem_alloc.h']]], ['vmadefragmentationinfo',['VmaDefragmentationInfo',['../struct_vma_defragmentation_info.html',1,'VmaDefragmentationInfo'],['../group__layer2.html#gae67f8573a0cf20f16f0a1eecbca566a0',1,'VmaDefragmentationInfo(): vk_mem_alloc.h']]], ['vmadefragmentationstats',['VmaDefragmentationStats',['../struct_vma_defragmentation_stats.html',1,'VmaDefragmentationStats'],['../group__layer2.html#gab0f9b06441c840fee560de4a2967f8c9',1,'VmaDefragmentationStats(): vk_mem_alloc.h']]], ['vmadestroyallocator',['vmaDestroyAllocator',['../group__general.html#gaa8d164061c88f22fb1fd3c8f3534bc1d',1,'vk_mem_alloc.h']]], ['vmadestroybuffer',['vmaDestroyBuffer',['../group__layer3.html#ga0d9f4e4ba5bf9aab1f1c746387753d77',1,'vk_mem_alloc.h']]], ['vmadestroyimage',['vmaDestroyImage',['../group__layer3.html#gae50d2cb3b4a3bfd4dd40987234e50e7e',1,'vk_mem_alloc.h']]], + ['vmadestroypool',['vmaDestroyPool',['../group__layer2.html#ga5485779c8f1948238fc4e92232fa65e1',1,'vk_mem_alloc.h']]], ['vmadevicememorycallbacks',['VmaDeviceMemoryCallbacks',['../struct_vma_device_memory_callbacks.html',1,'VmaDeviceMemoryCallbacks'],['../group__general.html#ga5e2eb68d727cfd4df25702b027b7aa31',1,'VmaDeviceMemoryCallbacks(): vk_mem_alloc.h']]], - ['vmafindmemorytypeindex',['vmaFindMemoryTypeIndex',['../group__layer1.html#gadf80663373e94bcef382f17534b8694e',1,'vk_mem_alloc.h']]], + ['vmafindmemorytypeindex',['vmaFindMemoryTypeIndex',['../group__layer1.html#gaef15a94b58fbcb0fe706d5720e84a74a',1,'vk_mem_alloc.h']]], ['vmafreememory',['vmaFreeMemory',['../group__layer2.html#ga11f0fbc034fa81a4efedd73d61ce7568',1,'vk_mem_alloc.h']]], ['vmafreestatsstring',['vmaFreeStatsString',['../group__general.html#ga3104eb30d8122c84dd8541063f145288',1,'vk_mem_alloc.h']]], ['vmagetallocationinfo',['vmaGetAllocationInfo',['../group__layer2.html#ga86dd08aba8633bfa4ad0df2e76481d8b',1,'vk_mem_alloc.h']]], ['vmagetmemoryproperties',['vmaGetMemoryProperties',['../group__general.html#gab88db292a17974f911182543fda52d19',1,'vk_mem_alloc.h']]], ['vmagetmemorytypeproperties',['vmaGetMemoryTypeProperties',['../group__general.html#ga8701444752eb5de4464adb5a2b514bca',1,'vk_mem_alloc.h']]], ['vmagetphysicaldeviceproperties',['vmaGetPhysicalDeviceProperties',['../group__general.html#gaecabf7b6e91ea87d0316fa0a9e014fe0',1,'vk_mem_alloc.h']]], + ['vmagetpoolstats',['vmaGetPoolStats',['../group__layer2.html#gae8bf76997b234ef68aad922616df4153',1,'vk_mem_alloc.h']]], + ['vmamakepoolallocationslost',['vmaMakePoolAllocationsLost',['../group__layer2.html#ga736bd6cbda886f36c891727e73bd4024',1,'vk_mem_alloc.h']]], ['vmamapmemory',['vmaMapMemory',['../group__layer2.html#gad5bd1243512d099706de88168992f069',1,'vk_mem_alloc.h']]], ['vmamappersistentlymappedmemory',['vmaMapPersistentlyMappedMemory',['../group__layer2.html#ga03366170bb8e186605518d2f5d65b85a',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirementflagbits',['VmaMemoryRequirementFlagBits',['../group__layer1.html#gadec54ccdb0fd3901e89ac6d38c2619ed',1,'VmaMemoryRequirementFlagBits(): vk_mem_alloc.h'],['../group__layer1.html#gaecb50b7fdb759cc4c4346c40819bf00b',1,'VmaMemoryRequirementFlagBits(): vk_mem_alloc.h']]], - ['vmamemoryrequirementflags',['VmaMemoryRequirementFlags',['../group__layer1.html#gab96b90d34cd1bd0f340fc48f8ca2664a',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirements',['VmaMemoryRequirements',['../struct_vma_memory_requirements.html',1,'VmaMemoryRequirements'],['../group__layer1.html#gae9ee98bebd6e474aa0ef679e10f1d8ca',1,'VmaMemoryRequirements(): vk_mem_alloc.h']]], ['vmamemoryusage',['VmaMemoryUsage',['../group__layer1.html#gaa5846affa1e9da3800e3e78fae2305cc',1,'VmaMemoryUsage(): vk_mem_alloc.h'],['../group__layer1.html#gad63b2113c0bfdbeade1cb498f5a8580d',1,'VmaMemoryUsage(): vk_mem_alloc.h']]], + ['vmapoolcreateflagbits',['VmaPoolCreateFlagBits',['../group__layer2.html#ga9a7c45f9c863695d98c83fa5ac940fe7',1,'VmaPoolCreateFlagBits(): vk_mem_alloc.h'],['../group__layer2.html#ga8f93195158e0e2ac80ca352064e71c1f',1,'VmaPoolCreateFlagBits(): vk_mem_alloc.h']]], + ['vmapoolcreateflags',['VmaPoolCreateFlags',['../group__layer2.html#ga2770e325ea42e087c1b91fdf46d0292a',1,'vk_mem_alloc.h']]], + ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../struct_vma_pool_create_info.html',1,'VmaPoolCreateInfo'],['../group__layer2.html#ga211706e9348dcee25a843ed4ea69bce7',1,'VmaPoolCreateInfo(): vk_mem_alloc.h']]], + ['vmapoolstats',['VmaPoolStats',['../struct_vma_pool_stats.html',1,'VmaPoolStats'],['../group__layer2.html#ga2e5612d871d64c5624087b837a338c34',1,'VmaPoolStats(): vk_mem_alloc.h']]], ['vmasetallocationuserdata',['vmaSetAllocationUserData',['../group__layer2.html#gaf9147d31ffc11d62fc187bde283ed14f',1,'vk_mem_alloc.h']]], + ['vmasetcurrentframeindex',['vmaSetCurrentFrameIndex',['../group__general.html#gade56bf8dc9f5a5eaddf5f119ed525236',1,'vk_mem_alloc.h']]], ['vmastatinfo',['VmaStatInfo',['../struct_vma_stat_info.html',1,'VmaStatInfo'],['../group__general.html#ga810b009a788ee8aac72a25b42ffbe31c',1,'VmaStatInfo(): vk_mem_alloc.h']]], ['vmastats',['VmaStats',['../struct_vma_stats.html',1,'']]], ['vmaunmapmemory',['vmaUnmapMemory',['../group__layer2.html#ga9bc268595cb33f6ec4d519cfce81ff45',1,'vk_mem_alloc.h']]], diff --git a/docs/html/search/classes_0.js b/docs/html/search/classes_0.js index 079a70a..b559087 100644 --- a/docs/html/search/classes_0.js +++ b/docs/html/search/classes_0.js @@ -1,11 +1,13 @@ var searchData= [ + ['vmaallocationcreateinfo',['VmaAllocationCreateInfo',['../struct_vma_allocation_create_info.html',1,'']]], ['vmaallocationinfo',['VmaAllocationInfo',['../struct_vma_allocation_info.html',1,'']]], ['vmaallocatorcreateinfo',['VmaAllocatorCreateInfo',['../struct_vma_allocator_create_info.html',1,'']]], ['vmadefragmentationinfo',['VmaDefragmentationInfo',['../struct_vma_defragmentation_info.html',1,'']]], ['vmadefragmentationstats',['VmaDefragmentationStats',['../struct_vma_defragmentation_stats.html',1,'']]], ['vmadevicememorycallbacks',['VmaDeviceMemoryCallbacks',['../struct_vma_device_memory_callbacks.html',1,'']]], - ['vmamemoryrequirements',['VmaMemoryRequirements',['../struct_vma_memory_requirements.html',1,'']]], + ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../struct_vma_pool_create_info.html',1,'']]], + ['vmapoolstats',['VmaPoolStats',['../struct_vma_pool_stats.html',1,'']]], ['vmastatinfo',['VmaStatInfo',['../struct_vma_stat_info.html',1,'']]], ['vmastats',['VmaStats',['../struct_vma_stats.html',1,'']]] ]; diff --git a/docs/html/search/enums_0.js b/docs/html/search/enums_0.js index f6e10b1..471af77 100644 --- a/docs/html/search/enums_0.js +++ b/docs/html/search/enums_0.js @@ -1,6 +1,7 @@ var searchData= [ + ['vmaallocationcreateflagbits',['VmaAllocationCreateFlagBits',['../group__layer1.html#gad9889c10c798b040d59c92f257cae597',1,'vk_mem_alloc.h']]], ['vmaallocatorflagbits',['VmaAllocatorFlagBits',['../group__general.html#ga34fff29c218d23a7ff7dff44b77b6b6f',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirementflagbits',['VmaMemoryRequirementFlagBits',['../group__layer1.html#gadec54ccdb0fd3901e89ac6d38c2619ed',1,'vk_mem_alloc.h']]], - ['vmamemoryusage',['VmaMemoryUsage',['../group__layer1.html#gaa5846affa1e9da3800e3e78fae2305cc',1,'vk_mem_alloc.h']]] + ['vmamemoryusage',['VmaMemoryUsage',['../group__layer1.html#gaa5846affa1e9da3800e3e78fae2305cc',1,'vk_mem_alloc.h']]], + ['vmapoolcreateflagbits',['VmaPoolCreateFlagBits',['../group__layer2.html#ga9a7c45f9c863695d98c83fa5ac940fe7',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/enumvalues_0.js b/docs/html/search/enumvalues_0.js index 13bd7ca..f624ca0 100644 --- a/docs/html/search/enumvalues_0.js +++ b/docs/html/search/enumvalues_0.js @@ -1,15 +1,20 @@ var searchData= [ + ['vma_5fallocation_5fcreate_5fcan_5fbecome_5flost_5fbit',['VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a5f436af6c8fe8540573a6d22627a6fd2',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fcan_5fmake_5fother_5flost_5fbit',['VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a68686d0ce9beb0d4d1b9f2b8b1389a7e',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597ae5633ec569f4899cf8f29e7385b2f882',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fnever_5fallocate_5fbit',['VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a89759603401014eb325eb22a3839f2ff',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fown_5fmemory_5fbit',['VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597a965e035b69f8728b317803ef2d523aa4',1,'vk_mem_alloc.h']]], + ['vma_5fallocation_5fcreate_5fpersistent_5fmap_5fbit',['VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT',['../group__layer1.html#ggad9889c10c798b040d59c92f257cae597ae443691ef3d077c0dc3de5576ac4c312',1,'vk_mem_alloc.h']]], ['vma_5fallocator_5fexternally_5fsynchronized_5fbit',['VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT',['../group__general.html#gga34fff29c218d23a7ff7dff44b77b6b6fabe92b706180652ceb320da5bc383aef4',1,'vk_mem_alloc.h']]], ['vma_5fallocator_5fflag_5fbits_5fmax_5fenum',['VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM',['../group__general.html#gga34fff29c218d23a7ff7dff44b77b6b6fa914e905a08c2e578f76b9d6c418626cc',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fflag_5fbits_5fmax_5fenum',['VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619eda11f849416519c61bebd90e2e7ecbab21',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fnever_5fallocate_5fbit',['VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619edae1a503c29619c0c9885521d371e9b4c4',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fown_5fmemory_5fbit',['VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619eda9831d98a698347b0cd1594db6a29e8ed',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5frequirement_5fpersistent_5fmap_5fbit',['VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT',['../group__layer1.html#ggadec54ccdb0fd3901e89ac6d38c2619edae75c9dde227c9d4f64693b1e004106dc',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fcpu_5fonly',['VMA_MEMORY_USAGE_CPU_ONLY',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca40bdf4cddeffeb12f43d45ca1286e0a5',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fcpu_5fto_5fgpu',['VMA_MEMORY_USAGE_CPU_TO_GPU',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca9066b52c5a7079bb74a69aaf8b92ff67',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fgpu_5fonly',['VMA_MEMORY_USAGE_GPU_ONLY',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305ccac6b5dc1432d88647aa4cd456246eadf7',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fgpu_5fto_5fcpu',['VMA_MEMORY_USAGE_GPU_TO_CPU',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca7b586d2fdaf82a463b58f581ed72be27',1,'vk_mem_alloc.h']]], ['vma_5fmemory_5fusage_5fmax_5fenum',['VMA_MEMORY_USAGE_MAX_ENUM',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305cca091e69437ef693e8d0d287f1c719ba6e',1,'vk_mem_alloc.h']]], - ['vma_5fmemory_5fusage_5funknown',['VMA_MEMORY_USAGE_UNKNOWN',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305ccaf50d27e34e0925cf3a63db8c839121dd',1,'vk_mem_alloc.h']]] + ['vma_5fmemory_5fusage_5funknown',['VMA_MEMORY_USAGE_UNKNOWN',['../group__layer1.html#ggaa5846affa1e9da3800e3e78fae2305ccaf50d27e34e0925cf3a63db8c839121dd',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fflag_5fbits_5fmax_5fenum',['VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a1c7312bea9ea246846b9054fd6bd6aec',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fignore_5fbuffer_5fimage_5fgranularity_5fbit',['VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a9f1a499508a8edb4e8ba40aa0290a3d2',1,'vk_mem_alloc.h']]], + ['vma_5fpool_5fcreate_5fpersistent_5fmap_5fbit',['VMA_POOL_CREATE_PERSISTENT_MAP_BIT',['../group__layer2.html#gga9a7c45f9c863695d98c83fa5ac940fe7a918441f7b40dca90481b114f5d224fe9',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/functions_0.js b/docs/html/search/functions_0.js index 070a2a6..92212fb 100644 --- a/docs/html/search/functions_0.js +++ b/docs/html/search/functions_0.js @@ -1,27 +1,33 @@ var searchData= [ - ['vmaallocatememory',['vmaAllocateMemory',['../group__layer2.html#ga1a7d45920877a53e9a8fbefd6a536119',1,'vk_mem_alloc.h']]], - ['vmaallocatememoryforbuffer',['vmaAllocateMemoryForBuffer',['../group__layer2.html#ga3536b4da2a2744dc3f3f138be555af02',1,'vk_mem_alloc.h']]], - ['vmaallocatememoryforimage',['vmaAllocateMemoryForImage',['../group__layer2.html#ga801f98c24eb928642742bb1e5615e86e',1,'vk_mem_alloc.h']]], + ['vmaallocatememory',['vmaAllocateMemory',['../group__layer2.html#gabf28077dbf82d0908b8acbe8ee8dd9b8',1,'vk_mem_alloc.h']]], + ['vmaallocatememoryforbuffer',['vmaAllocateMemoryForBuffer',['../group__layer2.html#ga7fdf64415b6c3d83c454f28d2c53df7b',1,'vk_mem_alloc.h']]], + ['vmaallocatememoryforimage',['vmaAllocateMemoryForImage',['../group__layer2.html#ga0faa3f9e5fb233d29d1e00390650febb',1,'vk_mem_alloc.h']]], ['vmabuildstatsstring',['vmaBuildStatsString',['../group__general.html#gaa4fee7eb5253377599ef4fd38c93c2a0',1,'vk_mem_alloc.h']]], ['vmacalculatestats',['vmaCalculateStats',['../group__general.html#ga333b61c1788cb23559177531e6a93ca3',1,'vk_mem_alloc.h']]], ['vmacreateallocator',['vmaCreateAllocator',['../group__general.html#ga200692051ddb34240248234f5f4c17bb',1,'vk_mem_alloc.h']]], - ['vmacreatebuffer',['vmaCreateBuffer',['../group__layer3.html#ga2f711e32e95cf9bf8dff4917230c2e9b',1,'vk_mem_alloc.h']]], - ['vmacreateimage',['vmaCreateImage',['../group__layer3.html#ga9e34bc318ff4b25d1958e79b9db3f1aa',1,'vk_mem_alloc.h']]], + ['vmacreatebuffer',['vmaCreateBuffer',['../group__layer3.html#gac72ee55598617e8eecca384e746bab51',1,'vk_mem_alloc.h']]], + ['vmacreateimage',['vmaCreateImage',['../group__layer3.html#ga02a94f25679275851a53e82eacbcfc73',1,'vk_mem_alloc.h']]], + ['vmacreatelostallocation',['vmaCreateLostAllocation',['../group__layer2.html#gae5c9657d9e94756269145b01c05d16f1',1,'vk_mem_alloc.h']]], + ['vmacreatepool',['vmaCreatePool',['../group__layer2.html#ga5c8770ded7c59c8caac6de0c2cb00b50',1,'vk_mem_alloc.h']]], ['vmadefragment',['vmaDefragment',['../group__layer2.html#ga6aced90fcc7b39882b6654a740a0b9bb',1,'vk_mem_alloc.h']]], ['vmadestroyallocator',['vmaDestroyAllocator',['../group__general.html#gaa8d164061c88f22fb1fd3c8f3534bc1d',1,'vk_mem_alloc.h']]], ['vmadestroybuffer',['vmaDestroyBuffer',['../group__layer3.html#ga0d9f4e4ba5bf9aab1f1c746387753d77',1,'vk_mem_alloc.h']]], ['vmadestroyimage',['vmaDestroyImage',['../group__layer3.html#gae50d2cb3b4a3bfd4dd40987234e50e7e',1,'vk_mem_alloc.h']]], - ['vmafindmemorytypeindex',['vmaFindMemoryTypeIndex',['../group__layer1.html#gadf80663373e94bcef382f17534b8694e',1,'vk_mem_alloc.h']]], + ['vmadestroypool',['vmaDestroyPool',['../group__layer2.html#ga5485779c8f1948238fc4e92232fa65e1',1,'vk_mem_alloc.h']]], + ['vmafindmemorytypeindex',['vmaFindMemoryTypeIndex',['../group__layer1.html#gaef15a94b58fbcb0fe706d5720e84a74a',1,'vk_mem_alloc.h']]], ['vmafreememory',['vmaFreeMemory',['../group__layer2.html#ga11f0fbc034fa81a4efedd73d61ce7568',1,'vk_mem_alloc.h']]], ['vmafreestatsstring',['vmaFreeStatsString',['../group__general.html#ga3104eb30d8122c84dd8541063f145288',1,'vk_mem_alloc.h']]], ['vmagetallocationinfo',['vmaGetAllocationInfo',['../group__layer2.html#ga86dd08aba8633bfa4ad0df2e76481d8b',1,'vk_mem_alloc.h']]], ['vmagetmemoryproperties',['vmaGetMemoryProperties',['../group__general.html#gab88db292a17974f911182543fda52d19',1,'vk_mem_alloc.h']]], ['vmagetmemorytypeproperties',['vmaGetMemoryTypeProperties',['../group__general.html#ga8701444752eb5de4464adb5a2b514bca',1,'vk_mem_alloc.h']]], ['vmagetphysicaldeviceproperties',['vmaGetPhysicalDeviceProperties',['../group__general.html#gaecabf7b6e91ea87d0316fa0a9e014fe0',1,'vk_mem_alloc.h']]], + ['vmagetpoolstats',['vmaGetPoolStats',['../group__layer2.html#gae8bf76997b234ef68aad922616df4153',1,'vk_mem_alloc.h']]], + ['vmamakepoolallocationslost',['vmaMakePoolAllocationsLost',['../group__layer2.html#ga736bd6cbda886f36c891727e73bd4024',1,'vk_mem_alloc.h']]], ['vmamapmemory',['vmaMapMemory',['../group__layer2.html#gad5bd1243512d099706de88168992f069',1,'vk_mem_alloc.h']]], ['vmamappersistentlymappedmemory',['vmaMapPersistentlyMappedMemory',['../group__layer2.html#ga03366170bb8e186605518d2f5d65b85a',1,'vk_mem_alloc.h']]], ['vmasetallocationuserdata',['vmaSetAllocationUserData',['../group__layer2.html#gaf9147d31ffc11d62fc187bde283ed14f',1,'vk_mem_alloc.h']]], + ['vmasetcurrentframeindex',['vmaSetCurrentFrameIndex',['../group__general.html#gade56bf8dc9f5a5eaddf5f119ed525236',1,'vk_mem_alloc.h']]], ['vmaunmapmemory',['vmaUnmapMemory',['../group__layer2.html#ga9bc268595cb33f6ec4d519cfce81ff45',1,'vk_mem_alloc.h']]], ['vmaunmappersistentlymappedmemory',['vmaUnmapPersistentlyMappedMemory',['../group__layer2.html#ga26b87244491c1fe77f11fe9ab5779c27',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/typedefs_1.js b/docs/html/search/typedefs_1.js index 639050c..7b9e407 100644 --- a/docs/html/search/typedefs_1.js +++ b/docs/html/search/typedefs_1.js @@ -1,5 +1,8 @@ var searchData= [ + ['vmaallocationcreateflagbits',['VmaAllocationCreateFlagBits',['../group__layer1.html#gabf6bf6748c7a9fe7ce5b7835c0f56af4',1,'vk_mem_alloc.h']]], + ['vmaallocationcreateflags',['VmaAllocationCreateFlags',['../group__layer1.html#ga5225e5e11f8376f6a31a1791f3d6e817',1,'vk_mem_alloc.h']]], + ['vmaallocationcreateinfo',['VmaAllocationCreateInfo',['../group__layer1.html#ga86c44f9950b40d50088ed93a17870a7a',1,'vk_mem_alloc.h']]], ['vmaallocationinfo',['VmaAllocationInfo',['../group__layer2.html#ga795e6ff02a21d5486c0565f403dd9255',1,'vk_mem_alloc.h']]], ['vmaallocatorcreateinfo',['VmaAllocatorCreateInfo',['../group__general.html#gae0f6d1d733dded220d28134da46b4283',1,'vk_mem_alloc.h']]], ['vmaallocatorflagbits',['VmaAllocatorFlagBits',['../group__general.html#gac51ae2c316ee760ec5aebbae06d31253',1,'vk_mem_alloc.h']]], @@ -7,9 +10,10 @@ var searchData= ['vmadefragmentationinfo',['VmaDefragmentationInfo',['../group__layer2.html#gae67f8573a0cf20f16f0a1eecbca566a0',1,'vk_mem_alloc.h']]], ['vmadefragmentationstats',['VmaDefragmentationStats',['../group__layer2.html#gab0f9b06441c840fee560de4a2967f8c9',1,'vk_mem_alloc.h']]], ['vmadevicememorycallbacks',['VmaDeviceMemoryCallbacks',['../group__general.html#ga5e2eb68d727cfd4df25702b027b7aa31',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirementflagbits',['VmaMemoryRequirementFlagBits',['../group__layer1.html#gaecb50b7fdb759cc4c4346c40819bf00b',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirementflags',['VmaMemoryRequirementFlags',['../group__layer1.html#gab96b90d34cd1bd0f340fc48f8ca2664a',1,'vk_mem_alloc.h']]], - ['vmamemoryrequirements',['VmaMemoryRequirements',['../group__layer1.html#gae9ee98bebd6e474aa0ef679e10f1d8ca',1,'vk_mem_alloc.h']]], ['vmamemoryusage',['VmaMemoryUsage',['../group__layer1.html#gad63b2113c0bfdbeade1cb498f5a8580d',1,'vk_mem_alloc.h']]], + ['vmapoolcreateflagbits',['VmaPoolCreateFlagBits',['../group__layer2.html#ga8f93195158e0e2ac80ca352064e71c1f',1,'vk_mem_alloc.h']]], + ['vmapoolcreateflags',['VmaPoolCreateFlags',['../group__layer2.html#ga2770e325ea42e087c1b91fdf46d0292a',1,'vk_mem_alloc.h']]], + ['vmapoolcreateinfo',['VmaPoolCreateInfo',['../group__layer2.html#ga211706e9348dcee25a843ed4ea69bce7',1,'vk_mem_alloc.h']]], + ['vmapoolstats',['VmaPoolStats',['../group__layer2.html#ga2e5612d871d64c5624087b837a338c34',1,'vk_mem_alloc.h']]], ['vmastatinfo',['VmaStatInfo',['../group__general.html#ga810b009a788ee8aac72a25b42ffbe31c',1,'vk_mem_alloc.h']]] ]; diff --git a/docs/html/search/variables_0.js b/docs/html/search/variables_0.js index d65975f..36e26b6 100644 --- a/docs/html/search/variables_0.js +++ b/docs/html/search/variables_0.js @@ -1,5 +1,8 @@ var searchData= [ - ['allocationcount',['AllocationCount',['../struct_vma_stat_info.html#a240402222ac6777e4079653c5d542cb0',1,'VmaStatInfo']]], + ['allocationcount',['AllocationCount',['../struct_vma_stat_info.html#a240402222ac6777e4079653c5d542cb0',1,'VmaStatInfo::AllocationCount()'],['../struct_vma_pool_stats.html#ad1924eb54fffa45e9e0e65670c8fe5eb',1,'VmaPoolStats::allocationCount()']]], + ['allocationsizeavg',['AllocationSizeAvg',['../struct_vma_stat_info.html#a0019702e4f4ea4ed824aebf250841f21',1,'VmaStatInfo']]], + ['allocationsizemax',['AllocationSizeMax',['../struct_vma_stat_info.html#aba2ef16e2ac09b97305c58a168a3805c',1,'VmaStatInfo']]], + ['allocationsizemin',['AllocationSizeMin',['../struct_vma_stat_info.html#a61a834fa78bcf41fb5a53b286b2e1580',1,'VmaStatInfo']]], ['allocationsmoved',['allocationsMoved',['../struct_vma_defragmentation_stats.html#aefeabf130022008eadd75999478af3f9',1,'VmaDefragmentationStats']]] ]; diff --git a/docs/html/search/variables_1.js b/docs/html/search/variables_1.js index cb221fc..e73151c 100644 --- a/docs/html/search/variables_1.js +++ b/docs/html/search/variables_1.js @@ -1,5 +1,7 @@ var searchData= [ + ['blockcount',['BlockCount',['../struct_vma_stat_info.html#a9af0d9809007cf4a0009d324b2b8bbc9',1,'VmaStatInfo']]], + ['blocksize',['blockSize',['../struct_vma_pool_create_info.html#aa4265160536cdb9be821b7686c16c676',1,'VmaPoolCreateInfo']]], ['bytesfreed',['bytesFreed',['../struct_vma_defragmentation_stats.html#ab0cb9ac0dbc106c77e384ea676422f28',1,'VmaDefragmentationStats']]], ['bytesmoved',['bytesMoved',['../struct_vma_defragmentation_stats.html#a36f9d5df2a10ba2a36b16e126d60572d',1,'VmaDefragmentationStats']]] ]; diff --git a/docs/html/search/variables_3.js b/docs/html/search/variables_3.js index 60408e6..06fd7c5 100644 --- a/docs/html/search/variables_3.js +++ b/docs/html/search/variables_3.js @@ -1,4 +1,5 @@ var searchData= [ - ['flags',['flags',['../struct_vma_allocator_create_info.html#a35d3b42ac5a3be3758926819dc053859',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_memory_requirements.html#a53c9ea8c3cbcc12dddbe9ae23bb85eb6',1,'VmaMemoryRequirements::flags()']]] + ['flags',['flags',['../struct_vma_allocator_create_info.html#a35d3b42ac5a3be3758926819dc053859',1,'VmaAllocatorCreateInfo::flags()'],['../struct_vma_allocation_create_info.html#add09658ac14fe290ace25470ddd6d41b',1,'VmaAllocationCreateInfo::flags()'],['../struct_vma_pool_create_info.html#a8405139f63d078340ae74513a59f5446',1,'VmaPoolCreateInfo::flags()']]], + ['frameinusecount',['frameInUseCount',['../struct_vma_allocator_create_info.html#a21ea188dd212b8171cb9ecbed4a2a3a7',1,'VmaAllocatorCreateInfo::frameInUseCount()'],['../struct_vma_pool_create_info.html#a9437e43ffbb644dbbf7fc4e50cfad6aa',1,'VmaPoolCreateInfo::frameInUseCount()']]] ]; diff --git a/docs/html/search/variables_4.js b/docs/html/search/variables_4.js index b23ec17..1002ad2 100644 --- a/docs/html/search/variables_4.js +++ b/docs/html/search/variables_4.js @@ -1,7 +1,10 @@ var searchData= [ ['maxallocationstomove',['maxAllocationsToMove',['../struct_vma_defragmentation_info.html#aa7c7304e13c71f604c907196c4e28fbc',1,'VmaDefragmentationInfo']]], + ['maxblockcount',['maxBlockCount',['../struct_vma_pool_create_info.html#ae41142f2834fcdc82baa4883c187b75c',1,'VmaPoolCreateInfo']]], ['maxbytestomove',['maxBytesToMove',['../struct_vma_defragmentation_info.html#acb311c940a777270e67e1b81c5ab6a1d',1,'VmaDefragmentationInfo']]], ['memoryheap',['memoryHeap',['../struct_vma_stats.html#a0e6611508c29a187f0fd14ff1a0329c0',1,'VmaStats']]], - ['memorytype',['memoryType',['../struct_vma_stats.html#a13e3caf754be79352c42408756309331',1,'VmaStats::memoryType()'],['../struct_vma_allocation_info.html#a7f6b0aa58c135e488e6b40a388dad9d5',1,'VmaAllocationInfo::memoryType()']]] + ['memorytype',['memoryType',['../struct_vma_stats.html#a13e3caf754be79352c42408756309331',1,'VmaStats::memoryType()'],['../struct_vma_allocation_info.html#a7f6b0aa58c135e488e6b40a388dad9d5',1,'VmaAllocationInfo::memoryType()']]], + ['memorytypeindex',['memoryTypeIndex',['../struct_vma_pool_create_info.html#a596fa76b685d3f1f688f84a709a5b319',1,'VmaPoolCreateInfo']]], + ['minblockcount',['minBlockCount',['../struct_vma_pool_create_info.html#ad8006fb803185c0a699d30f3e9a865ae',1,'VmaPoolCreateInfo']]] ]; diff --git a/docs/html/search/variables_6.js b/docs/html/search/variables_6.js index 7119d4d..2b404e1 100644 --- a/docs/html/search/variables_6.js +++ b/docs/html/search/variables_6.js @@ -6,8 +6,9 @@ var searchData= ['pfnfree',['pfnFree',['../struct_vma_device_memory_callbacks.html#abe8a3328bbc916f6f712fdb6b299444c',1,'VmaDeviceMemoryCallbacks']]], ['physicaldevice',['physicalDevice',['../struct_vma_allocator_create_info.html#a08230f04ae6ccf8a78150a9e829a7156',1,'VmaAllocatorCreateInfo']]], ['pmappeddata',['pMappedData',['../struct_vma_allocation_info.html#a5eeffbe2d2f30f53370ff14aefbadbe2',1,'VmaAllocationInfo']]], - ['preferredflags',['preferredFlags',['../struct_vma_memory_requirements.html#a6e105f836c2288034c711815b18226dc',1,'VmaMemoryRequirements']]], + ['pool',['pool',['../struct_vma_allocation_create_info.html#a6272c0555cfd1fe28bff1afeb6190150',1,'VmaAllocationCreateInfo']]], + ['preferredflags',['preferredFlags',['../struct_vma_allocation_create_info.html#a7fe8d81a1ad10b2a2faacacee5b15d6d',1,'VmaAllocationCreateInfo']]], ['preferredlargeheapblocksize',['preferredLargeHeapBlockSize',['../struct_vma_allocator_create_info.html#a8e4714298e3121cdd8b214a1ae7a637a',1,'VmaAllocatorCreateInfo']]], ['preferredsmallheapblocksize',['preferredSmallHeapBlockSize',['../struct_vma_allocator_create_info.html#ab435423d84d5ab26e2c347c51771f90a',1,'VmaAllocatorCreateInfo']]], - ['puserdata',['pUserData',['../struct_vma_memory_requirements.html#a8470093e93ed07ed2557490cdc67566a',1,'VmaMemoryRequirements::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]] + ['puserdata',['pUserData',['../struct_vma_allocation_create_info.html#a8259e85c272683434f4abb4ddddffe19',1,'VmaAllocationCreateInfo::pUserData()'],['../struct_vma_allocation_info.html#adc507656149c04de7ed95d0042ba2a13',1,'VmaAllocationInfo::pUserData()']]] ]; diff --git a/docs/html/search/variables_7.js b/docs/html/search/variables_7.js index 0a576b4..9505699 100644 --- a/docs/html/search/variables_7.js +++ b/docs/html/search/variables_7.js @@ -1,4 +1,4 @@ var searchData= [ - ['requiredflags',['requiredFlags',['../struct_vma_memory_requirements.html#a8876c1b0f112e13a277f16967064cfe0',1,'VmaMemoryRequirements']]] + ['requiredflags',['requiredFlags',['../struct_vma_allocation_create_info.html#a9166390303ff42d783305bc31c2b6b90',1,'VmaAllocationCreateInfo']]] ]; diff --git a/docs/html/search/variables_8.js b/docs/html/search/variables_8.js index 5261aad..740c955 100644 --- a/docs/html/search/variables_8.js +++ b/docs/html/search/variables_8.js @@ -1,8 +1,4 @@ var searchData= [ - ['size',['size',['../struct_vma_allocation_info.html#aac76d113a6a5ccbb09fea00fb25fd18f',1,'VmaAllocationInfo']]], - ['suballocationcount',['SuballocationCount',['../struct_vma_stat_info.html#a09fb04b5491661c2e838d098d51bcead',1,'VmaStatInfo']]], - ['suballocationsizeavg',['SuballocationSizeAvg',['../struct_vma_stat_info.html#abb6c3e160a136787f474a18a8264d83b',1,'VmaStatInfo']]], - ['suballocationsizemax',['SuballocationSizeMax',['../struct_vma_stat_info.html#a6be7faf2b7fcff5a9bc017d90aed9271',1,'VmaStatInfo']]], - ['suballocationsizemin',['SuballocationSizeMin',['../struct_vma_stat_info.html#a9dc0b50fab2f10ab99366b79424bf14b',1,'VmaStatInfo']]] + ['size',['size',['../struct_vma_pool_stats.html#a326807b2de2b0931cee4ed9a5f2e420c',1,'VmaPoolStats::size()'],['../struct_vma_allocation_info.html#aac76d113a6a5ccbb09fea00fb25fd18f',1,'VmaAllocationInfo::size()']]] ]; diff --git a/docs/html/search/variables_a.js b/docs/html/search/variables_a.js index cc1a3e7..91686af 100644 --- a/docs/html/search/variables_a.js +++ b/docs/html/search/variables_a.js @@ -1,10 +1,11 @@ var searchData= [ ['unusedbytes',['UnusedBytes',['../struct_vma_stat_info.html#a394d2aef4348cb58abf73764804b4f2d',1,'VmaStatInfo']]], - ['unusedrangecount',['UnusedRangeCount',['../struct_vma_stat_info.html#a56c4fb4dba646479180b601854cde2a6',1,'VmaStatInfo']]], + ['unusedrangecount',['UnusedRangeCount',['../struct_vma_stat_info.html#a56c4fb4dba646479180b601854cde2a6',1,'VmaStatInfo::UnusedRangeCount()'],['../struct_vma_pool_stats.html#ae4f3546ffa4d1e598b64d8e6134854f4',1,'VmaPoolStats::unusedRangeCount()']]], ['unusedrangesizeavg',['UnusedRangeSizeAvg',['../struct_vma_stat_info.html#a88ad9bdc2b3a98964a4d0c338c0c9060',1,'VmaStatInfo']]], ['unusedrangesizemax',['UnusedRangeSizeMax',['../struct_vma_stat_info.html#a10c52c0841f01ca704e8ddb1ea6a635d',1,'VmaStatInfo']]], ['unusedrangesizemin',['UnusedRangeSizeMin',['../struct_vma_stat_info.html#a07c508f42a4d3424bd0c259784a7f2d6',1,'VmaStatInfo']]], - ['usage',['usage',['../struct_vma_memory_requirements.html#ab588497177a57847ed04e0a1aef54bbe',1,'VmaMemoryRequirements']]], + ['unusedsize',['unusedSize',['../struct_vma_pool_stats.html#ad7c54874724fce7b06aba526202d82a8',1,'VmaPoolStats']]], + ['usage',['usage',['../struct_vma_allocation_create_info.html#accb8b06b1f677d858cb9af20705fa910',1,'VmaAllocationCreateInfo']]], ['usedbytes',['UsedBytes',['../struct_vma_stat_info.html#a86f82cb9cffd456b9da63eaf26c9ff04',1,'VmaStatInfo']]] ]; diff --git a/docs/html/struct_vma_allocation_create_info-members.html b/docs/html/struct_vma_allocation_create_info-members.html new file mode 100644 index 0000000..1b63856 --- /dev/null +++ b/docs/html/struct_vma_allocation_create_info-members.html @@ -0,0 +1,82 @@ + + + + + + + +Vulkan Memory Allocator: Member List + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +
      VmaAllocationCreateInfo Member List
      +
      +
      + +

      This is the complete list of members for VmaAllocationCreateInfo, including all inherited members.

      + + + + + + + +
      flagsVmaAllocationCreateInfo
      poolVmaAllocationCreateInfo
      preferredFlagsVmaAllocationCreateInfo
      pUserDataVmaAllocationCreateInfo
      requiredFlagsVmaAllocationCreateInfo
      usageVmaAllocationCreateInfo
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_allocation_create_info.html b/docs/html/struct_vma_allocation_create_info.html new file mode 100644 index 0000000..691f0eb --- /dev/null +++ b/docs/html/struct_vma_allocation_create_info.html @@ -0,0 +1,208 @@ + + + + + + + +Vulkan Memory Allocator: VmaAllocationCreateInfo Struct Reference + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +Public Attributes | +List of all members
      +
      +
      VmaAllocationCreateInfo Struct Reference
      Layer 1 Choosing Memory Type
      +
      +
      + +

      #include <vk_mem_alloc.h>

      + + + + + + + + + + + + + + + + + + + + +

      +Public Attributes

      VmaAllocationCreateFlags flags
       Use VmaAllocationCreateFlagBits enum. More...
       
      VmaMemoryUsage usage
       Intended usage of memory. More...
       
      VkMemoryPropertyFlags requiredFlags
       Flags that must be set in a Memory Type chosen for an allocation. More...
       
      VkMemoryPropertyFlags preferredFlags
       Flags that preferably should be set in a Memory Type chosen for an allocation. More...
       
      void * pUserData
       Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData(). More...
       
      VmaPool pool
       Pool that this allocation should be created in. More...
       
      +

      Member Data Documentation

      + +

      ◆ flags

      + +
      +
      + + + + +
      VmaAllocationCreateFlags VmaAllocationCreateInfo::flags
      +
      + +

      Use VmaAllocationCreateFlagBits enum.

      + +
      +
      + +

      ◆ pool

      + +
      +
      + + + + +
      VmaPool VmaAllocationCreateInfo::pool
      +
      + +

      Pool that this allocation should be created in.

      +

      Leave VK_NULL_HANDLE to allocate from general memory.

      + +
      +
      + +

      ◆ preferredFlags

      + +
      +
      + + + + +
      VkMemoryPropertyFlags VmaAllocationCreateInfo::preferredFlags
      +
      + +

      Flags that preferably should be set in a Memory Type chosen for an allocation.

      +

      Set to 0 if no additional flags are prefered and only requiredFlags should be used.
      +If not 0, it must be a superset or equal to requiredFlags.
      +If pool is not null, this member is ignored.

      + +
      +
      + +

      ◆ pUserData

      + +
      +
      + + + + +
      void* VmaAllocationCreateInfo::pUserData
      +
      + +

      Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData().

      + +
      +
      + +

      ◆ requiredFlags

      + +
      +
      + + + + +
      VkMemoryPropertyFlags VmaAllocationCreateInfo::requiredFlags
      +
      + +

      Flags that must be set in a Memory Type chosen for an allocation.

      +

      Leave 0 if you specify requirement via usage.
      +If pool is not null, this member is ignored.

      + +
      +
      + +

      ◆ usage

      + +
      +
      + + + + +
      VmaMemoryUsage VmaAllocationCreateInfo::usage
      +
      + +

      Intended usage of memory.

      +

      Leave VMA_MEMORY_USAGE_UNKNOWN if you specify requiredFlags. You can also use both.
      +If pool is not null, this member is ignored.

      + +
      +
      +
      The documentation for this struct was generated from the following file: +
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_allocation_info.html b/docs/html/struct_vma_allocation_info.html index aff38a5..9b32904 100644 --- a/docs/html/struct_vma_allocation_info.html +++ b/docs/html/struct_vma_allocation_info.html @@ -66,7 +66,7 @@ $(function() {
      -

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). +

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...

      #include <vk_mem_alloc.h>

      @@ -89,11 +89,11 @@ Public Attributes  Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistently mapped. More...
        void * pUserData - Custom general-purpose pointer that was passed as VmaMemoryRequirements::pUserData or set using vmaSetAllocationUserData(). More...
      + Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData(). More...
       

      Detailed Description

      -

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().

      +

      Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().

      Member Data Documentation

      ◆ deviceMemory

      @@ -109,7 +109,8 @@ Public Attributes

      Handle to Vulkan memory object.

      Same memory object can be shared by multiple allocations.

      -

      It can change after call to vmaDefragment() if this allocation is passed to the function.

      +

      It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.

      +

      If the allocation is lost, it is equal to VK_NULL_HANDLE.

      @@ -143,7 +144,7 @@ Public Attributes

      Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.

      -

      It can change after call to vmaDefragment() if this allocation is passed to the function.

      +

      It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.

      @@ -160,7 +161,7 @@ Public Attributes

      Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistently mapped.

      -

      It can change after call to vmaUnmapPersistentlyMappedMemory(), vmaMapPersistentlyMappedMemory(). It can also change after call to vmaDefragment() if this allocation is passed to the function.

      +

      It can change after call to vmaUnmapPersistentlyMappedMemory(), vmaMapPersistentlyMappedMemory(). It can also change after call to vmaDefragment() if this allocation is passed to the function.

      @@ -176,7 +177,7 @@ Public Attributes
      -

      Custom general-purpose pointer that was passed as VmaMemoryRequirements::pUserData or set using vmaSetAllocationUserData().

      +

      Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData().

      It can change after call to vmaSetAllocationUserData() for this allocation.

      @@ -194,7 +195,7 @@ Public Attributes

      Size of this allocation, in bytes.

      -

      It never changes.

      +

      It never changes, unless allocation is lost.

      diff --git a/docs/html/struct_vma_allocator_create_info-members.html b/docs/html/struct_vma_allocator_create_info-members.html index dbc84dd..63f4c83 100644 --- a/docs/html/struct_vma_allocator_create_info-members.html +++ b/docs/html/struct_vma_allocator_create_info-members.html @@ -67,11 +67,12 @@ $(function() { - - - - - + + + + + +
      deviceVmaAllocatorCreateInfo
      flagsVmaAllocatorCreateInfo
      pAllocationCallbacksVmaAllocatorCreateInfo
      pDeviceMemoryCallbacksVmaAllocatorCreateInfo
      physicalDeviceVmaAllocatorCreateInfo
      preferredLargeHeapBlockSizeVmaAllocatorCreateInfo
      preferredSmallHeapBlockSizeVmaAllocatorCreateInfo
      frameInUseCountVmaAllocatorCreateInfo
      pAllocationCallbacksVmaAllocatorCreateInfo
      pDeviceMemoryCallbacksVmaAllocatorCreateInfo
      physicalDeviceVmaAllocatorCreateInfo
      preferredLargeHeapBlockSizeVmaAllocatorCreateInfo
      preferredSmallHeapBlockSizeVmaAllocatorCreateInfo
      diff --git a/docs/html/struct_vma_allocator_create_info.html b/docs/html/struct_vma_allocator_create_info.html index ec129b1..1f2157e 100644 --- a/docs/html/struct_vma_allocator_create_info.html +++ b/docs/html/struct_vma_allocator_create_info.html @@ -89,11 +89,14 @@ Public Attributes  Size of a single memory block to allocate for resources from a small heap <= 512 MB. More...
        const VkAllocationCallbacks * pAllocationCallbacks - Custom allocation callbacks. More...
      + Custom CPU memory allocation callbacks. More...
        const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks  Informative callbacks for vkAllocateMemory, vkFreeMemory. More...
        +uint32_t frameInUseCount + Maximum number of additional frames that are in use at the same time as current frame. More...
      + 

      Detailed Description

      Description of a Allocator to be created.

      @@ -111,7 +114,7 @@ Public Attributes

      Vulkan device.

      -

      It must be valid throughout whole lifetime of created Allocator.

      +

      It must be valid throughout whole lifetime of created allocator.

      @@ -129,6 +132,25 @@ Public Attributes

      Flags for created allocator. Use VmaAllocatorFlagBits enum.

      + + + +

      ◆ frameInUseCount

      + +
      +
      + + + + +
      uint32_t VmaAllocatorCreateInfo::frameInUseCount
      +
      + +

      Maximum number of additional frames that are in use at the same time as current frame.

      +

      This value is used only when you make allocations with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.

      +

      For example, if you double-buffer your command buffers, so resources used for rendering in previous frame may still be in use by the GPU at the moment you allocate resources needed for the current frame, set this value to 1.

      +

      If you want to allow any allocations other than used in the current frame to become lost, set this value to 0.

      +
      @@ -143,7 +165,7 @@ Public Attributes
      -

      Custom allocation callbacks.

      +

      Custom CPU memory allocation callbacks.

      Optional, can be null. When specified, will also be used for all CPU-side memory allocations.

      @@ -178,7 +200,7 @@ Public Attributes

      Vulkan physical device.

      -

      It must be valid throughout whole lifetime of created Allocator.

      +

      It must be valid throughout whole lifetime of created allocator.

      diff --git a/docs/html/struct_vma_defragmentation_info.html b/docs/html/struct_vma_defragmentation_info.html index e4be271..225cd0d 100644 --- a/docs/html/struct_vma_defragmentation_info.html +++ b/docs/html/struct_vma_defragmentation_info.html @@ -96,7 +96,7 @@ Public Attributes

      Maximum number of allocations that can be moved to different place.

      -

      Default is UINT32_MAX, which means no limit.

      +

      Default is UINT32_MAX, which means no limit.

      @@ -113,7 +113,7 @@ Public Attributes

      Maximum total numbers of bytes that can be copied while moving allocations to different places.

      -

      Default is VK_WHOLE_SIZE, which means no limit.

      +

      Default is VK_WHOLE_SIZE, which means no limit.

      diff --git a/docs/html/struct_vma_defragmentation_stats.html b/docs/html/struct_vma_defragmentation_stats.html index 7c92ff4..72f38cb 100644 --- a/docs/html/struct_vma_defragmentation_stats.html +++ b/docs/html/struct_vma_defragmentation_stats.html @@ -77,13 +77,13 @@ Public Attributes  Total number of bytes that have been copied while moving allocations to different places. More...
        VkDeviceSize bytesFreed - Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects. More...
      + Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects. More...
        uint32_t allocationsMoved  Number of allocations that have been moved to different places. More...
        uint32_t deviceMemoryBlocksFreed - Number of empty VkDeviceMemory objects that have been released to the system. More...
      + Number of empty VkDeviceMemory objects that have been released to the system. More...
       

      Detailed Description

      @@ -117,7 +117,7 @@ Public Attributes
      -

      Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects.

      +

      Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects.

      @@ -149,7 +149,7 @@ Public Attributes
      -

      Number of empty VkDeviceMemory objects that have been released to the system.

      +

      Number of empty VkDeviceMemory objects that have been released to the system.

      diff --git a/docs/html/struct_vma_pool_create_info-members.html b/docs/html/struct_vma_pool_create_info-members.html new file mode 100644 index 0000000..49088c2 --- /dev/null +++ b/docs/html/struct_vma_pool_create_info-members.html @@ -0,0 +1,82 @@ + + + + + + + +Vulkan Memory Allocator: Member List + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +
      VmaPoolCreateInfo Member List
      +
      +
      + +

      This is the complete list of members for VmaPoolCreateInfo, including all inherited members.

      + + + + + + + +
      blockSizeVmaPoolCreateInfo
      flagsVmaPoolCreateInfo
      frameInUseCountVmaPoolCreateInfo
      maxBlockCountVmaPoolCreateInfo
      memoryTypeIndexVmaPoolCreateInfo
      minBlockCountVmaPoolCreateInfo
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_pool_create_info.html b/docs/html/struct_vma_pool_create_info.html new file mode 100644 index 0000000..b05638f --- /dev/null +++ b/docs/html/struct_vma_pool_create_info.html @@ -0,0 +1,212 @@ + + + + + + + +Vulkan Memory Allocator: VmaPoolCreateInfo Struct Reference + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +Public Attributes | +List of all members
      +
      +
      VmaPoolCreateInfo Struct Reference
      Layer 2 Allocating Memory
      +
      +
      + +

      Describes parameter of created VmaPool. + More...

      + +

      #include <vk_mem_alloc.h>

      + + + + + + + + + + + + + + + + + + + + +

      +Public Attributes

      uint32_t memoryTypeIndex
       Vulkan memory type index to allocate this pool from. More...
       
      VmaPoolCreateFlags flags
       Use combination of VmaPoolCreateFlagBits. More...
       
      VkDeviceSize blockSize
       Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes. More...
       
      size_t minBlockCount
       Minimum number of blocks to be always allocated in this pool, even if they stay empty. More...
       
      size_t maxBlockCount
       Maximum number of blocks that can be allocated in this pool. More...
       
      uint32_t frameInUseCount
       Maximum number of additional frames that are in use at the same time as current frame. More...
       
      +

      Detailed Description

      +

      Describes parameter of created VmaPool.

      +

      Member Data Documentation

      + +

      ◆ blockSize

      + +
      +
      + + + + +
      VkDeviceSize VmaPoolCreateInfo::blockSize
      +
      + +

      Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.

      +

      Optional. Leave 0 to use default.

      + +
      +
      + +

      ◆ flags

      + +
      +
      + + + + +
      VmaPoolCreateFlags VmaPoolCreateInfo::flags
      +
      + +

      Use combination of VmaPoolCreateFlagBits.

      + +
      +
      + +

      ◆ frameInUseCount

      + +
      +
      + + + + +
      uint32_t VmaPoolCreateInfo::frameInUseCount
      +
      + +

      Maximum number of additional frames that are in use at the same time as current frame.

      +

      This value is used only when you make allocations with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.

      +

      For example, if you double-buffer your command buffers, so resources used for rendering in previous frame may still be in use by the GPU at the moment you allocate resources needed for the current frame, set this value to 1.

      +

      If you want to allow any allocations other than used in the current frame to become lost, set this value to 0.

      + +
      +
      + +

      ◆ maxBlockCount

      + +
      +
      + + + + +
      size_t VmaPoolCreateInfo::maxBlockCount
      +
      + +

      Maximum number of blocks that can be allocated in this pool.

      +

      Optional. Set to 0 to use SIZE_MAX, which means no limit.

      +

      Set to same value as minBlockCount to have fixed amount of memory allocated throuout whole lifetime of this pool.

      + +
      +
      + +

      ◆ memoryTypeIndex

      + +
      +
      + + + + +
      uint32_t VmaPoolCreateInfo::memoryTypeIndex
      +
      + +

      Vulkan memory type index to allocate this pool from.

      + +
      +
      + +

      ◆ minBlockCount

      + +
      +
      + + + + +
      size_t VmaPoolCreateInfo::minBlockCount
      +
      + +

      Minimum number of blocks to be always allocated in this pool, even if they stay empty.

      +

      Set to 0 to have no preallocated blocks and let the pool be completely empty.

      + +
      +
      +
      The documentation for this struct was generated from the following file: +
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_pool_stats-members.html b/docs/html/struct_vma_pool_stats-members.html new file mode 100644 index 0000000..2e96611 --- /dev/null +++ b/docs/html/struct_vma_pool_stats-members.html @@ -0,0 +1,80 @@ + + + + + + + +Vulkan Memory Allocator: Member List + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +
      VmaPoolStats Member List
      +
      +
      + +

      This is the complete list of members for VmaPoolStats, including all inherited members.

      + + + + + +
      allocationCountVmaPoolStats
      sizeVmaPoolStats
      unusedRangeCountVmaPoolStats
      unusedSizeVmaPoolStats
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_pool_stats.html b/docs/html/struct_vma_pool_stats.html new file mode 100644 index 0000000..2090cb4 --- /dev/null +++ b/docs/html/struct_vma_pool_stats.html @@ -0,0 +1,167 @@ + + + + + + + +Vulkan Memory Allocator: VmaPoolStats Struct Reference + + + + + + + + + +
      +
      + + + + + + +
      +
      Vulkan Memory Allocator +
      +
      +
      + + + + + + + + +
      +
      + + +
      + +
      + +
      +
      +
      +Public Attributes | +List of all members
      +
      +
      VmaPoolStats Struct Reference
      Layer 2 Allocating Memory
      +
      +
      + +

      Describes parameter of existing VmaPool. + More...

      + +

      #include <vk_mem_alloc.h>

      + + + + + + + + + + + + + + +

      +Public Attributes

      VkDeviceSize size
       Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes. More...
       
      VkDeviceSize unusedSize
       Total number of bytes in the pool not used by any VmaAllocation. More...
       
      size_t allocationCount
       Number of VmaAllocation objects created from this pool that were not destroyed or lost. More...
       
      size_t unusedRangeCount
       Number of continuous memory ranges in the pool not used by any VmaAllocation. More...
       
      +

      Detailed Description

      +

      Describes parameter of existing VmaPool.

      +

      Member Data Documentation

      + +

      ◆ allocationCount

      + +
      +
      + + + + +
      size_t VmaPoolStats::allocationCount
      +
      + +

      Number of VmaAllocation objects created from this pool that were not destroyed or lost.

      + +
      +
      + +

      ◆ size

      + +
      +
      + + + + +
      VkDeviceSize VmaPoolStats::size
      +
      + +

      Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.

      + +
      +
      + +

      ◆ unusedRangeCount

      + +
      +
      + + + + +
      size_t VmaPoolStats::unusedRangeCount
      +
      + +

      Number of continuous memory ranges in the pool not used by any VmaAllocation.

      + +
      +
      + +

      ◆ unusedSize

      + +
      +
      + + + + +
      VkDeviceSize VmaPoolStats::unusedSize
      +
      + +

      Total number of bytes in the pool not used by any VmaAllocation.

      + +
      +
      +
      The documentation for this struct was generated from the following file: +
      + +
      +Generated by   +doxygen + 1.8.13 +
      + + diff --git a/docs/html/struct_vma_stat_info-members.html b/docs/html/struct_vma_stat_info-members.html index 10ff724..2801b11 100644 --- a/docs/html/struct_vma_stat_info-members.html +++ b/docs/html/struct_vma_stat_info-members.html @@ -66,10 +66,10 @@ $(function() {

      This is the complete list of members for VmaStatInfo, including all inherited members.

      - - - - + + + + diff --git a/docs/html/struct_vma_stat_info.html b/docs/html/struct_vma_stat_info.html index 7c0a4d3..c206d82 100644 --- a/docs/html/struct_vma_stat_info.html +++ b/docs/html/struct_vma_stat_info.html @@ -70,22 +70,27 @@ $(function() {
      AllocationCountVmaStatInfo
      SuballocationCountVmaStatInfo
      SuballocationSizeAvgVmaStatInfo
      SuballocationSizeMaxVmaStatInfo
      SuballocationSizeMinVmaStatInfo
      AllocationSizeAvgVmaStatInfo
      AllocationSizeMaxVmaStatInfo
      AllocationSizeMinVmaStatInfo
      BlockCountVmaStatInfo
      UnusedBytesVmaStatInfo
      UnusedRangeCountVmaStatInfo
      UnusedRangeSizeAvgVmaStatInfo
      + + + + - - + + + - - - - - - + + + + + + @@ -106,62 +111,66 @@ Public Attributes

      Public Attributes

      uint32_t BlockCount
       Number of VkDeviceMemory Vulkan memory blocks allocated. More...
       
      uint32_t AllocationCount
       Number of VmaAllocation allocation objects allocated. More...
       
      uint32_t SuballocationCount
       
      uint32_t UnusedRangeCount
       Number of free ranges of memory between allocations. More...
       
      VkDeviceSize UsedBytes
       Total number of bytes occupied by all allocations. More...
       
      VkDeviceSize UnusedBytes
       Total number of bytes occupied by unused ranges. More...
       
      VkDeviceSize SuballocationSizeMin
       
      VkDeviceSize SuballocationSizeAvg
       
      VkDeviceSize SuballocationSizeMax
       
      VkDeviceSize AllocationSizeMin
       
      VkDeviceSize AllocationSizeAvg
       
      VkDeviceSize AllocationSizeMax
       
      VkDeviceSize UnusedRangeSizeMin
       
      VkDeviceSize UnusedRangeSizeAvg
      +

      Number of VmaAllocation allocation objects allocated.

      +
      - -

      ◆ SuballocationCount

      + +

      ◆ AllocationSizeAvg

      - +
      uint32_t VmaStatInfo::SuballocationCountVkDeviceSize VmaStatInfo::AllocationSizeAvg
      - -

      ◆ SuballocationSizeAvg

      + +

      ◆ AllocationSizeMax

      - +
      VkDeviceSize VmaStatInfo::SuballocationSizeAvgVkDeviceSize VmaStatInfo::AllocationSizeMax
      - -

      ◆ SuballocationSizeMax

      + +

      ◆ AllocationSizeMin

      - +
      VkDeviceSize VmaStatInfo::SuballocationSizeMaxVkDeviceSize VmaStatInfo::AllocationSizeMin
      - -

      ◆ SuballocationSizeMin

      + +

      ◆ BlockCount

      - +
      VkDeviceSize VmaStatInfo::SuballocationSizeMinuint32_t VmaStatInfo::BlockCount
      +

      Number of VkDeviceMemory Vulkan memory blocks allocated.

      +
      @@ -176,6 +185,8 @@ Public Attributes
      +

      Total number of bytes occupied by unused ranges.

      +
      @@ -190,6 +201,8 @@ Public Attributes
      +

      Number of free ranges of memory between allocations.

      +
      @@ -246,6 +259,8 @@ Public Attributes
      +

      Total number of bytes occupied by all allocations.

      +
      The documentation for this struct was generated from the following file:
        diff --git a/docs/html/vk__mem__alloc_8h.html b/docs/html/vk__mem__alloc_8h.html index db24e33..38b4c8d 100644 --- a/docs/html/vk__mem__alloc_8h.html +++ b/docs/html/vk__mem__alloc_8h.html @@ -85,10 +85,16 @@ Classes struct  VmaStats  General statistics from current state of Allocator. More...
          -struct  VmaMemoryRequirements +struct  VmaAllocationCreateInfo +  +struct  VmaPoolCreateInfo + Describes parameter of created VmaPool. More...
        +  +struct  VmaPoolStats + Describes parameter of existing VmaPool. More...
          struct  VmaAllocationInfo - Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
        + Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
          struct  VmaDefragmentationInfo  Optional configuration parameters to be passed to function vmaDefragment(). More...
        @@ -125,15 +131,26 @@ Typedefs   typedef enum VmaMemoryUsage VmaMemoryUsage   -typedef enum VmaMemoryRequirementFlagBits VmaMemoryRequirementFlagBits - Flags to be passed as VmaMemoryRequirements::flags. More...
        -  -typedef VkFlags VmaMemoryRequirementFlags -  -typedef struct VmaMemoryRequirements VmaMemoryRequirements -  +typedef enum VmaAllocationCreateFlagBits VmaAllocationCreateFlagBits + Flags to be passed as VmaAllocationCreateInfo::flags. More...
        +  +typedef VkFlags VmaAllocationCreateFlags +  +typedef struct VmaAllocationCreateInfo VmaAllocationCreateInfo +  +typedef enum VmaPoolCreateFlagBits VmaPoolCreateFlagBits + Flags to be passed as VmaPoolCreateInfo::flags. More...
        +  +typedef VkFlags VmaPoolCreateFlags +  +typedef struct VmaPoolCreateInfo VmaPoolCreateInfo + Describes parameter of created VmaPool. More...
        +  +typedef struct VmaPoolStats VmaPoolStats + Describes parameter of existing VmaPool. More...
        +  typedef struct VmaAllocationInfo VmaAllocationInfo - Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
        + Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). More...
          typedef struct VmaDefragmentationInfo VmaDefragmentationInfo  Optional configuration parameters to be passed to function vmaDefragment(). More...
        @@ -160,13 +177,24 @@ Enumerations
        }   -enum  VmaMemoryRequirementFlagBits { VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001, -VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002, -VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004, -VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF - } Flags to be passed as VmaMemoryRequirements::flags. More...
        +enum  VmaAllocationCreateFlagBits {
        +  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001, +VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002, +VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004, +VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008, +
        +  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010, +VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +
        + } Flags to be passed as VmaAllocationCreateInfo::flags. More...
        -  +  +enum  VmaPoolCreateFlagBits { VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001, +VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002, +VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF + } Flags to be passed as VmaPoolCreateInfo::flags. More...
        + +  @@ -183,6 +211,9 @@ Functions + + + @@ -191,18 +222,30 @@ Functions - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - + @@ -210,26 +253,29 @@ Functions + + + - + - + - - + + - - - + + +

        Functions

        void vmaGetMemoryTypeProperties (VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
         Given Memory Type Index, returns Property Flags of this memory type. More...
         
        void vmaSetCurrentFrameIndex (VmaAllocator allocator, uint32_t frameIndex)
         Sets index of the current frame. More...
         
        void vmaCalculateStats (VmaAllocator allocator, VmaStats *pStats)
         Retrieves statistics from current state of the Allocator. More...
         
         
        void vmaFreeStatsString (VmaAllocator allocator, char *pStatsString)
         
        VkResult vmaFindMemoryTypeIndex (VmaAllocator allocator, uint32_t memoryTypeBits, const VmaMemoryRequirements *pMemoryRequirements, uint32_t *pMemoryTypeIndex)
         
        VkResult vmaAllocateMemory (VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaMemoryRequirements *pVmaMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         General purpose memory allocation. More...
         
        VkResult vmaAllocateMemoryForBuffer (VmaAllocator allocator, VkBuffer buffer, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         
        VkResult vmaAllocateMemoryForImage (VmaAllocator allocator, VkImage image, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         Function similar to vmaAllocateMemoryForBuffer(). More...
         
        VkResult vmaFindMemoryTypeIndex (VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
         
        VkResult vmaCreatePool (VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
         Allocates Vulkan device memory and creates VmaPool object. More...
         
        void vmaDestroyPool (VmaAllocator allocator, VmaPool pool)
         Destroys VmaPool object and frees Vulkan device memory. More...
         
        void vmaGetPoolStats (VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
         Retrieves statistics of existing VmaPool object. More...
         
        void vmaMakePoolAllocationsLost (VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
         Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now. More...
         
        VkResult vmaAllocateMemory (VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         General purpose memory allocation. More...
         
        VkResult vmaAllocateMemoryForBuffer (VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         
        VkResult vmaAllocateMemoryForImage (VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         Function similar to vmaAllocateMemoryForBuffer(). More...
         
        void vmaFreeMemory (VmaAllocator allocator, VmaAllocation allocation)
         Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). More...
         Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). More...
         
        void vmaGetAllocationInfo (VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
         Returns current information about specified allocation. More...
        void vmaSetAllocationUserData (VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
         Sets pUserData in given allocation to new value. More...
         
        void vmaCreateLostAllocation (VmaAllocator allocator, VmaAllocation *pAllocation)
         Creates new allocation that is in lost state from the beginning. More...
         
        VkResult vmaMapMemory (VmaAllocator allocator, VmaAllocation allocation, void **ppData)
         
        void vmaUnmapMemory (VmaAllocator allocator, VmaAllocation allocation)
         
        void vmaUnmapPersistentlyMappedMemory (VmaAllocator allocator)
         Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. More...
         Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL. More...
         
        VkResult vmaMapPersistentlyMappedMemory (VmaAllocator allocator)
         Maps back persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. More...
         Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL. More...
         
        VkResult vmaDefragment (VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
         Compacts memory by moving allocations. More...
         
        VkResult vmaCreateBuffer (VmaAllocator allocator, const VkBufferCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         
        VkResult vmaCreateBuffer (VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         
        void vmaDestroyBuffer (VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
         
        VkResult vmaCreateImage (VmaAllocator allocator, const VkImageCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         Function similar to vmaCreateBuffer(). More...
         
        VkResult vmaCreateImage (VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
         Function similar to vmaCreateBuffer(). More...
         
        void vmaDestroyImage (VmaAllocator allocator, VkImage image, VmaAllocation allocation)
         
        diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 8378926..6c043ec 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,104 +62,133 @@ $(function() {
        vk_mem_alloc.h
        -Go to the documentation of this file.
        1 //
        2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
        3 //
        4 // Permission is hereby granted, free of charge, to any person obtaining a copy
        5 // of this software and associated documentation files (the "Software"), to deal
        6 // in the Software without restriction, including without limitation the rights
        7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
        8 // copies of the Software, and to permit persons to whom the Software is
        9 // furnished to do so, subject to the following conditions:
        10 //
        11 // The above copyright notice and this permission notice shall be included in
        12 // all copies or substantial portions of the Software.
        13 //
        14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
        15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
        16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
        17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
        18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
        19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
        20 // THE SOFTWARE.
        21 //
        22 
        23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
        24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
        25 
        161 #include <vulkan/vulkan.h>
        162 
        164 
        168 VK_DEFINE_HANDLE(VmaAllocator)
        169 
        170 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
        172  VmaAllocator allocator,
        173  uint32_t memoryType,
        174  VkDeviceMemory memory,
        175  VkDeviceSize size);
        177 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
        178  VmaAllocator allocator,
        179  uint32_t memoryType,
        180  VkDeviceMemory memory,
        181  VkDeviceSize size);
        182 
        188 typedef struct VmaDeviceMemoryCallbacks {
        190  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
        192  PFN_vmaFreeDeviceMemoryFunction pfnFree;
        193 } VmaDeviceMemoryCallbacks;
        194 
        196 typedef enum VmaAllocatorFlagBits {
        201  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
        202 
        203  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
        204 } VmaAllocatorFlagBits;
        205 typedef VkFlags VmaAllocatorFlags;
        206 
        208 typedef struct VmaAllocatorCreateInfo
        209 {
        211  VmaAllocatorFlags flags;
        213 
        214  VkPhysicalDevice physicalDevice;
        216 
        217  VkDevice device;
        219 
        220  VkDeviceSize preferredLargeHeapBlockSize;
        222 
        223  VkDeviceSize preferredSmallHeapBlockSize;
        225 
        226  const VkAllocationCallbacks* pAllocationCallbacks;
        228 
        229  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
        230 } VmaAllocatorCreateInfo;
        231 
        233 VkResult vmaCreateAllocator(
        234  const VmaAllocatorCreateInfo* pCreateInfo,
        235  VmaAllocator* pAllocator);
        236 
        238 void vmaDestroyAllocator(
        239  VmaAllocator allocator);
        240 
        245 void vmaGetPhysicalDeviceProperties(
        246  VmaAllocator allocator,
        247  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
        248 
        253 void vmaGetMemoryProperties(
        254  VmaAllocator allocator,
        255  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
        256 
        263 void vmaGetMemoryTypeProperties(
        264  VmaAllocator allocator,
        265  uint32_t memoryTypeIndex,
        266  VkMemoryPropertyFlags* pFlags);
        267 
        268 typedef struct VmaStatInfo
        269 {
        270  uint32_t AllocationCount;
        271  uint32_t SuballocationCount;
        272  uint32_t UnusedRangeCount;
        273  VkDeviceSize UsedBytes;
        274  VkDeviceSize UnusedBytes;
        275  VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax;
        276  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
        277 } VmaStatInfo;
        278 
        280 struct VmaStats
        281 {
        282  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
        283  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
        284  VmaStatInfo total;
        285 };
        286 
        288 void vmaCalculateStats(
        289  VmaAllocator allocator,
        290  VmaStats* pStats);
        291 
        292 #define VMA_STATS_STRING_ENABLED 1
        293 
        294 #if VMA_STATS_STRING_ENABLED
        295 
        297 
        299 void vmaBuildStatsString(
        300  VmaAllocator allocator,
        301  char** ppStatsString,
        302  VkBool32 detailedMap);
        303 
        304 void vmaFreeStatsString(
        305  VmaAllocator allocator,
        306  char* pStatsString);
        307 
        308 #endif // #if VMA_STATS_STRING_ENABLED
        309 
        312 
        317 typedef enum VmaMemoryUsage
        318 {
        320  VMA_MEMORY_USAGE_UNKNOWN = 0,
        322  VMA_MEMORY_USAGE_GPU_ONLY = 1,
        324 
        325  VMA_MEMORY_USAGE_CPU_ONLY = 2,
        327 
        328  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
        330 
        331  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
        332  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
        333 } VmaMemoryUsage;
        334 
        336 typedef enum VmaMemoryRequirementFlagBits {
        345  VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001,
        346 
        354  VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002,
        361  VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004,
        362 
        363  VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
        364 } VmaMemoryRequirementFlagBits;
        365 typedef VkFlags VmaMemoryRequirementFlags;
        366 
        367 typedef struct VmaMemoryRequirements
        368 {
        369  VmaMemoryRequirementFlags flags;
        374  VmaMemoryUsage usage;
        378  VkMemoryPropertyFlags requiredFlags;
        383  VkMemoryPropertyFlags preferredFlags;
        385  void* pUserData;
        386 } VmaMemoryRequirements;
        387 
        402 VkResult vmaFindMemoryTypeIndex(
        403  VmaAllocator allocator,
        404  uint32_t memoryTypeBits,
        405  const VmaMemoryRequirements* pMemoryRequirements,
        406  uint32_t* pMemoryTypeIndex);
        407 
        410 
        415 VK_DEFINE_HANDLE(VmaAllocation)
        416 
        417 
        419 typedef struct VmaAllocationInfo {
        424  uint32_t memoryType;
        431  VkDeviceMemory deviceMemory;
        436  VkDeviceSize offset;
        441  VkDeviceSize size;
        447  void* pMappedData;
        452  void* pUserData;
        453 } VmaAllocationInfo;
        454 
        465 VkResult vmaAllocateMemory(
        466  VmaAllocator allocator,
        467  const VkMemoryRequirements* pVkMemoryRequirements,
        468  const VmaMemoryRequirements* pVmaMemoryRequirements,
        469  VmaAllocation* pAllocation,
        470  VmaAllocationInfo* pAllocationInfo);
        471 
        478 VkResult vmaAllocateMemoryForBuffer(
        479  VmaAllocator allocator,
        480  VkBuffer buffer,
        481  const VmaMemoryRequirements* pMemoryRequirements,
        482  VmaAllocation* pAllocation,
        483  VmaAllocationInfo* pAllocationInfo);
        484 
        486 VkResult vmaAllocateMemoryForImage(
        487  VmaAllocator allocator,
        488  VkImage image,
        489  const VmaMemoryRequirements* pMemoryRequirements,
        490  VmaAllocation* pAllocation,
        491  VmaAllocationInfo* pAllocationInfo);
        492 
        494 void vmaFreeMemory(
        495  VmaAllocator allocator,
        496  VmaAllocation allocation);
        497 
        499 void vmaGetAllocationInfo(
        500  VmaAllocator allocator,
        501  VmaAllocation allocation,
        502  VmaAllocationInfo* pAllocationInfo);
        503 
        505 void vmaSetAllocationUserData(
        506  VmaAllocator allocator,
        507  VmaAllocation allocation,
        508  void* pUserData);
        509 
        518 VkResult vmaMapMemory(
        519  VmaAllocator allocator,
        520  VmaAllocation allocation,
        521  void** ppData);
        522 
        523 void vmaUnmapMemory(
        524  VmaAllocator allocator,
        525  VmaAllocation allocation);
        526 
        545 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
        546 
        554 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
        555 
        557 typedef struct VmaDefragmentationInfo {
        562  VkDeviceSize maxBytesToMove;
        567  uint32_t maxAllocationsToMove;
        568 } VmaDefragmentationInfo;
        569 
        571 typedef struct VmaDefragmentationStats {
        573  VkDeviceSize bytesMoved;
        575  VkDeviceSize bytesFreed;
        577  uint32_t allocationsMoved;
        579  uint32_t deviceMemoryBlocksFreed;
        580 } VmaDefragmentationStats;
        581 
        652 VkResult vmaDefragment(
        653  VmaAllocator allocator,
        654  VmaAllocation* pAllocations,
        655  size_t allocationCount,
        656  VkBool32* pAllocationsChanged,
        657  const VmaDefragmentationInfo *pDefragmentationInfo,
        658  VmaDefragmentationStats* pDefragmentationStats);
        659 
        662 
        685 VkResult vmaCreateBuffer(
        686  VmaAllocator allocator,
        687  const VkBufferCreateInfo* pCreateInfo,
        688  const VmaMemoryRequirements* pMemoryRequirements,
        689  VkBuffer* pBuffer,
        690  VmaAllocation* pAllocation,
        691  VmaAllocationInfo* pAllocationInfo);
        692 
        693 void vmaDestroyBuffer(
        694  VmaAllocator allocator,
        695  VkBuffer buffer,
        696  VmaAllocation allocation);
        697 
        699 VkResult vmaCreateImage(
        700  VmaAllocator allocator,
        701  const VkImageCreateInfo* pCreateInfo,
        702  const VmaMemoryRequirements* pMemoryRequirements,
        703  VkImage* pImage,
        704  VmaAllocation* pAllocation,
        705  VmaAllocationInfo* pAllocationInfo);
        706 
        707 void vmaDestroyImage(
        708  VmaAllocator allocator,
        709  VkImage image,
        710  VmaAllocation allocation);
        711 
        714 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
        715 
        716 #ifdef VMA_IMPLEMENTATION
        717 #undef VMA_IMPLEMENTATION
        718 
        719 #include <cstdint>
        720 #include <cstdlib>
        721 
        722 /*******************************************************************************
        723 CONFIGURATION SECTION
        724 
        725 Define some of these macros before each #include of this header or change them
        726 here if you need other then default behavior depending on your environment.
        727 */
        728 
        729 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
        730 //#define VMA_USE_STL_CONTAINERS 1
        731 
        732 /* Set this macro to 1 to make the library including and using STL containers:
        733 std::pair, std::vector, std::list, std::unordered_map.
        734 
        735 Set it to 0 or undefined to make the library using its own implementation of
        736 the containers.
        737 */
        738 #if VMA_USE_STL_CONTAINERS
        739  #define VMA_USE_STL_VECTOR 1
        740  #define VMA_USE_STL_UNORDERED_MAP 1
        741  #define VMA_USE_STL_LIST 1
        742 #endif
        743 
        744 #if VMA_USE_STL_VECTOR
        745  #include <vector>
        746 #endif
        747 
        748 #if VMA_USE_STL_UNORDERED_MAP
        749  #include <unordered_map>
        750 #endif
        751 
        752 #if VMA_USE_STL_LIST
        753  #include <list>
        754 #endif
        755 
        756 /*
        757 Following headers are used in this CONFIGURATION section only, so feel free to
        758 remove them if not needed.
        759 */
        760 #include <cassert> // for assert
        761 #include <algorithm> // for min, max
        762 #include <mutex> // for std::mutex
        763 
        764 #if !defined(_WIN32)
        765  #include <malloc.h> // for aligned_alloc()
        766 #endif
        767 
        768 // Normal assert to check for programmer's errors, especially in Debug configuration.
        769 #ifndef VMA_ASSERT
        770  #ifdef _DEBUG
        771  #define VMA_ASSERT(expr) assert(expr)
        772  #else
        773  #define VMA_ASSERT(expr)
        774  #endif
        775 #endif
        776 
        777 // Assert that will be called very often, like inside data structures e.g. operator[].
        778 // Making it non-empty can make program slow.
        779 #ifndef VMA_HEAVY_ASSERT
        780  #ifdef _DEBUG
        781  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
        782  #else
        783  #define VMA_HEAVY_ASSERT(expr)
        784  #endif
        785 #endif
        786 
        787 #ifndef VMA_NULL
        788  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
        789  #define VMA_NULL nullptr
        790 #endif
        791 
        792 #ifndef VMA_ALIGN_OF
        793  #define VMA_ALIGN_OF(type) (__alignof(type))
        794 #endif
        795 
        796 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
        797  #if defined(_WIN32)
        798  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
        799  #else
        800  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
        801  #endif
        802 #endif
        803 
        804 #ifndef VMA_SYSTEM_FREE
        805  #if defined(_WIN32)
        806  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
        807  #else
        808  #define VMA_SYSTEM_FREE(ptr) free(ptr)
        809  #endif
        810 #endif
        811 
        812 #ifndef VMA_MIN
        813  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
        814 #endif
        815 
        816 #ifndef VMA_MAX
        817  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
        818 #endif
        819 
        820 #ifndef VMA_SWAP
        821  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
        822 #endif
        823 
        824 #ifndef VMA_SORT
        825  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
        826 #endif
        827 
        828 #ifndef VMA_DEBUG_LOG
        829  #define VMA_DEBUG_LOG(format, ...)
        830  /*
        831  #define VMA_DEBUG_LOG(format, ...) do { \
        832  printf(format, __VA_ARGS__); \
        833  printf("\n"); \
        834  } while(false)
        835  */
        836 #endif
        837 
        838 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
        839 #if VMA_STATS_STRING_ENABLED
        840  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
        841  {
        842  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
        843  }
        844  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
        845  {
        846  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
        847  }
        848 #endif
        849 
        850 #ifndef VMA_MUTEX
        851  class VmaMutex
        852  {
        853  public:
        854  VmaMutex() { }
        855  ~VmaMutex() { }
        856  void Lock() { m_Mutex.lock(); }
        857  void Unlock() { m_Mutex.unlock(); }
        858  private:
        859  std::mutex m_Mutex;
        860  };
        861  #define VMA_MUTEX VmaMutex
        862 #endif
        863 
        864 #ifndef VMA_BEST_FIT
        865 
        877  #define VMA_BEST_FIT (1)
        878 #endif
        879 
        880 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
        881 
        885  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
        886 #endif
        887 
        888 #ifndef VMA_DEBUG_ALIGNMENT
        889 
        893  #define VMA_DEBUG_ALIGNMENT (1)
        894 #endif
        895 
        896 #ifndef VMA_DEBUG_MARGIN
        897 
        901  #define VMA_DEBUG_MARGIN (0)
        902 #endif
        903 
        904 #ifndef VMA_DEBUG_GLOBAL_MUTEX
        905 
        909  #define VMA_DEBUG_GLOBAL_MUTEX (0)
        910 #endif
        911 
        912 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
        913 
        917  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
        918 #endif
        919 
        920 #ifndef VMA_SMALL_HEAP_MAX_SIZE
        921  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
        923 #endif
        924 
        925 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
        926  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
        928 #endif
        929 
        930 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
        931  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
        933 #endif
        934 
        935 /*******************************************************************************
        936 END OF CONFIGURATION
        937 */
        938 
        939 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
        940  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
        941 
        942 // Returns number of bits set to 1 in (v).
        943 static inline uint32_t CountBitsSet(uint32_t v)
        944 {
        945  uint32_t c = v - ((v >> 1) & 0x55555555);
        946  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
        947  c = ((c >> 4) + c) & 0x0F0F0F0F;
        948  c = ((c >> 8) + c) & 0x00FF00FF;
        949  c = ((c >> 16) + c) & 0x0000FFFF;
        950  return c;
        951 }
        952 
        953 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
        954 // Use types like uint32_t, uint64_t as T.
        955 template <typename T>
        956 static inline T VmaAlignUp(T val, T align)
        957 {
        958  return (val + align - 1) / align * align;
        959 }
        960 
        961 // Division with mathematical rounding to nearest number.
        962 template <typename T>
        963 inline T VmaRoundDiv(T x, T y)
        964 {
        965  return (x + (y / (T)2)) / y;
        966 }
        967 
        968 #ifndef VMA_SORT
        969 
        970 template<typename Iterator, typename Compare>
        971 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
        972 {
        973  Iterator centerValue = end; --centerValue;
        974  Iterator insertIndex = beg;
        975  for(Iterator i = beg; i < centerValue; ++i)
        976  {
        977  if(cmp(*i, *centerValue))
        978  {
        979  if(insertIndex != i)
        980  {
        981  VMA_SWAP(*i, *insertIndex);
        982  }
        983  ++insertIndex;
        984  }
        985  }
        986  if(insertIndex != centerValue)
        987  {
        988  VMA_SWAP(*insertIndex, *centerValue);
        989  }
        990  return insertIndex;
        991 }
        992 
        993 template<typename Iterator, typename Compare>
        994 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
        995 {
        996  if(beg < end)
        997  {
        998  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
        999  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
        1000  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
        1001  }
        1002 }
        1003 
        1004 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
        1005 
        1006 #endif // #ifndef VMA_SORT
        1007 
        1008 /*
        1009 Returns true if two memory blocks occupy overlapping pages.
        1010 ResourceA must be in less memory offset than ResourceB.
        1011 
        1012 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
        1013 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
        1014 */
        1015 static inline bool VmaBlocksOnSamePage(
        1016  VkDeviceSize resourceAOffset,
        1017  VkDeviceSize resourceASize,
        1018  VkDeviceSize resourceBOffset,
        1019  VkDeviceSize pageSize)
        1020 {
        1021  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
        1022  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
        1023  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
        1024  VkDeviceSize resourceBStart = resourceBOffset;
        1025  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
        1026  return resourceAEndPage == resourceBStartPage;
        1027 }
        1028 
        1029 enum VmaSuballocationType
        1030 {
        1031  VMA_SUBALLOCATION_TYPE_FREE = 0,
        1032  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
        1033  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
        1034  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
        1035  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
        1036  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
        1037  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
        1038 };
        1039 
        1040 /*
        1041 Returns true if given suballocation types could conflict and must respect
        1042 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
        1043 or linear image and another one is optimal image. If type is unknown, behave
        1044 conservatively.
        1045 */
        1046 static inline bool VmaIsBufferImageGranularityConflict(
        1047  VmaSuballocationType suballocType1,
        1048  VmaSuballocationType suballocType2)
        1049 {
        1050  if(suballocType1 > suballocType2)
        1051  {
        1052  VMA_SWAP(suballocType1, suballocType2);
        1053  }
        1054 
        1055  switch(suballocType1)
        1056  {
        1057  case VMA_SUBALLOCATION_TYPE_FREE:
        1058  return false;
        1059  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
        1060  return true;
        1061  case VMA_SUBALLOCATION_TYPE_BUFFER:
        1062  return
        1063  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
        1064  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1065  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
        1066  return
        1067  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
        1068  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
        1069  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1070  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
        1071  return
        1072  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1073  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
        1074  return false;
        1075  default:
        1076  VMA_ASSERT(0);
        1077  return true;
        1078  }
        1079 }
        1080 
        1081 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
        1082 struct VmaMutexLock
        1083 {
        1084 public:
        1085  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
        1086  m_pMutex(useMutex ? &mutex : VMA_NULL)
        1087  {
        1088  if(m_pMutex)
        1089  {
        1090  m_pMutex->Lock();
        1091  }
        1092  }
        1093 
        1094  ~VmaMutexLock()
        1095  {
        1096  if(m_pMutex)
        1097  {
        1098  m_pMutex->Unlock();
        1099  }
        1100  }
        1101 
        1102 private:
        1103  VMA_MUTEX* m_pMutex;
        1104 };
        1105 
        1106 #if VMA_DEBUG_GLOBAL_MUTEX
        1107  static VMA_MUTEX gDebugGlobalMutex;
        1108  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex);
        1109 #else
        1110  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
        1111 #endif
        1112 
        1113 // Minimum size of a free suballocation to register it in the free suballocation collection.
        1114 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
        1115 
        1116 /*
        1117 Performs binary search and returns iterator to first element that is greater or
        1118 equal to (key), according to comparison (cmp).
        1119 
        1120 Cmp should return true if first argument is less than second argument.
        1121 
        1122 Returned value is the found element, if present in the collection or place where
        1123 new element with value (key) should be inserted.
        1124 */
        1125 template <typename IterT, typename KeyT, typename CmpT>
        1126 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
        1127 {
        1128  size_t down = 0, up = (end - beg);
        1129  while(down < up)
        1130  {
        1131  const size_t mid = (down + up) / 2;
        1132  if(cmp(*(beg+mid), key))
        1133  {
        1134  down = mid + 1;
        1135  }
        1136  else
        1137  {
        1138  up = mid;
        1139  }
        1140  }
        1141  return beg + down;
        1142 }
        1143 
        1145 // Memory allocation
        1146 
        1147 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
        1148 {
        1149  if((pAllocationCallbacks != VMA_NULL) &&
        1150  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
        1151  {
        1152  return (*pAllocationCallbacks->pfnAllocation)(
        1153  pAllocationCallbacks->pUserData,
        1154  size,
        1155  alignment,
        1156  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
        1157  }
        1158  else
        1159  {
        1160  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
        1161  }
        1162 }
        1163 
        1164 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
        1165 {
        1166  if((pAllocationCallbacks != VMA_NULL) &&
        1167  (pAllocationCallbacks->pfnFree != VMA_NULL))
        1168  {
        1169  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
        1170  }
        1171  else
        1172  {
        1173  VMA_SYSTEM_FREE(ptr);
        1174  }
        1175 }
        1176 
        1177 template<typename T>
        1178 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
        1179 {
        1180  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
        1181 }
        1182 
        1183 template<typename T>
        1184 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
        1185 {
        1186  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
        1187 }
        1188 
        1189 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
        1190 
        1191 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
        1192 
        1193 template<typename T>
        1194 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
        1195 {
        1196  ptr->~T();
        1197  VmaFree(pAllocationCallbacks, ptr);
        1198 }
        1199 
        1200 template<typename T>
        1201 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
        1202 {
        1203  if(ptr != VMA_NULL)
        1204  {
        1205  for(size_t i = count; i--; )
        1206  {
        1207  ptr[i].~T();
        1208  }
        1209  VmaFree(pAllocationCallbacks, ptr);
        1210  }
        1211 }
        1212 
        1213 // STL-compatible allocator.
        1214 template<typename T>
        1215 class VmaStlAllocator
        1216 {
        1217 public:
        1218  const VkAllocationCallbacks* const m_pCallbacks;
        1219  typedef T value_type;
        1220 
        1221  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
        1222  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
        1223 
        1224  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
        1225  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
        1226 
        1227  template<typename U>
        1228  bool operator==(const VmaStlAllocator<U>& rhs) const
        1229  {
        1230  return m_pCallbacks == rhs.m_pCallbacks;
        1231  }
        1232  template<typename U>
        1233  bool operator!=(const VmaStlAllocator<U>& rhs) const
        1234  {
        1235  return m_pCallbacks != rhs.m_pCallbacks;
        1236  }
        1237 
        1238  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
        1239 };
        1240 
        1241 #if VMA_USE_STL_VECTOR
        1242 
        1243 #define VmaVector std::vector
        1244 
        1245 template<typename T, typename allocatorT>
        1246 static void VectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
        1247 {
        1248  vec.insert(vec.begin() + index, item);
        1249 }
        1250 
        1251 template<typename T, typename allocatorT>
        1252 static void VectorRemove(std::vector<T, allocatorT>& vec, size_t index)
        1253 {
        1254  vec.erase(vec.begin() + index);
        1255 }
        1256 
        1257 #else // #if VMA_USE_STL_VECTOR
        1258 
        1259 /* Class with interface compatible with subset of std::vector.
        1260 T must be POD because constructors and destructors are not called and memcpy is
        1261 used for these objects. */
        1262 template<typename T, typename AllocatorT>
        1263 class VmaVector
        1264 {
        1265 public:
        1266  VmaVector(const AllocatorT& allocator) :
        1267  m_Allocator(allocator),
        1268  m_pArray(VMA_NULL),
        1269  m_Count(0),
        1270  m_Capacity(0)
        1271  {
        1272  }
        1273 
        1274  VmaVector(size_t count, const AllocatorT& allocator) :
        1275  m_Allocator(allocator),
        1276  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator->m_pCallbacks, count) : VMA_NULL),
        1277  m_Count(count),
        1278  m_Capacity(count)
        1279  {
        1280  }
        1281 
        1282  VmaVector(const VmaVector<T, AllocatorT>& src) :
        1283  m_Allocator(src.m_Allocator),
        1284  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src->m_pCallbacks, src.m_Count) : VMA_NULL),
        1285  m_Count(src.m_Count),
        1286  m_Capacity(src.m_Count)
        1287  {
        1288  if(m_Count != 0)
        1289  {
        1290  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
        1291  }
        1292  }
        1293 
        1294  ~VmaVector()
        1295  {
        1296  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1297  }
        1298 
        1299  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
        1300  {
        1301  if(&rhs != this)
        1302  {
        1303  Resize(rhs.m_Count);
        1304  if(m_Count != 0)
        1305  {
        1306  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
        1307  }
        1308  }
        1309  return *this;
        1310  }
        1311 
        1312  bool empty() const { return m_Count == 0; }
        1313  size_t size() const { return m_Count; }
        1314  T* data() { return m_pArray; }
        1315  const T* data() const { return m_pArray; }
        1316 
        1317  T& operator[](size_t index)
        1318  {
        1319  VMA_HEAVY_ASSERT(index < m_Count);
        1320  return m_pArray[index];
        1321  }
        1322  const T& operator[](size_t index) const
        1323  {
        1324  VMA_HEAVY_ASSERT(index < m_Count);
        1325  return m_pArray[index];
        1326  }
        1327 
        1328  T& front()
        1329  {
        1330  VMA_HEAVY_ASSERT(m_Count > 0);
        1331  return m_pArray[0];
        1332  }
        1333  const T& front() const
        1334  {
        1335  VMA_HEAVY_ASSERT(m_Count > 0);
        1336  return m_pArray[0];
        1337  }
        1338  T& back()
        1339  {
        1340  VMA_HEAVY_ASSERT(m_Count > 0);
        1341  return m_pArray[m_Count - 1];
        1342  }
        1343  const T& back() const
        1344  {
        1345  VMA_HEAVY_ASSERT(m_Count > 0);
        1346  return m_pArray[m_Count - 1];
        1347  }
        1348 
        1349  void reserve(size_t newCapacity, bool freeMemory = false)
        1350  {
        1351  newCapacity = VMA_MAX(newCapacity, m_Count);
        1352 
        1353  if((newCapacity < m_Capacity) && !freeMemory)
        1354  {
        1355  newCapacity = m_Capacity;
        1356  }
        1357 
        1358  if(newCapacity != m_Capacity)
        1359  {
        1360  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
        1361  if(m_Count != 0)
        1362  {
        1363  memcpy(newArray, m_pArray, m_Count * sizeof(T));
        1364  }
        1365  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1366  m_Capacity = newCapacity;
        1367  m_pArray = newArray;
        1368  }
        1369  }
        1370 
        1371  void resize(size_t newCount, bool freeMemory = false)
        1372  {
        1373  size_t newCapacity = m_Capacity;
        1374  if(newCount > m_Capacity)
        1375  {
        1376  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
        1377  }
        1378  else if(freeMemory)
        1379  {
        1380  newCapacity = newCount;
        1381  }
        1382 
        1383  if(newCapacity != m_Capacity)
        1384  {
        1385  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
        1386  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
        1387  if(elementsToCopy != 0)
        1388  {
        1389  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
        1390  }
        1391  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1392  m_Capacity = newCapacity;
        1393  m_pArray = newArray;
        1394  }
        1395 
        1396  m_Count = newCount;
        1397  }
        1398 
        1399  void clear(bool freeMemory = false)
        1400  {
        1401  resize(0, freeMemory);
        1402  }
        1403 
        1404  void insert(size_t index, const T& src)
        1405  {
        1406  VMA_HEAVY_ASSERT(index <= m_Count);
        1407  const size_t oldCount = size();
        1408  resize(oldCount + 1);
        1409  if(index < oldCount)
        1410  {
        1411  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
        1412  }
        1413  m_pArray[index] = src;
        1414  }
        1415 
        1416  void remove(size_t index)
        1417  {
        1418  VMA_HEAVY_ASSERT(index < m_Count);
        1419  const size_t oldCount = size();
        1420  if(index < oldCount - 1)
        1421  {
        1422  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
        1423  }
        1424  resize(oldCount - 1);
        1425  }
        1426 
        1427  void push_back(const T& src)
        1428  {
        1429  const size_t newIndex = size();
        1430  resize(newIndex + 1);
        1431  m_pArray[newIndex] = src;
        1432  }
        1433 
        1434  void pop_back()
        1435  {
        1436  VMA_HEAVY_ASSERT(m_Count > 0);
        1437  resize(size() - 1);
        1438  }
        1439 
        1440  void push_front(const T& src)
        1441  {
        1442  insert(0, src);
        1443  }
        1444 
        1445  void pop_front()
        1446  {
        1447  VMA_HEAVY_ASSERT(m_Count > 0);
        1448  remove(0);
        1449  }
        1450 
        1451  typedef T* iterator;
        1452 
        1453  iterator begin() { return m_pArray; }
        1454  iterator end() { return m_pArray + m_Count; }
        1455 
        1456 private:
        1457  AllocatorT m_Allocator;
        1458  T* m_pArray;
        1459  size_t m_Count;
        1460  size_t m_Capacity;
        1461 };
        1462 
        1463 template<typename T, typename allocatorT>
        1464 static void VectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
        1465 {
        1466  vec.insert(index, item);
        1467 }
        1468 
        1469 template<typename T, typename allocatorT>
        1470 static void VectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
        1471 {
        1472  vec.remove(index);
        1473 }
        1474 
        1475 #endif // #if VMA_USE_STL_VECTOR
        1476 
        1478 // class VmaPoolAllocator
        1479 
        1480 /*
        1481 Allocator for objects of type T using a list of arrays (pools) to speed up
        1482 allocation. Number of elements that can be allocated is not bounded because
        1483 allocator can create multiple blocks.
        1484 */
        1485 template<typename T>
        1486 class VmaPoolAllocator
        1487 {
        1488 public:
        1489  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
        1490  ~VmaPoolAllocator();
        1491  void Clear();
        1492  T* Alloc();
        1493  void Free(T* ptr);
        1494 
        1495 private:
        1496  union Item
        1497  {
        1498  uint32_t NextFreeIndex;
        1499  T Value;
        1500  };
        1501 
        1502  struct ItemBlock
        1503  {
        1504  Item* pItems;
        1505  uint32_t FirstFreeIndex;
        1506  };
        1507 
        1508  const VkAllocationCallbacks* m_pAllocationCallbacks;
        1509  size_t m_ItemsPerBlock;
        1510  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
        1511 
        1512  ItemBlock& CreateNewBlock();
        1513 };
        1514 
        1515 template<typename T>
        1516 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
        1517  m_pAllocationCallbacks(pAllocationCallbacks),
        1518  m_ItemsPerBlock(itemsPerBlock),
        1519  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
        1520 {
        1521  VMA_ASSERT(itemsPerBlock > 0);
        1522 }
        1523 
        1524 template<typename T>
        1525 VmaPoolAllocator<T>::~VmaPoolAllocator()
        1526 {
        1527  Clear();
        1528 }
        1529 
        1530 template<typename T>
        1531 void VmaPoolAllocator<T>::Clear()
        1532 {
        1533  for(size_t i = m_ItemBlocks.size(); i--; )
        1534  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
        1535  m_ItemBlocks.clear();
        1536 }
        1537 
        1538 template<typename T>
        1539 T* VmaPoolAllocator<T>::Alloc()
        1540 {
        1541  for(size_t i = m_ItemBlocks.size(); i--; )
        1542  {
        1543  ItemBlock& block = m_ItemBlocks[i];
        1544  // This block has some free items: Use first one.
        1545  if(block.FirstFreeIndex != UINT32_MAX)
        1546  {
        1547  Item* const pItem = &block.pItems[block.FirstFreeIndex];
        1548  block.FirstFreeIndex = pItem->NextFreeIndex;
        1549  return &pItem->Value;
        1550  }
        1551  }
        1552 
        1553  // No block has free item: Create new one and use it.
        1554  ItemBlock& newBlock = CreateNewBlock();
        1555  Item* const pItem = &newBlock.pItems[0];
        1556  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
        1557  return &pItem->Value;
        1558 }
        1559 
        1560 template<typename T>
        1561 void VmaPoolAllocator<T>::Free(T* ptr)
        1562 {
        1563  // Search all memory blocks to find ptr.
        1564  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
        1565  {
        1566  ItemBlock& block = m_ItemBlocks[i];
        1567 
        1568  // Casting to union.
        1569  Item* pItemPtr;
        1570  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
        1571 
        1572  // Check if pItemPtr is in address range of this block.
        1573  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
        1574  {
        1575  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
        1576  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
        1577  block.FirstFreeIndex = index;
        1578  return;
        1579  }
        1580  }
        1581  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
        1582 }
        1583 
        1584 template<typename T>
        1585 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
        1586 {
        1587  ItemBlock newBlock = {
        1588  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
        1589 
        1590  m_ItemBlocks.push_back(newBlock);
        1591 
        1592  // Setup singly-linked list of all free items in this block.
        1593  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
        1594  newBlock.pItems[i].NextFreeIndex = i + 1;
        1595  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
        1596  return m_ItemBlocks.back();
        1597 }
        1598 
        1600 // class VmaRawList, VmaList
        1601 
        1602 #if VMA_USE_STL_LIST
        1603 
        1604 #define VmaList std::list
        1605 
        1606 #else // #if VMA_USE_STL_LIST
        1607 
        1608 template<typename T>
        1609 struct VmaListItem
        1610 {
        1611  VmaListItem* pPrev;
        1612  VmaListItem* pNext;
        1613  T Value;
        1614 };
        1615 
        1616 // Doubly linked list.
        1617 template<typename T>
        1618 class VmaRawList
        1619 {
        1620 public:
        1621  typedef VmaListItem<T> ItemType;
        1622 
        1623  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
        1624  ~VmaRawList();
        1625  void Clear();
        1626 
        1627  size_t GetCount() const { return m_Count; }
        1628  bool IsEmpty() const { return m_Count == 0; }
        1629 
        1630  ItemType* Front() { return m_pFront; }
        1631  const ItemType* Front() const { return m_pFront; }
        1632  ItemType* Back() { return m_pBack; }
        1633  const ItemType* Back() const { return m_pBack; }
        1634 
        1635  ItemType* PushBack();
        1636  ItemType* PushFront();
        1637  ItemType* PushBack(const T& value);
        1638  ItemType* PushFront(const T& value);
        1639  void PopBack();
        1640  void PopFront();
        1641 
        1642  // Item can be null - it means PushBack.
        1643  ItemType* InsertBefore(ItemType* pItem);
        1644  // Item can be null - it means PushFront.
        1645  ItemType* InsertAfter(ItemType* pItem);
        1646 
        1647  ItemType* InsertBefore(ItemType* pItem, const T& value);
        1648  ItemType* InsertAfter(ItemType* pItem, const T& value);
        1649 
        1650  void Remove(ItemType* pItem);
        1651 
        1652 private:
        1653  const VkAllocationCallbacks* const m_pAllocationCallbacks;
        1654  VmaPoolAllocator<ItemType> m_ItemAllocator;
        1655  ItemType* m_pFront;
        1656  ItemType* m_pBack;
        1657  size_t m_Count;
        1658 
        1659  // Declared not defined, to block copy constructor and assignment operator.
        1660  VmaRawList(const VmaRawList<T>& src);
        1661  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
        1662 };
        1663 
        1664 template<typename T>
        1665 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
        1666  m_pAllocationCallbacks(pAllocationCallbacks),
        1667  m_ItemAllocator(pAllocationCallbacks, 128),
        1668  m_pFront(VMA_NULL),
        1669  m_pBack(VMA_NULL),
        1670  m_Count(0)
        1671 {
        1672 }
        1673 
        1674 template<typename T>
        1675 VmaRawList<T>::~VmaRawList()
        1676 {
        1677  // Intentionally not calling Clear, because that would be unnecessary
        1678  // computations to return all items to m_ItemAllocator as free.
        1679 }
        1680 
        1681 template<typename T>
        1682 void VmaRawList<T>::Clear()
        1683 {
        1684  if(IsEmpty() == false)
        1685  {
        1686  ItemType* pItem = m_pBack;
        1687  while(pItem != VMA_NULL)
        1688  {
        1689  ItemType* const pPrevItem = pItem->pPrev;
        1690  m_ItemAllocator.Free(pItem);
        1691  pItem = pPrevItem;
        1692  }
        1693  m_pFront = VMA_NULL;
        1694  m_pBack = VMA_NULL;
        1695  m_Count = 0;
        1696  }
        1697 }
        1698 
        1699 template<typename T>
        1700 VmaListItem<T>* VmaRawList<T>::PushBack()
        1701 {
        1702  ItemType* const pNewItem = m_ItemAllocator.Alloc();
        1703  pNewItem->pNext = VMA_NULL;
        1704  if(IsEmpty())
        1705  {
        1706  pNewItem->pPrev = VMA_NULL;
        1707  m_pFront = pNewItem;
        1708  m_pBack = pNewItem;
        1709  m_Count = 1;
        1710  }
        1711  else
        1712  {
        1713  pNewItem->pPrev = m_pBack;
        1714  m_pBack->pNext = pNewItem;
        1715  m_pBack = pNewItem;
        1716  ++m_Count;
        1717  }
        1718  return pNewItem;
        1719 }
        1720 
        1721 template<typename T>
        1722 VmaListItem<T>* VmaRawList<T>::PushFront()
        1723 {
        1724  ItemType* const pNewItem = m_ItemAllocator.Alloc();
        1725  pNewItem->pPrev = VMA_NULL;
        1726  if(IsEmpty())
        1727  {
        1728  pNewItem->pNext = VMA_NULL;
        1729  m_pFront = pNewItem;
        1730  m_pBack = pNewItem;
        1731  m_Count = 1;
        1732  }
        1733  else
        1734  {
        1735  pNewItem->pNext = m_pFront;
        1736  m_pFront->pPrev = pNewItem;
        1737  m_pFront = pNewItem;
        1738  ++m_Count;
        1739  }
        1740  return pNewItem;
        1741 }
        1742 
        1743 template<typename T>
        1744 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
        1745 {
        1746  ItemType* const pNewItem = PushBack();
        1747  pNewItem->Value = value;
        1748  return pNewItem;
        1749 }
        1750 
        1751 template<typename T>
        1752 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
        1753 {
        1754  ItemType* const pNewItem = PushFront();
        1755  pNewItem->Value = value;
        1756  return pNewItem;
        1757 }
        1758 
        1759 template<typename T>
        1760 void VmaRawList<T>::PopBack()
        1761 {
        1762  VMA_HEAVY_ASSERT(m_Count > 0);
        1763  ItemType* const pBackItem = m_pBack;
        1764  ItemType* const pPrevItem = pBackItem->pPrev;
        1765  if(pPrevItem != VMA_NULL)
        1766  {
        1767  pPrevItem->pNext = VMA_NULL;
        1768  }
        1769  m_pBack = pPrevItem;
        1770  m_ItemAllocator.Free(pBackItem);
        1771  --m_Count;
        1772 }
        1773 
        1774 template<typename T>
        1775 void VmaRawList<T>::PopFront()
        1776 {
        1777  VMA_HEAVY_ASSERT(m_Count > 0);
        1778  ItemType* const pFrontItem = m_pFront;
        1779  ItemType* const pNextItem = pFrontItem->pNext;
        1780  if(pNextItem != VMA_NULL)
        1781  {
        1782  pNextItem->pPrev = VMA_NULL;
        1783  }
        1784  m_pFront = pNextItem;
        1785  m_ItemAllocator.Free(pFrontItem);
        1786  --m_Count;
        1787 }
        1788 
        1789 template<typename T>
        1790 void VmaRawList<T>::Remove(ItemType* pItem)
        1791 {
        1792  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
        1793  VMA_HEAVY_ASSERT(m_Count > 0);
        1794 
        1795  if(pItem->pPrev != VMA_NULL)
        1796  {
        1797  pItem->pPrev->pNext = pItem->pNext;
        1798  }
        1799  else
        1800  {
        1801  VMA_HEAVY_ASSERT(m_pFront == pItem);
        1802  m_pFront = pItem->pNext;
        1803  }
        1804 
        1805  if(pItem->pNext != VMA_NULL)
        1806  {
        1807  pItem->pNext->pPrev = pItem->pPrev;
        1808  }
        1809  else
        1810  {
        1811  VMA_HEAVY_ASSERT(m_pBack == pItem);
        1812  m_pBack = pItem->pPrev;
        1813  }
        1814 
        1815  m_ItemAllocator.Free(pItem);
        1816  --m_Count;
        1817 }
        1818 
        1819 template<typename T>
        1820 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
        1821 {
        1822  if(pItem != VMA_NULL)
        1823  {
        1824  ItemType* const prevItem = pItem->pPrev;
        1825  ItemType* const newItem = m_ItemAllocator.Alloc();
        1826  newItem->pPrev = prevItem;
        1827  newItem->pNext = pItem;
        1828  pItem->pPrev = newItem;
        1829  if(prevItem != VMA_NULL)
        1830  {
        1831  prevItem->pNext = newItem;
        1832  }
        1833  else
        1834  {
        1835  VMA_HEAVY_ASSERT(m_pFront == pItem);
        1836  m_pFront = newItem;
        1837  }
        1838  ++m_Count;
        1839  return newItem;
        1840  }
        1841  else
        1842  return PushBack();
        1843 }
        1844 
        1845 template<typename T>
        1846 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
        1847 {
        1848  if(pItem != VMA_NULL)
        1849  {
        1850  ItemType* const nextItem = pItem->pNext;
        1851  ItemType* const newItem = m_ItemAllocator.Alloc();
        1852  newItem->pNext = nextItem;
        1853  newItem->pPrev = pItem;
        1854  pItem->pNext = newItem;
        1855  if(nextItem != VMA_NULL)
        1856  {
        1857  nextItem->pPrev = newItem;
        1858  }
        1859  else
        1860  {
        1861  VMA_HEAVY_ASSERT(m_pBack == pItem);
        1862  m_pBack = newItem;
        1863  }
        1864  ++m_Count;
        1865  return newItem;
        1866  }
        1867  else
        1868  return PushFront();
        1869 }
        1870 
        1871 template<typename T>
        1872 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
        1873 {
        1874  ItemType* const newItem = InsertBefore(pItem);
        1875  newItem->Value = value;
        1876  return newItem;
        1877 }
        1878 
        1879 template<typename T>
        1880 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
        1881 {
        1882  ItemType* const newItem = InsertAfter(pItem);
        1883  newItem->Value = value;
        1884  return newItem;
        1885 }
        1886 
        1887 template<typename T, typename AllocatorT>
        1888 class VmaList
        1889 {
        1890 public:
        1891  class iterator
        1892  {
        1893  public:
        1894  iterator() :
        1895  m_pList(VMA_NULL),
        1896  m_pItem(VMA_NULL)
        1897  {
        1898  }
        1899 
        1900  T& operator*() const
        1901  {
        1902  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1903  return m_pItem->Value;
        1904  }
        1905  T* operator->() const
        1906  {
        1907  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1908  return &m_pItem->Value;
        1909  }
        1910 
        1911  iterator& operator++()
        1912  {
        1913  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1914  m_pItem = m_pItem->pNext;
        1915  return *this;
        1916  }
        1917  iterator& operator--()
        1918  {
        1919  if(m_pItem != VMA_NULL)
        1920  {
        1921  m_pItem = m_pItem->pPrev;
        1922  }
        1923  else
        1924  {
        1925  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
        1926  m_pItem = m_pList->Back();
        1927  }
        1928  return *this;
        1929  }
        1930 
        1931  iterator operator++(int)
        1932  {
        1933  iterator result = *this;
        1934  ++*this;
        1935  return result;
        1936  }
        1937  iterator operator--(int)
        1938  {
        1939  iterator result = *this;
        1940  --*this;
        1941  return result;
        1942  }
        1943 
        1944  bool operator==(const iterator& rhs) const
        1945  {
        1946  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        1947  return m_pItem == rhs.m_pItem;
        1948  }
        1949  bool operator!=(const iterator& rhs) const
        1950  {
        1951  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        1952  return m_pItem != rhs.m_pItem;
        1953  }
        1954 
        1955  private:
        1956  VmaRawList<T>* m_pList;
        1957  VmaListItem<T>* m_pItem;
        1958 
        1959  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
        1960  m_pList(pList),
        1961  m_pItem(pItem)
        1962  {
        1963  }
        1964 
        1965  friend class VmaList<T, AllocatorT>;
        1966  friend class VmaList<T, AllocatorT>:: const_iterator;
        1967  };
        1968 
        1969  class const_iterator
        1970  {
        1971  public:
        1972  const_iterator() :
        1973  m_pList(VMA_NULL),
        1974  m_pItem(VMA_NULL)
        1975  {
        1976  }
        1977 
        1978  const_iterator(const iterator& src) :
        1979  m_pList(src.m_pList),
        1980  m_pItem(src.m_pItem)
        1981  {
        1982  }
        1983 
        1984  const T& operator*() const
        1985  {
        1986  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1987  return m_pItem->Value;
        1988  }
        1989  const T* operator->() const
        1990  {
        1991  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1992  return &m_pItem->Value;
        1993  }
        1994 
        1995  const_iterator& operator++()
        1996  {
        1997  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        1998  m_pItem = m_pItem->pNext;
        1999  return *this;
        2000  }
        2001  const_iterator& operator--()
        2002  {
        2003  if(m_pItem != VMA_NULL)
        2004  {
        2005  m_pItem = m_pItem->pPrev;
        2006  }
        2007  else
        2008  {
        2009  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
        2010  m_pItem = m_pList->Back();
        2011  }
        2012  return *this;
        2013  }
        2014 
        2015  const_iterator operator++(int)
        2016  {
        2017  const_iterator result = *this;
        2018  ++*this;
        2019  return result;
        2020  }
        2021  const_iterator operator--(int)
        2022  {
        2023  const_iterator result = *this;
        2024  --*this;
        2025  return result;
        2026  }
        2027 
        2028  bool operator==(const const_iterator& rhs) const
        2029  {
        2030  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2031  return m_pItem == rhs.m_pItem;
        2032  }
        2033  bool operator!=(const const_iterator& rhs) const
        2034  {
        2035  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2036  return m_pItem != rhs.m_pItem;
        2037  }
        2038 
        2039  private:
        2040  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
        2041  m_pList(pList),
        2042  m_pItem(pItem)
        2043  {
        2044  }
        2045 
        2046  const VmaRawList<T>* m_pList;
        2047  const VmaListItem<T>* m_pItem;
        2048 
        2049  friend class VmaList<T, AllocatorT>;
        2050  };
        2051 
        2052  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
        2053 
        2054  bool empty() const { return m_RawList.IsEmpty(); }
        2055  size_t size() const { return m_RawList.GetCount(); }
        2056 
        2057  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
        2058  iterator end() { return iterator(&m_RawList, VMA_NULL); }
        2059 
        2060  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
        2061  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
        2062 
        2063  void clear() { m_RawList.Clear(); }
        2064  void push_back(const T& value) { m_RawList.PushBack(value); }
        2065  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
        2066  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
        2067 
        2068 private:
        2069  VmaRawList<T> m_RawList;
        2070 };
        2071 
        2072 #endif // #if VMA_USE_STL_LIST
        2073 
        2075 // class VmaMap
        2076 
        2077 #if VMA_USE_STL_UNORDERED_MAP
        2078 
        2079 #define VmaPair std::pair
        2080 
        2081 #define VMA_MAP_TYPE(KeyT, ValueT) \
        2082  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
        2083 
        2084 #else // #if VMA_USE_STL_UNORDERED_MAP
        2085 
        2086 template<typename T1, typename T2>
        2087 struct VmaPair
        2088 {
        2089  T1 first;
        2090  T2 second;
        2091 
        2092  VmaPair() : first(), second() { }
        2093  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
        2094 };
        2095 
        2096 /* Class compatible with subset of interface of std::unordered_map.
        2097 KeyT, ValueT must be POD because they will be stored in VmaVector.
        2098 */
        2099 template<typename KeyT, typename ValueT>
        2100 class VmaMap
        2101 {
        2102 public:
        2103  typedef VmaPair<KeyT, ValueT> PairType;
        2104  typedef PairType* iterator;
        2105 
        2106  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
        2107 
        2108  iterator begin() { return m_Vector.begin(); }
        2109  iterator end() { return m_Vector.end(); }
        2110 
        2111  void insert(const PairType& pair);
        2112  iterator find(const KeyT& key);
        2113  void erase(iterator it);
        2114 
        2115 private:
        2116  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
        2117 };
        2118 
        2119 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
        2120 
        2121 template<typename FirstT, typename SecondT>
        2122 struct VmaPairFirstLess
        2123 {
        2124  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
        2125  {
        2126  return lhs.first < rhs.first;
        2127  }
        2128  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
        2129  {
        2130  return lhs.first < rhsFirst;
        2131  }
        2132 };
        2133 
        2134 template<typename KeyT, typename ValueT>
        2135 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
        2136 {
        2137  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
        2138  m_Vector.data(),
        2139  m_Vector.data() + m_Vector.size(),
        2140  pair,
        2141  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
        2142  VectorInsert(m_Vector, indexToInsert, pair);
        2143 }
        2144 
        2145 template<typename KeyT, typename ValueT>
        2146 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
        2147 {
        2148  PairType* it = VmaBinaryFindFirstNotLess(
        2149  m_Vector.data(),
        2150  m_Vector.data() + m_Vector.size(),
        2151  key,
        2152  VmaPairFirstLess<KeyT, ValueT>());
        2153  if((it != m_Vector.end()) && (it->first == key))
        2154  {
        2155  return it;
        2156  }
        2157  else
        2158  {
        2159  return m_Vector.end();
        2160  }
        2161 }
        2162 
        2163 template<typename KeyT, typename ValueT>
        2164 void VmaMap<KeyT, ValueT>::erase(iterator it)
        2165 {
        2166  VectorRemove(m_Vector, it - m_Vector.begin());
        2167 }
        2168 
        2169 #endif // #if VMA_USE_STL_UNORDERED_MAP
        2170 
        2172 
        2173 class VmaBlock;
        2174 
        2175 enum VMA_BLOCK_VECTOR_TYPE
        2176 {
        2177  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
        2178  VMA_BLOCK_VECTOR_TYPE_MAPPED,
        2179  VMA_BLOCK_VECTOR_TYPE_COUNT
        2180 };
        2181 
        2182 static VMA_BLOCK_VECTOR_TYPE VmaMemoryRequirementFlagsToBlockVectorType(VmaMemoryRequirementFlags flags)
        2183 {
        2184  return (flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0 ?
        2185  VMA_BLOCK_VECTOR_TYPE_MAPPED :
        2186  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
        2187 }
        2188 
        2189 struct VmaAllocation_T
        2190 {
        2191 public:
        2192  enum ALLOCATION_TYPE
        2193  {
        2194  ALLOCATION_TYPE_NONE,
        2195  ALLOCATION_TYPE_BLOCK,
        2196  ALLOCATION_TYPE_OWN,
        2197  };
        2198 
        2199  VmaAllocation_T()
        2200  {
        2201  memset(this, 0, sizeof(VmaAllocation_T));
        2202  }
        2203 
        2204  void InitBlockAllocation(
        2205  VmaBlock* block,
        2206  VkDeviceSize offset,
        2207  VkDeviceSize alignment,
        2208  VkDeviceSize size,
        2209  VmaSuballocationType suballocationType,
        2210  void* pUserData)
        2211  {
        2212  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
        2213  VMA_ASSERT(block != VMA_NULL);
        2214  m_Type = ALLOCATION_TYPE_BLOCK;
        2215  m_Alignment = alignment;
        2216  m_Size = size;
        2217  m_pUserData = pUserData;
        2218  m_SuballocationType = suballocationType;
        2219  m_BlockAllocation.m_Block = block;
        2220  m_BlockAllocation.m_Offset = offset;
        2221  }
        2222 
        2223  void ChangeBlockAllocation(
        2224  VmaBlock* block,
        2225  VkDeviceSize offset)
        2226  {
        2227  VMA_ASSERT(block != VMA_NULL);
        2228  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
        2229  m_BlockAllocation.m_Block = block;
        2230  m_BlockAllocation.m_Offset = offset;
        2231  }
        2232 
        2233  void InitOwnAllocation(
        2234  uint32_t memoryTypeIndex,
        2235  VkDeviceMemory hMemory,
        2236  VmaSuballocationType suballocationType,
        2237  bool persistentMap,
        2238  void* pMappedData,
        2239  VkDeviceSize size,
        2240  void* pUserData)
        2241  {
        2242  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
        2243  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
        2244  m_Type = ALLOCATION_TYPE_OWN;
        2245  m_Alignment = 0;
        2246  m_Size = size;
        2247  m_pUserData = pUserData;
        2248  m_SuballocationType = suballocationType;
        2249  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
        2250  m_OwnAllocation.m_hMemory = hMemory;
        2251  m_OwnAllocation.m_PersistentMap = persistentMap;
        2252  m_OwnAllocation.m_pMappedData = pMappedData;
        2253  }
        2254 
        2255  ALLOCATION_TYPE GetType() const { return m_Type; }
        2256  VkDeviceSize GetAlignment() const { return m_Alignment; }
        2257  VkDeviceSize GetSize() const { return m_Size; }
        2258  void* GetUserData() const { return m_pUserData; }
        2259  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
        2260  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
        2261 
        2262  VmaBlock* GetBlock() const
        2263  {
        2264  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
        2265  return m_BlockAllocation.m_Block;
        2266  }
        2267  VkDeviceSize GetOffset() const
        2268  {
        2269  return (m_Type == ALLOCATION_TYPE_BLOCK) ? m_BlockAllocation.m_Offset : 0;
        2270  }
        2271  VkDeviceMemory GetMemory() const;
        2272  uint32_t GetMemoryTypeIndex() const;
        2273  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
        2274  void* GetMappedData() const;
        2275 
        2276  VkResult OwnAllocMapPersistentlyMappedMemory(VkDevice hDevice)
        2277  {
        2278  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
        2279  if(m_OwnAllocation.m_PersistentMap)
        2280  {
        2281  return vkMapMemory(hDevice, m_OwnAllocation.m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_OwnAllocation.m_pMappedData);
        2282  }
        2283  return VK_SUCCESS;
        2284  }
        2285  void OwnAllocUnmapPersistentlyMappedMemory(VkDevice hDevice)
        2286  {
        2287  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
        2288  if(m_OwnAllocation.m_pMappedData)
        2289  {
        2290  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
        2291  vkUnmapMemory(hDevice, m_OwnAllocation.m_hMemory);
        2292  m_OwnAllocation.m_pMappedData = VMA_NULL;
        2293  }
        2294  }
        2295 
        2296 private:
        2297  VkDeviceSize m_Alignment;
        2298  VkDeviceSize m_Size;
        2299  void* m_pUserData;
        2300  ALLOCATION_TYPE m_Type;
        2301  VmaSuballocationType m_SuballocationType;
        2302 
        2303  // Allocation out of VmaBlock.
        2304  struct BlockAllocation
        2305  {
        2306  VmaBlock* m_Block;
        2307  VkDeviceSize m_Offset;
        2308  };
        2309 
        2310  // Allocation for an object that has its own private VkDeviceMemory.
        2311  struct OwnAllocation
        2312  {
        2313  uint32_t m_MemoryTypeIndex;
        2314  VkDeviceMemory m_hMemory;
        2315  bool m_PersistentMap;
        2316  void* m_pMappedData;
        2317  };
        2318 
        2319  union
        2320  {
        2321  // Allocation out of VmaBlock.
        2322  BlockAllocation m_BlockAllocation;
        2323  // Allocation for an object that has its own private VkDeviceMemory.
        2324  OwnAllocation m_OwnAllocation;
        2325  };
        2326 };
        2327 
        2328 /*
        2329 Represents a region of VmaBlock that is either assigned and returned as
        2330 allocated memory block or free.
        2331 */
        2332 struct VmaSuballocation
        2333 {
        2334  VkDeviceSize offset;
        2335  VkDeviceSize size;
        2336  VmaSuballocationType type;
        2337 };
        2338 
        2339 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
        2340 
        2341 // Parameters of an allocation.
        2342 struct VmaAllocationRequest
        2343 {
        2344  VmaSuballocationList::iterator freeSuballocationItem;
        2345  VkDeviceSize offset;
        2346 };
        2347 
        2348 /* Single block of memory - VkDeviceMemory with all the data about its regions
        2349 assigned or free. */
        2350 class VmaBlock
        2351 {
        2352 public:
        2353  uint32_t m_MemoryTypeIndex;
        2354  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
        2355  VkDeviceMemory m_hMemory;
        2356  VkDeviceSize m_Size;
        2357  bool m_PersistentMap;
        2358  void* m_pMappedData;
        2359  uint32_t m_FreeCount;
        2360  VkDeviceSize m_SumFreeSize;
        2361  VmaSuballocationList m_Suballocations;
        2362  // Suballocations that are free and have size greater than certain threshold.
        2363  // Sorted by size, ascending.
        2364  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
        2365 
        2366  VmaBlock(VmaAllocator hAllocator);
        2367 
        2368  ~VmaBlock()
        2369  {
        2370  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
        2371  }
        2372 
        2373  // Always call after construction.
        2374  void Init(
        2375  uint32_t newMemoryTypeIndex,
        2376  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
        2377  VkDeviceMemory newMemory,
        2378  VkDeviceSize newSize,
        2379  bool persistentMap,
        2380  void* pMappedData);
        2381  // Always call before destruction.
        2382  void Destroy(VmaAllocator allocator);
        2383 
        2384  // Validates all data structures inside this object. If not valid, returns false.
        2385  bool Validate() const;
        2386 
        2387  // Tries to find a place for suballocation with given parameters inside this allocation.
        2388  // If succeeded, fills pAllocationRequest and returns true.
        2389  // If failed, returns false.
        2390  bool CreateAllocationRequest(
        2391  VkDeviceSize bufferImageGranularity,
        2392  VkDeviceSize allocSize,
        2393  VkDeviceSize allocAlignment,
        2394  VmaSuballocationType allocType,
        2395  VmaAllocationRequest* pAllocationRequest);
        2396 
        2397  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
        2398  // If yes, fills pOffset and returns true. If no, returns false.
        2399  bool CheckAllocation(
        2400  VkDeviceSize bufferImageGranularity,
        2401  VkDeviceSize allocSize,
        2402  VkDeviceSize allocAlignment,
        2403  VmaSuballocationType allocType,
        2404  VmaSuballocationList::const_iterator freeSuballocItem,
        2405  VkDeviceSize* pOffset) const;
        2406 
        2407  // Returns true if this allocation is empty - contains only single free suballocation.
        2408  bool IsEmpty() const;
        2409 
        2410  // Makes actual allocation based on request. Request must already be checked
        2411  // and valid.
        2412  void Alloc(
        2413  const VmaAllocationRequest& request,
        2414  VmaSuballocationType type,
        2415  VkDeviceSize allocSize);
        2416 
        2417  // Frees suballocation assigned to given memory region.
        2418  void Free(const VmaAllocation allocation);
        2419 
        2420 #if VMA_STATS_STRING_ENABLED
        2421  void PrintDetailedMap(class VmaStringBuilder& sb) const;
        2422 #endif
        2423 
        2424 private:
        2425  // Given free suballocation, it merges it with following one, which must also be free.
        2426  void MergeFreeWithNext(VmaSuballocationList::iterator item);
        2427  // Releases given suballocation, making it free. Merges it with adjacent free
        2428  // suballocations if applicable.
        2429  void FreeSuballocation(VmaSuballocationList::iterator suballocItem);
        2430  // Given free suballocation, it inserts it into sorted list of
        2431  // m_FreeSuballocationsBySize if it's suitable.
        2432  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
        2433  // Given free suballocation, it removes it from sorted list of
        2434  // m_FreeSuballocationsBySize if it's suitable.
        2435  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
        2436 };
        2437 
        2438 struct VmaPointerLess
        2439 {
        2440  bool operator()(const void* lhs, const void* rhs) const
        2441  {
        2442  return lhs < rhs;
        2443  }
        2444 };
        2445 
        2446 /* Sequence of VmaBlock. Represents memory blocks allocated for a specific
        2447 Vulkan memory type. */
        2448 struct VmaBlockVector
        2449 {
        2450  // Incrementally sorted by sumFreeSize, ascending.
        2451  VmaVector< VmaBlock*, VmaStlAllocator<VmaBlock*> > m_Blocks;
        2452 
        2453  VmaBlockVector(VmaAllocator hAllocator);
        2454  ~VmaBlockVector();
        2455 
        2456  bool IsEmpty() const { return m_Blocks.empty(); }
        2457 
        2458  // Finds and removes given block from vector.
        2459  void Remove(VmaBlock* pBlock);
        2460 
        2461  // Performs single step in sorting m_Blocks. They may not be fully sorted
        2462  // after this call.
        2463  void IncrementallySortBlocks();
        2464 
        2465  // Adds statistics of this BlockVector to pStats.
        2466  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const;
        2467 
        2468 #if VMA_STATS_STRING_ENABLED
        2469  void PrintDetailedMap(class VmaStringBuilder& sb) const;
        2470 #endif
        2471 
        2472  void UnmapPersistentlyMappedMemory();
        2473  VkResult MapPersistentlyMappedMemory();
        2474 
        2475 private:
        2476  VmaAllocator m_hAllocator;
        2477 };
        2478 
        2479 // Main allocator object.
        2480 struct VmaAllocator_T
        2481 {
        2482  bool m_UseMutex;
        2483  VkDevice m_hDevice;
        2484  bool m_AllocationCallbacksSpecified;
        2485  VkAllocationCallbacks m_AllocationCallbacks;
        2486  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
        2487  VkDeviceSize m_PreferredLargeHeapBlockSize;
        2488  VkDeviceSize m_PreferredSmallHeapBlockSize;
        2489  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
        2490  // Counter to allow nested calls to these functions.
        2491  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
        2492 
        2493  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
        2494  VkPhysicalDeviceMemoryProperties m_MemProps;
        2495 
        2496  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
        2497  /* There can be at most one allocation that is completely empty - a
        2498  hysteresis to avoid pessimistic case of alternating creation and destruction
        2499  of a VkDeviceMemory. */
        2500  bool m_HasEmptyBlock[VK_MAX_MEMORY_TYPES];
        2501  VMA_MUTEX m_BlocksMutex[VK_MAX_MEMORY_TYPES];
        2502 
        2503  // Each vector is sorted by memory (handle value).
        2504  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
        2505  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
        2506  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
        2507 
        2508  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
        2509  ~VmaAllocator_T();
        2510 
        2511  const VkAllocationCallbacks* GetAllocationCallbacks() const
        2512  {
        2513  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
        2514  }
        2515 
        2516  VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const;
        2517 
        2518  VkDeviceSize GetBufferImageGranularity() const
        2519  {
        2520  return VMA_MAX(
        2521  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
        2522  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
        2523  }
        2524 
        2525  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
        2526  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
        2527 
        2528  // Main allocation function.
        2529  VkResult AllocateMemory(
        2530  const VkMemoryRequirements& vkMemReq,
        2531  const VmaMemoryRequirements& vmaMemReq,
        2532  VmaSuballocationType suballocType,
        2533  VmaAllocation* pAllocation);
        2534 
        2535  // Main deallocation function.
        2536  void FreeMemory(const VmaAllocation allocation);
        2537 
        2538  void CalculateStats(VmaStats* pStats);
        2539 
        2540 #if VMA_STATS_STRING_ENABLED
        2541  void PrintDetailedMap(class VmaStringBuilder& sb);
        2542 #endif
        2543 
        2544  void UnmapPersistentlyMappedMemory();
        2545  VkResult MapPersistentlyMappedMemory();
        2546 
        2547  VkResult Defragment(
        2548  VmaAllocation* pAllocations,
        2549  size_t allocationCount,
        2550  VkBool32* pAllocationsChanged,
        2551  const VmaDefragmentationInfo* pDefragmentationInfo,
        2552  VmaDefragmentationStats* pDefragmentationStats);
        2553 
        2554  static void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
        2555 
        2556 private:
        2557  VkPhysicalDevice m_PhysicalDevice;
        2558 
        2559  VkResult AllocateMemoryOfType(
        2560  const VkMemoryRequirements& vkMemReq,
        2561  const VmaMemoryRequirements& vmaMemReq,
        2562  uint32_t memTypeIndex,
        2563  VmaSuballocationType suballocType,
        2564  VmaAllocation* pAllocation);
        2565 
        2566  // Allocates and registers new VkDeviceMemory specifically for single allocation.
        2567  VkResult AllocateOwnMemory(
        2568  VkDeviceSize size,
        2569  VmaSuballocationType suballocType,
        2570  uint32_t memTypeIndex,
        2571  bool map,
        2572  void* pUserData,
        2573  VmaAllocation* pAllocation);
        2574 
        2575  // Tries to free pMemory as Own Memory. Returns true if found and freed.
        2576  void FreeOwnMemory(VmaAllocation allocation);
        2577 };
        2578 
        2580 // Memory allocation #2 after VmaAllocator_T definition
        2581 
        2582 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
        2583 {
        2584  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
        2585 }
        2586 
        2587 static void VmaFree(VmaAllocator hAllocator, void* ptr)
        2588 {
        2589  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
        2590 }
        2591 
        2592 template<typename T>
        2593 static T* VmaAllocate(VmaAllocator hAllocator)
        2594 {
        2595  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
        2596 }
        2597 
        2598 template<typename T>
        2599 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
        2600 {
        2601  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
        2602 }
        2603 
        2604 template<typename T>
        2605 static void vma_delete(VmaAllocator hAllocator, T* ptr)
        2606 {
        2607  if(ptr != VMA_NULL)
        2608  {
        2609  ptr->~T();
        2610  VmaFree(hAllocator, ptr);
        2611  }
        2612 }
        2613 
        2614 template<typename T>
        2615 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
        2616 {
        2617  if(ptr != VMA_NULL)
        2618  {
        2619  for(size_t i = count; i--; )
        2620  ptr[i].~T();
        2621  VmaFree(hAllocator, ptr);
        2622  }
        2623 }
        2624 
        2626 // VmaStringBuilder
        2627 
        2628 #if VMA_STATS_STRING_ENABLED
        2629 
        2630 class VmaStringBuilder
        2631 {
        2632 public:
        2633  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
        2634  size_t GetLength() const { return m_Data.size(); }
        2635  const char* GetData() const { return m_Data.data(); }
        2636 
        2637  void Add(char ch) { m_Data.push_back(ch); }
        2638  void Add(const char* pStr);
        2639  void AddNewLine() { Add('\n'); }
        2640  void AddNumber(uint32_t num);
        2641  void AddNumber(uint64_t num);
        2642  void AddBool(bool b) { Add(b ? "true" : "false"); }
        2643  void AddNull() { Add("null"); }
        2644  void AddString(const char* pStr);
        2645 
        2646 private:
        2647  VmaVector< char, VmaStlAllocator<char> > m_Data;
        2648 };
        2649 
        2650 void VmaStringBuilder::Add(const char* pStr)
        2651 {
        2652  const size_t strLen = strlen(pStr);
        2653  if(strLen > 0)
        2654  {
        2655  const size_t oldCount = m_Data.size();
        2656  m_Data.resize(oldCount + strLen);
        2657  memcpy(m_Data.data() + oldCount, pStr, strLen);
        2658  }
        2659 }
        2660 
        2661 void VmaStringBuilder::AddNumber(uint32_t num)
        2662 {
        2663  char buf[11];
        2664  VmaUint32ToStr(buf, sizeof(buf), num);
        2665  Add(buf);
        2666 }
        2667 
        2668 void VmaStringBuilder::AddNumber(uint64_t num)
        2669 {
        2670  char buf[21];
        2671  VmaUint64ToStr(buf, sizeof(buf), num);
        2672  Add(buf);
        2673 }
        2674 
        2675 void VmaStringBuilder::AddString(const char* pStr)
        2676 {
        2677  Add('"');
        2678  const size_t strLen = strlen(pStr);
        2679  for(size_t i = 0; i < strLen; ++i)
        2680  {
        2681  char ch = pStr[i];
        2682  if(ch == '\'')
        2683  {
        2684  Add("\\\\");
        2685  }
        2686  else if(ch == '"')
        2687  {
        2688  Add("\\\"");
        2689  }
        2690  else if(ch >= 32)
        2691  {
        2692  Add(ch);
        2693  }
        2694  else switch(ch)
        2695  {
        2696  case '\n':
        2697  Add("\\n");
        2698  break;
        2699  case '\r':
        2700  Add("\\r");
        2701  break;
        2702  case '\t':
        2703  Add("\\t");
        2704  break;
        2705  default:
        2706  VMA_ASSERT(0 && "Character not currently supported.");
        2707  break;
        2708  }
        2709  }
        2710  Add('"');
        2711 }
        2712 
        2714 
        2715 VkDeviceMemory VmaAllocation_T::GetMemory() const
        2716 {
        2717  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
        2718  m_BlockAllocation.m_Block->m_hMemory : m_OwnAllocation.m_hMemory;
        2719 }
        2720 
        2721 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
        2722 {
        2723  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
        2724  m_BlockAllocation.m_Block->m_MemoryTypeIndex : m_OwnAllocation.m_MemoryTypeIndex;
        2725 }
        2726 
        2727 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
        2728 {
        2729  return (m_Type == ALLOCATION_TYPE_BLOCK) ?
        2730  m_BlockAllocation.m_Block->m_BlockVectorType :
        2731  (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
        2732 }
        2733 
        2734 void* VmaAllocation_T::GetMappedData() const
        2735 {
        2736  switch(m_Type)
        2737  {
        2738  case ALLOCATION_TYPE_BLOCK:
        2739  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
        2740  {
        2741  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
        2742  }
        2743  else
        2744  {
        2745  return VMA_NULL;
        2746  }
        2747  break;
        2748  case ALLOCATION_TYPE_OWN:
        2749  return m_OwnAllocation.m_pMappedData;
        2750  default:
        2751  VMA_ASSERT(0);
        2752  return VMA_NULL;
        2753  }
        2754 }
        2755 
        2756 // Correspond to values of enum VmaSuballocationType.
        2757 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
        2758  "FREE",
        2759  "UNKNOWN",
        2760  "BUFFER",
        2761  "IMAGE_UNKNOWN",
        2762  "IMAGE_LINEAR",
        2763  "IMAGE_OPTIMAL",
        2764 };
        2765 
        2766 static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat)
        2767 {
        2768  sb.Add("{ \"Allocations\": ");
        2769  sb.AddNumber(stat.AllocationCount);
        2770  sb.Add(", \"Suballocations\": ");
        2771  sb.AddNumber(stat.SuballocationCount);
        2772  sb.Add(", \"UnusedRanges\": ");
        2773  sb.AddNumber(stat.UnusedRangeCount);
        2774  sb.Add(", \"UsedBytes\": ");
        2775  sb.AddNumber(stat.UsedBytes);
        2776  sb.Add(", \"UnusedBytes\": ");
        2777  sb.AddNumber(stat.UnusedBytes);
        2778  sb.Add(", \"SuballocationSize\": { \"Min\": ");
        2779  sb.AddNumber(stat.SuballocationSizeMin);
        2780  sb.Add(", \"Avg\": ");
        2781  sb.AddNumber(stat.SuballocationSizeAvg);
        2782  sb.Add(", \"Max\": ");
        2783  sb.AddNumber(stat.SuballocationSizeMax);
        2784  sb.Add(" }, \"UnusedRangeSize\": { \"Min\": ");
        2785  sb.AddNumber(stat.UnusedRangeSizeMin);
        2786  sb.Add(", \"Avg\": ");
        2787  sb.AddNumber(stat.UnusedRangeSizeAvg);
        2788  sb.Add(", \"Max\": ");
        2789  sb.AddNumber(stat.UnusedRangeSizeMax);
        2790  sb.Add(" } }");
        2791 }
        2792 
        2793 #endif // #if VMA_STATS_STRING_ENABLED
        2794 
        2795 struct VmaSuballocationItemSizeLess
        2796 {
        2797  bool operator()(
        2798  const VmaSuballocationList::iterator lhs,
        2799  const VmaSuballocationList::iterator rhs) const
        2800  {
        2801  return lhs->size < rhs->size;
        2802  }
        2803  bool operator()(
        2804  const VmaSuballocationList::iterator lhs,
        2805  VkDeviceSize rhsSize) const
        2806  {
        2807  return lhs->size < rhsSize;
        2808  }
        2809 };
        2810 
        2811 VmaBlock::VmaBlock(VmaAllocator hAllocator) :
        2812  m_MemoryTypeIndex(UINT32_MAX),
        2813  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
        2814  m_hMemory(VK_NULL_HANDLE),
        2815  m_Size(0),
        2816  m_PersistentMap(false),
        2817  m_pMappedData(VMA_NULL),
        2818  m_FreeCount(0),
        2819  m_SumFreeSize(0),
        2820  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
        2821  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
        2822 {
        2823 }
        2824 
        2825 void VmaBlock::Init(
        2826  uint32_t newMemoryTypeIndex,
        2827  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
        2828  VkDeviceMemory newMemory,
        2829  VkDeviceSize newSize,
        2830  bool persistentMap,
        2831  void* pMappedData)
        2832 {
        2833  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
        2834 
        2835  m_MemoryTypeIndex = newMemoryTypeIndex;
        2836  m_BlockVectorType = newBlockVectorType;
        2837  m_hMemory = newMemory;
        2838  m_Size = newSize;
        2839  m_PersistentMap = persistentMap;
        2840  m_pMappedData = pMappedData;
        2841  m_FreeCount = 1;
        2842  m_SumFreeSize = newSize;
        2843 
        2844  m_Suballocations.clear();
        2845  m_FreeSuballocationsBySize.clear();
        2846 
        2847  VmaSuballocation suballoc = {};
        2848  suballoc.offset = 0;
        2849  suballoc.size = newSize;
        2850  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        2851 
        2852  m_Suballocations.push_back(suballoc);
        2853  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
        2854  --suballocItem;
        2855  m_FreeSuballocationsBySize.push_back(suballocItem);
        2856 }
        2857 
        2858 void VmaBlock::Destroy(VmaAllocator allocator)
        2859 {
        2860  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
        2861  if(m_pMappedData != VMA_NULL)
        2862  {
        2863  vkUnmapMemory(allocator->m_hDevice, m_hMemory);
        2864  m_pMappedData = VMA_NULL;
        2865  }
        2866 
        2867  // Callback.
        2868  if(allocator->m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
        2869  {
        2870  (*allocator->m_DeviceMemoryCallbacks.pfnFree)(allocator, m_MemoryTypeIndex, m_hMemory, m_Size);
        2871  }
        2872 
        2873  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
        2874  m_hMemory = VK_NULL_HANDLE;
        2875 }
        2876 
        2877 bool VmaBlock::Validate() const
        2878 {
        2879  if((m_hMemory == VK_NULL_HANDLE) ||
        2880  (m_Size == 0) ||
        2881  m_Suballocations.empty())
        2882  {
        2883  return false;
        2884  }
        2885 
        2886  // Expected offset of new suballocation as calculates from previous ones.
        2887  VkDeviceSize calculatedOffset = 0;
        2888  // Expected number of free suballocations as calculated from traversing their list.
        2889  uint32_t calculatedFreeCount = 0;
        2890  // Expected sum size of free suballocations as calculated from traversing their list.
        2891  VkDeviceSize calculatedSumFreeSize = 0;
        2892  // Expected number of free suballocations that should be registered in
        2893  // m_FreeSuballocationsBySize calculated from traversing their list.
        2894  size_t freeSuballocationsToRegister = 0;
        2895  // True if previous visisted suballocation was free.
        2896  bool prevFree = false;
        2897 
        2898  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
        2899  suballocItem != m_Suballocations.cend();
        2900  ++suballocItem)
        2901  {
        2902  const VmaSuballocation& subAlloc = *suballocItem;
        2903 
        2904  // Actual offset of this suballocation doesn't match expected one.
        2905  if(subAlloc.offset != calculatedOffset)
        2906  {
        2907  return false;
        2908  }
        2909 
        2910  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
        2911  // Two adjacent free suballocations are invalid. They should be merged.
        2912  if(prevFree && currFree)
        2913  {
        2914  return false;
        2915  }
        2916  prevFree = currFree;
        2917 
        2918  if(currFree)
        2919  {
        2920  calculatedSumFreeSize += subAlloc.size;
        2921  ++calculatedFreeCount;
        2922  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        2923  {
        2924  ++freeSuballocationsToRegister;
        2925  }
        2926  }
        2927 
        2928  calculatedOffset += subAlloc.size;
        2929  }
        2930 
        2931  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
        2932  // match expected one.
        2933  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
        2934  {
        2935  return false;
        2936  }
        2937 
        2938  VkDeviceSize lastSize = 0;
        2939  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
        2940  {
        2941  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
        2942 
        2943  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
        2944  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
        2945  {
        2946  return false;
        2947  }
        2948  // They must be sorted by size ascending.
        2949  if(suballocItem->size < lastSize)
        2950  {
        2951  return false;
        2952  }
        2953 
        2954  lastSize = suballocItem->size;
        2955  }
        2956 
        2957  // Check if totals match calculacted values.
        2958  return
        2959  (calculatedOffset == m_Size) &&
        2960  (calculatedSumFreeSize == m_SumFreeSize) &&
        2961  (calculatedFreeCount == m_FreeCount);
        2962 }
        2963 
        2964 /*
        2965 How many suitable free suballocations to analyze before choosing best one.
        2966 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
        2967  be chosen.
        2968 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
        2969  suballocations will be analized and best one will be chosen.
        2970 - Any other value is also acceptable.
        2971 */
        2972 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
        2973 
        2974 bool VmaBlock::CreateAllocationRequest(
        2975  VkDeviceSize bufferImageGranularity,
        2976  VkDeviceSize allocSize,
        2977  VkDeviceSize allocAlignment,
        2978  VmaSuballocationType allocType,
        2979  VmaAllocationRequest* pAllocationRequest)
        2980 {
        2981  VMA_ASSERT(allocSize > 0);
        2982  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
        2983  VMA_ASSERT(pAllocationRequest != VMA_NULL);
        2984  VMA_HEAVY_ASSERT(Validate());
        2985 
        2986  // There is not enough total free space in this allocation to fullfill the request: Early return.
        2987  if(m_SumFreeSize < allocSize)
        2988  {
        2989  return false;
        2990  }
        2991 
        2992  // Old brute-force algorithm, linearly searching suballocations.
        2993  /*
        2994  uint32_t suitableSuballocationsFound = 0;
        2995  for(VmaSuballocationList::iterator suballocItem = suballocations.Front();
        2996  suballocItem != VMA_NULL &&
        2997  suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK;
        2998  suballocItem = suballocItem->Next)
        2999  {
        3000  if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE)
        3001  {
        3002  VkDeviceSize offset = 0, cost = 0;
        3003  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost))
        3004  {
        3005  ++suitableSuballocationsFound;
        3006  if(cost < costLimit)
        3007  {
        3008  pAllocationRequest->freeSuballocationItem = suballocItem;
        3009  pAllocationRequest->offset = offset;
        3010  pAllocationRequest->cost = cost;
        3011  if(cost == 0)
        3012  return true;
        3013  costLimit = cost;
        3014  betterSuballocationFound = true;
        3015  }
        3016  }
        3017  }
        3018  }
        3019  */
        3020 
        3021  // New algorithm, efficiently searching freeSuballocationsBySize.
        3022  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
        3023  if(freeSuballocCount > 0)
        3024  {
        3025  if(VMA_BEST_FIT)
        3026  {
        3027  // Find first free suballocation with size not less than allocSize.
        3028  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
        3029  m_FreeSuballocationsBySize.data(),
        3030  m_FreeSuballocationsBySize.data() + freeSuballocCount,
        3031  allocSize,
        3032  VmaSuballocationItemSizeLess());
        3033  size_t index = it - m_FreeSuballocationsBySize.data();
        3034  for(; index < freeSuballocCount; ++index)
        3035  {
        3036  VkDeviceSize offset = 0;
        3037  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
        3038  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
        3039  {
        3040  pAllocationRequest->freeSuballocationItem = suballocItem;
        3041  pAllocationRequest->offset = offset;
        3042  return true;
        3043  }
        3044  }
        3045  }
        3046  else
        3047  {
        3048  // Search staring from biggest suballocations.
        3049  for(size_t index = freeSuballocCount; index--; )
        3050  {
        3051  VkDeviceSize offset = 0;
        3052  const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index];
        3053  if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset))
        3054  {
        3055  pAllocationRequest->freeSuballocationItem = suballocItem;
        3056  pAllocationRequest->offset = offset;
        3057  return true;
        3058  }
        3059  }
        3060  }
        3061  }
        3062 
        3063  return false;
        3064 }
        3065 
        3066 bool VmaBlock::CheckAllocation(
        3067  VkDeviceSize bufferImageGranularity,
        3068  VkDeviceSize allocSize,
        3069  VkDeviceSize allocAlignment,
        3070  VmaSuballocationType allocType,
        3071  VmaSuballocationList::const_iterator freeSuballocItem,
        3072  VkDeviceSize* pOffset) const
        3073 {
        3074  VMA_ASSERT(allocSize > 0);
        3075  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
        3076  VMA_ASSERT(freeSuballocItem != m_Suballocations.cend());
        3077  VMA_ASSERT(pOffset != VMA_NULL);
        3078 
        3079  const VmaSuballocation& suballoc = *freeSuballocItem;
        3080  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
        3081 
        3082  // Size of this suballocation is too small for this request: Early return.
        3083  if(suballoc.size < allocSize)
        3084  {
        3085  return false;
        3086  }
        3087 
        3088  // Start from offset equal to beginning of this suballocation.
        3089  *pOffset = suballoc.offset;
        3090 
        3091  // Apply VMA_DEBUG_MARGIN at the beginning.
        3092  if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin())
        3093  {
        3094  *pOffset += VMA_DEBUG_MARGIN;
        3095  }
        3096 
        3097  // Apply alignment.
        3098  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
        3099  *pOffset = VmaAlignUp(*pOffset, alignment);
        3100 
        3101  // Check previous suballocations for BufferImageGranularity conflicts.
        3102  // Make bigger alignment if necessary.
        3103  if(bufferImageGranularity > 1)
        3104  {
        3105  bool bufferImageGranularityConflict = false;
        3106  VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem;
        3107  while(prevSuballocItem != m_Suballocations.cbegin())
        3108  {
        3109  --prevSuballocItem;
        3110  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
        3111  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
        3112  {
        3113  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
        3114  {
        3115  bufferImageGranularityConflict = true;
        3116  break;
        3117  }
        3118  }
        3119  else
        3120  // Already on previous page.
        3121  break;
        3122  }
        3123  if(bufferImageGranularityConflict)
        3124  {
        3125  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
        3126  }
        3127  }
        3128 
        3129  // Calculate padding at the beginning based on current offset.
        3130  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
        3131 
        3132  // Calculate required margin at the end if this is not last suballocation.
        3133  VmaSuballocationList::const_iterator next = freeSuballocItem;
        3134  ++next;
        3135  const VkDeviceSize requiredEndMargin =
        3136  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
        3137 
        3138  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
        3139  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
        3140  {
        3141  return false;
        3142  }
        3143 
        3144  // Check next suballocations for BufferImageGranularity conflicts.
        3145  // If conflict exists, allocation cannot be made here.
        3146  if(bufferImageGranularity > 1)
        3147  {
        3148  VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem;
        3149  ++nextSuballocItem;
        3150  while(nextSuballocItem != m_Suballocations.cend())
        3151  {
        3152  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
        3153  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
        3154  {
        3155  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
        3156  {
        3157  return false;
        3158  }
        3159  }
        3160  else
        3161  {
        3162  // Already on next page.
        3163  break;
        3164  }
        3165  ++nextSuballocItem;
        3166  }
        3167  }
        3168 
        3169  // All tests passed: Success. pOffset is already filled.
        3170  return true;
        3171 }
        3172 
        3173 bool VmaBlock::IsEmpty() const
        3174 {
        3175  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
        3176 }
        3177 
        3178 void VmaBlock::Alloc(
        3179  const VmaAllocationRequest& request,
        3180  VmaSuballocationType type,
        3181  VkDeviceSize allocSize)
        3182 {
        3183  VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end());
        3184  VmaSuballocation& suballoc = *request.freeSuballocationItem;
        3185  // Given suballocation is a free block.
        3186  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
        3187  // Given offset is inside this suballocation.
        3188  VMA_ASSERT(request.offset >= suballoc.offset);
        3189  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
        3190  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
        3191  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
        3192 
        3193  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
        3194  // it to become used.
        3195  UnregisterFreeSuballocation(request.freeSuballocationItem);
        3196 
        3197  suballoc.offset = request.offset;
        3198  suballoc.size = allocSize;
        3199  suballoc.type = type;
        3200 
        3201  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
        3202  if(paddingEnd)
        3203  {
        3204  VmaSuballocation paddingSuballoc = {};
        3205  paddingSuballoc.offset = request.offset + allocSize;
        3206  paddingSuballoc.size = paddingEnd;
        3207  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        3208  VmaSuballocationList::iterator next = request.freeSuballocationItem;
        3209  ++next;
        3210  const VmaSuballocationList::iterator paddingEndItem =
        3211  m_Suballocations.insert(next, paddingSuballoc);
        3212  RegisterFreeSuballocation(paddingEndItem);
        3213  }
        3214 
        3215  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
        3216  if(paddingBegin)
        3217  {
        3218  VmaSuballocation paddingSuballoc = {};
        3219  paddingSuballoc.offset = request.offset - paddingBegin;
        3220  paddingSuballoc.size = paddingBegin;
        3221  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        3222  const VmaSuballocationList::iterator paddingBeginItem =
        3223  m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc);
        3224  RegisterFreeSuballocation(paddingBeginItem);
        3225  }
        3226 
        3227  // Update totals.
        3228  m_FreeCount = m_FreeCount - 1;
        3229  if(paddingBegin > 0)
        3230  {
        3231  ++m_FreeCount;
        3232  }
        3233  if(paddingEnd > 0)
        3234  {
        3235  ++m_FreeCount;
        3236  }
        3237  m_SumFreeSize -= allocSize;
        3238 }
        3239 
        3240 void VmaBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
        3241 {
        3242  // Change this suballocation to be marked as free.
        3243  VmaSuballocation& suballoc = *suballocItem;
        3244  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        3245 
        3246  // Update totals.
        3247  ++m_FreeCount;
        3248  m_SumFreeSize += suballoc.size;
        3249 
        3250  // Merge with previous and/or next suballocation if it's also free.
        3251  bool mergeWithNext = false;
        3252  bool mergeWithPrev = false;
        3253 
        3254  VmaSuballocationList::iterator nextItem = suballocItem;
        3255  ++nextItem;
        3256  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
        3257  {
        3258  mergeWithNext = true;
        3259  }
        3260 
        3261  VmaSuballocationList::iterator prevItem = suballocItem;
        3262  if(suballocItem != m_Suballocations.begin())
        3263  {
        3264  --prevItem;
        3265  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
        3266  {
        3267  mergeWithPrev = true;
        3268  }
        3269  }
        3270 
        3271  if(mergeWithNext)
        3272  {
        3273  UnregisterFreeSuballocation(nextItem);
        3274  MergeFreeWithNext(suballocItem);
        3275  }
        3276 
        3277  if(mergeWithPrev)
        3278  {
        3279  UnregisterFreeSuballocation(prevItem);
        3280  MergeFreeWithNext(prevItem);
        3281  RegisterFreeSuballocation(prevItem);
        3282  }
        3283  else
        3284  RegisterFreeSuballocation(suballocItem);
        3285 }
        3286 
        3287 void VmaBlock::Free(const VmaAllocation allocation)
        3288 {
        3289  const VkDeviceSize allocationOffset = allocation->GetOffset();
        3290  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
        3291  suballocItem != m_Suballocations.end();
        3292  ++suballocItem)
        3293  {
        3294  VmaSuballocation& suballoc = *suballocItem;
        3295  if(suballoc.offset == allocationOffset)
        3296  {
        3297  FreeSuballocation(suballocItem);
        3298  VMA_HEAVY_ASSERT(Validate());
        3299  return;
        3300  }
        3301  }
        3302  VMA_ASSERT(0 && "Not found!");
        3303 }
        3304 
        3305 #if VMA_STATS_STRING_ENABLED
        3306 
        3307 void VmaBlock::PrintDetailedMap(class VmaStringBuilder& sb) const
        3308 {
        3309  sb.Add("{\n\t\t\t\"Bytes\": ");
        3310  sb.AddNumber(m_Size);
        3311  sb.Add(",\n\t\t\t\"FreeBytes\": ");
        3312  sb.AddNumber(m_SumFreeSize);
        3313  sb.Add(",\n\t\t\t\"Suballocations\": ");
        3314  sb.AddNumber(m_Suballocations.size());
        3315  sb.Add(",\n\t\t\t\"FreeSuballocations\": ");
        3316  sb.AddNumber(m_FreeCount);
        3317  sb.Add(",\n\t\t\t\"SuballocationList\": [");
        3318 
        3319  size_t i = 0;
        3320  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
        3321  suballocItem != m_Suballocations.cend();
        3322  ++suballocItem, ++i)
        3323  {
        3324  if(i > 0)
        3325  {
        3326  sb.Add(",\n\t\t\t\t{ \"Type\": ");
        3327  }
        3328  else
        3329  {
        3330  sb.Add("\n\t\t\t\t{ \"Type\": ");
        3331  }
        3332  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
        3333  sb.Add(", \"Size\": ");
        3334  sb.AddNumber(suballocItem->size);
        3335  sb.Add(", \"Offset\": ");
        3336  sb.AddNumber(suballocItem->offset);
        3337  sb.Add(" }");
        3338  }
        3339 
        3340  sb.Add("\n\t\t\t]\n\t\t}");
        3341 }
        3342 
        3343 #endif // #if VMA_STATS_STRING_ENABLED
        3344 
        3345 void VmaBlock::MergeFreeWithNext(VmaSuballocationList::iterator item)
        3346 {
        3347  VMA_ASSERT(item != m_Suballocations.end());
        3348  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        3349 
        3350  VmaSuballocationList::iterator nextItem = item;
        3351  ++nextItem;
        3352  VMA_ASSERT(nextItem != m_Suballocations.end());
        3353  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
        3354 
        3355  item->size += nextItem->size;
        3356  --m_FreeCount;
        3357  m_Suballocations.erase(nextItem);
        3358 }
        3359 
        3360 void VmaBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
        3361 {
        3362  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        3363  VMA_ASSERT(item->size > 0);
        3364 
        3365  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        3366  {
        3367  if(m_FreeSuballocationsBySize.empty())
        3368  {
        3369  m_FreeSuballocationsBySize.push_back(item);
        3370  }
        3371  else
        3372  {
        3373  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
        3374  m_FreeSuballocationsBySize.data(),
        3375  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
        3376  item,
        3377  VmaSuballocationItemSizeLess());
        3378  size_t index = it - m_FreeSuballocationsBySize.data();
        3379  VectorInsert(m_FreeSuballocationsBySize, index, item);
        3380  }
        3381  }
        3382 }
        3383 
        3384 void VmaBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
        3385 {
        3386  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        3387  VMA_ASSERT(item->size > 0);
        3388 
        3389  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        3390  {
        3391  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
        3392  m_FreeSuballocationsBySize.data(),
        3393  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
        3394  item,
        3395  VmaSuballocationItemSizeLess());
        3396  for(size_t index = it - m_FreeSuballocationsBySize.data();
        3397  index < m_FreeSuballocationsBySize.size();
        3398  ++index)
        3399  {
        3400  if(m_FreeSuballocationsBySize[index] == item)
        3401  {
        3402  VectorRemove(m_FreeSuballocationsBySize, index);
        3403  return;
        3404  }
        3405  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
        3406  }
        3407  VMA_ASSERT(0 && "Not found.");
        3408  }
        3409 }
        3410 
        3411 static void InitStatInfo(VmaStatInfo& outInfo)
        3412 {
        3413  memset(&outInfo, 0, sizeof(outInfo));
        3414  outInfo.SuballocationSizeMin = UINT64_MAX;
        3415  outInfo.UnusedRangeSizeMin = UINT64_MAX;
        3416 }
        3417 
        3418 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc)
        3419 {
        3420  outInfo.AllocationCount = 1;
        3421 
        3422  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
        3423  outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount;
        3424  outInfo.UnusedRangeCount = alloc.m_FreeCount;
        3425 
        3426  outInfo.UnusedBytes = alloc.m_SumFreeSize;
        3427  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
        3428 
        3429  outInfo.SuballocationSizeMin = UINT64_MAX;
        3430  outInfo.SuballocationSizeMax = 0;
        3431  outInfo.UnusedRangeSizeMin = UINT64_MAX;
        3432  outInfo.UnusedRangeSizeMax = 0;
        3433 
        3434  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
        3435  suballocItem != alloc.m_Suballocations.cend();
        3436  ++suballocItem)
        3437  {
        3438  const VmaSuballocation& suballoc = *suballocItem;
        3439  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
        3440  {
        3441  outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size);
        3442  outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size);
        3443  }
        3444  else
        3445  {
        3446  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
        3447  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
        3448  }
        3449  }
        3450 }
        3451 
        3452 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
        3453 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
        3454 {
        3455  inoutInfo.AllocationCount += srcInfo.AllocationCount;
        3456  inoutInfo.SuballocationCount += srcInfo.SuballocationCount;
        3457  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
        3458  inoutInfo.UsedBytes += srcInfo.UsedBytes;
        3459  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
        3460  inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin);
        3461  inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax);
        3462  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
        3463  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
        3464 }
        3465 
        3466 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
        3467 {
        3468  inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ?
        3469  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0;
        3470  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
        3471  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
        3472 }
        3473 
        3474 VmaBlockVector::VmaBlockVector(VmaAllocator hAllocator) :
        3475  m_hAllocator(hAllocator),
        3476  m_Blocks(VmaStlAllocator<VmaBlock*>(hAllocator->GetAllocationCallbacks()))
        3477 {
        3478 }
        3479 
        3480 VmaBlockVector::~VmaBlockVector()
        3481 {
        3482  for(size_t i = m_Blocks.size(); i--; )
        3483  {
        3484  m_Blocks[i]->Destroy(m_hAllocator);
        3485  vma_delete(m_hAllocator, m_Blocks[i]);
        3486  }
        3487 }
        3488 
        3489 void VmaBlockVector::Remove(VmaBlock* pBlock)
        3490 {
        3491  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
        3492  {
        3493  if(m_Blocks[blockIndex] == pBlock)
        3494  {
        3495  VectorRemove(m_Blocks, blockIndex);
        3496  return;
        3497  }
        3498  }
        3499  VMA_ASSERT(0);
        3500 }
        3501 
        3502 void VmaBlockVector::IncrementallySortBlocks()
        3503 {
        3504  // Bubble sort only until first swap.
        3505  for(size_t i = 1; i < m_Blocks.size(); ++i)
        3506  {
        3507  if(m_Blocks[i - 1]->m_SumFreeSize > m_Blocks[i]->m_SumFreeSize)
        3508  {
        3509  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
        3510  return;
        3511  }
        3512  }
        3513 }
        3514 
        3515 #if VMA_STATS_STRING_ENABLED
        3516 
        3517 void VmaBlockVector::PrintDetailedMap(class VmaStringBuilder& sb) const
        3518 {
        3519  for(size_t i = 0; i < m_Blocks.size(); ++i)
        3520  {
        3521  if(i > 0)
        3522  {
        3523  sb.Add(",\n\t\t");
        3524  }
        3525  else
        3526  {
        3527  sb.Add("\n\t\t");
        3528  }
        3529  m_Blocks[i]->PrintDetailedMap(sb);
        3530  }
        3531 }
        3532 
        3533 #endif // #if VMA_STATS_STRING_ENABLED
        3534 
        3535 void VmaBlockVector::UnmapPersistentlyMappedMemory()
        3536 {
        3537  for(size_t i = m_Blocks.size(); i--; )
        3538  {
        3539  VmaBlock* pBlock = m_Blocks[i];
        3540  if(pBlock->m_pMappedData != VMA_NULL)
        3541  {
        3542  VMA_ASSERT(pBlock->m_PersistentMap != false);
        3543  vkUnmapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory);
        3544  pBlock->m_pMappedData = VMA_NULL;
        3545  }
        3546  }
        3547 }
        3548 
        3549 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
        3550 {
        3551  VkResult finalResult = VK_SUCCESS;
        3552  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
        3553  {
        3554  VmaBlock* pBlock = m_Blocks[i];
        3555  if(pBlock->m_PersistentMap)
        3556  {
        3557  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
        3558  VkResult localResult = vkMapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &pBlock->m_pMappedData);
        3559  if(localResult != VK_SUCCESS)
        3560  {
        3561  finalResult = localResult;
        3562  }
        3563  }
        3564  }
        3565  return finalResult;
        3566 }
        3567 
        3568 void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const
        3569 {
        3570  for(uint32_t allocIndex = 0; allocIndex < m_Blocks.size(); ++allocIndex)
        3571  {
        3572  const VmaBlock* const pBlock = m_Blocks[allocIndex];
        3573  VMA_ASSERT(pBlock);
        3574  VMA_HEAVY_ASSERT(pBlock->Validate());
        3575  VmaStatInfo allocationStatInfo;
        3576  CalcAllocationStatInfo(allocationStatInfo, *pBlock);
        3577  VmaAddStatInfo(pStats->total, allocationStatInfo);
        3578  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
        3579  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
        3580  }
        3581 }
        3582 
        3584 // VmaDefragmentator
        3585 
        3586 class VmaDefragmentator
        3587 {
        3588  VkDevice m_hDevice;
        3589  const VkAllocationCallbacks* m_pAllocationCallbacks;
        3590  VkDeviceSize m_BufferImageGranularity;
        3591  uint32_t m_MemTypeIndex;
        3592  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
        3593  VkDeviceSize m_BytesMoved;
        3594  uint32_t m_AllocationsMoved;
        3595 
        3596  struct AllocationInfo
        3597  {
        3598  VmaAllocation m_hAllocation;
        3599  VkBool32* m_pChanged;
        3600 
        3601  AllocationInfo() :
        3602  m_hAllocation(VK_NULL_HANDLE),
        3603  m_pChanged(VMA_NULL)
        3604  {
        3605  }
        3606  };
        3607 
        3608  struct AllocationInfoSizeGreater
        3609  {
        3610  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
        3611  {
        3612  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
        3613  }
        3614  };
        3615 
        3616  // Used between AddAllocation and Defragment.
        3617  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
        3618 
        3619  struct BlockInfo
        3620  {
        3621  VmaBlock* m_pBlock;
        3622  bool m_HasNonMovableAllocations;
        3623  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
        3624 
        3625  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
        3626  m_pBlock(VMA_NULL),
        3627  m_HasNonMovableAllocations(true),
        3628  m_Allocations(pAllocationCallbacks),
        3629  m_pMappedDataForDefragmentation(VMA_NULL)
        3630  {
        3631  }
        3632 
        3633  void CalcHasNonMovableAllocations()
        3634  {
        3635  const size_t blockAllocCount =
        3636  m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount;
        3637  const size_t defragmentAllocCount = m_Allocations.size();
        3638  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
        3639  }
        3640 
        3641  void SortAllocationsBySizeDescecnding()
        3642  {
        3643  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
        3644  }
        3645 
        3646  VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData)
        3647  {
        3648  // It has already been mapped for defragmentation.
        3649  if(m_pMappedDataForDefragmentation)
        3650  {
        3651  *ppMappedData = m_pMappedDataForDefragmentation;
        3652  return VK_SUCCESS;
        3653  }
        3654 
        3655  // It is persistently mapped.
        3656  if(m_pBlock->m_PersistentMap)
        3657  {
        3658  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
        3659  *ppMappedData = m_pBlock->m_pMappedData;
        3660  return VK_SUCCESS;
        3661  }
        3662 
        3663  // Map on first usage.
        3664  VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation);
        3665  *ppMappedData = m_pMappedDataForDefragmentation;
        3666  return res;
        3667  }
        3668 
        3669  void Unmap(VkDevice hDevice)
        3670  {
        3671  if(m_pMappedDataForDefragmentation != VMA_NULL)
        3672  {
        3673  vkUnmapMemory(hDevice, m_pBlock->m_hMemory);
        3674  }
        3675  }
        3676 
        3677  private:
        3678  // Not null if mapped for defragmentation only, not persistently mapped.
        3679  void* m_pMappedDataForDefragmentation;
        3680  };
        3681 
        3682  struct BlockPointerLess
        3683  {
        3684  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaBlock* pRhsBlock) const
        3685  {
        3686  return pLhsBlockInfo->m_pBlock < pRhsBlock;
        3687  }
        3688  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
        3689  {
        3690  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
        3691  }
        3692  };
        3693 
        3694  // 1. Blocks with some non-movable allocations go first.
        3695  // 2. Blocks with smaller sumFreeSize go first.
        3696  struct BlockInfoCompareMoveDestination
        3697  {
        3698  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
        3699  {
        3700  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
        3701  {
        3702  return true;
        3703  }
        3704  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
        3705  {
        3706  return false;
        3707  }
        3708  if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize)
        3709  {
        3710  return true;
        3711  }
        3712  return false;
        3713  }
        3714  };
        3715 
        3716  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
        3717  BlockInfoVector m_Blocks;
        3718 
        3719  VkResult DefragmentRound(
        3720  VkDeviceSize maxBytesToMove,
        3721  uint32_t maxAllocationsToMove);
        3722 
        3723  static bool MoveMakesSense(
        3724  size_t dstBlockIndex, VkDeviceSize dstOffset,
        3725  size_t srcBlockIndex, VkDeviceSize srcOffset);
        3726 
        3727 public:
        3728  VmaDefragmentator(
        3729  VkDevice hDevice,
        3730  const VkAllocationCallbacks* pAllocationCallbacks,
        3731  VkDeviceSize bufferImageGranularity,
        3732  uint32_t memTypeIndex,
        3733  VMA_BLOCK_VECTOR_TYPE blockVectorType);
        3734 
        3735  ~VmaDefragmentator();
        3736 
        3737  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
        3738  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
        3739 
        3740  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
        3741 
        3742  VkResult Defragment(
        3743  VmaBlockVector* pBlockVector,
        3744  VkDeviceSize maxBytesToMove,
        3745  uint32_t maxAllocationsToMove);
        3746 };
        3747 
        3748 VmaDefragmentator::VmaDefragmentator(
        3749  VkDevice hDevice,
        3750  const VkAllocationCallbacks* pAllocationCallbacks,
        3751  VkDeviceSize bufferImageGranularity,
        3752  uint32_t memTypeIndex,
        3753  VMA_BLOCK_VECTOR_TYPE blockVectorType) :
        3754  m_hDevice(hDevice),
        3755  m_pAllocationCallbacks(pAllocationCallbacks),
        3756  m_BufferImageGranularity(bufferImageGranularity),
        3757  m_MemTypeIndex(memTypeIndex),
        3758  m_BlockVectorType(blockVectorType),
        3759  m_BytesMoved(0),
        3760  m_AllocationsMoved(0),
        3761  m_Allocations(VmaStlAllocator<AllocationInfo>(pAllocationCallbacks)),
        3762  m_Blocks(VmaStlAllocator<BlockInfo*>(pAllocationCallbacks))
        3763 {
        3764 }
        3765 
        3766 VmaDefragmentator::~VmaDefragmentator()
        3767 {
        3768  for(size_t i = m_Blocks.size(); i--; )
        3769  {
        3770  vma_delete(m_pAllocationCallbacks, m_Blocks[i]);
        3771  }
        3772 }
        3773 
        3774 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
        3775 {
        3776  AllocationInfo allocInfo;
        3777  allocInfo.m_hAllocation = hAlloc;
        3778  allocInfo.m_pChanged = pChanged;
        3779  m_Allocations.push_back(allocInfo);
        3780 }
        3781 
        3782 VkResult VmaDefragmentator::DefragmentRound(
        3783  VkDeviceSize maxBytesToMove,
        3784  uint32_t maxAllocationsToMove)
        3785 {
        3786  if(m_Blocks.empty())
        3787  {
        3788  return VK_SUCCESS;
        3789  }
        3790 
        3791  size_t srcBlockIndex = m_Blocks.size() - 1;
        3792  size_t srcAllocIndex = SIZE_MAX;
        3793  for(;;)
        3794  {
        3795  // 1. Find next allocation to move.
        3796  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
        3797  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
        3798  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
        3799  {
        3800  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
        3801  {
        3802  // Finished: no more allocations to process.
        3803  if(srcBlockIndex == 0)
        3804  {
        3805  return VK_SUCCESS;
        3806  }
        3807  else
        3808  {
        3809  --srcBlockIndex;
        3810  srcAllocIndex = SIZE_MAX;
        3811  }
        3812  }
        3813  else
        3814  {
        3815  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
        3816  }
        3817  }
        3818 
        3819  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
        3820  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
        3821 
        3822  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
        3823  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
        3824  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
        3825  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
        3826 
        3827  // 2. Try to find new place for this allocation in preceding or current block.
        3828  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
        3829  {
        3830  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
        3831  VmaAllocationRequest dstAllocRequest;
        3832  if(pDstBlockInfo->m_pBlock->CreateAllocationRequest(
        3833  m_BufferImageGranularity,
        3834  size,
        3835  alignment,
        3836  suballocType,
        3837  &dstAllocRequest) &&
        3838  MoveMakesSense(
        3839  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
        3840  {
        3841  // Reached limit on number of allocations or bytes to move.
        3842  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
        3843  (m_BytesMoved + size > maxBytesToMove))
        3844  {
        3845  return VK_INCOMPLETE;
        3846  }
        3847 
        3848  void* pDstMappedData = VMA_NULL;
        3849  VkResult res = pDstBlockInfo->EnsureMapping(m_hDevice, &pDstMappedData);
        3850  if(res != VK_SUCCESS)
        3851  {
        3852  return res;
        3853  }
        3854 
        3855  void* pSrcMappedData = VMA_NULL;
        3856  res = pSrcBlockInfo->EnsureMapping(m_hDevice, &pSrcMappedData);
        3857  if(res != VK_SUCCESS)
        3858  {
        3859  return res;
        3860  }
        3861 
        3862  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
        3863  memcpy(
        3864  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
        3865  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
        3866  size);
        3867 
        3868  pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size);
        3869  pSrcBlockInfo->m_pBlock->Free(allocInfo.m_hAllocation);
        3870 
        3871  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
        3872 
        3873  if(allocInfo.m_pChanged != VMA_NULL)
        3874  {
        3875  *allocInfo.m_pChanged = VK_TRUE;
        3876  }
        3877 
        3878  ++m_AllocationsMoved;
        3879  m_BytesMoved += size;
        3880 
        3881  VectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
        3882 
        3883  break;
        3884  }
        3885  }
        3886 
        3887  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
        3888 
        3889  if(srcAllocIndex > 0)
        3890  {
        3891  --srcAllocIndex;
        3892  }
        3893  else
        3894  {
        3895  if(srcBlockIndex > 0)
        3896  {
        3897  --srcBlockIndex;
        3898  srcAllocIndex = SIZE_MAX;
        3899  }
        3900  else
        3901  {
        3902  return VK_SUCCESS;
        3903  }
        3904  }
        3905  }
        3906 }
        3907 
        3908 VkResult VmaDefragmentator::Defragment(
        3909  VmaBlockVector* pBlockVector,
        3910  VkDeviceSize maxBytesToMove,
        3911  uint32_t maxAllocationsToMove)
        3912 {
        3913  if(m_Allocations.empty())
        3914  {
        3915  return VK_SUCCESS;
        3916  }
        3917 
        3918  // Create block info for each block.
        3919  const size_t blockCount = pBlockVector->m_Blocks.size();
        3920  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        3921  {
        3922  BlockInfo* pBlockInfo = vma_new(m_pAllocationCallbacks, BlockInfo)(m_pAllocationCallbacks);
        3923  pBlockInfo->m_pBlock = pBlockVector->m_Blocks[blockIndex];
        3924  m_Blocks.push_back(pBlockInfo);
        3925  }
        3926 
        3927  // Sort them by m_pBlock pointer value.
        3928  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
        3929 
        3930  // Move allocation infos from m_Allocations to appropriate m_Blocks[i].m_Allocations.
        3931  for(size_t allocIndex = 0, allocCount = m_Allocations.size(); allocIndex < allocCount; ++allocIndex)
        3932  {
        3933  AllocationInfo& allocInfo = m_Allocations[allocIndex];
        3934  VmaBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
        3935  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
        3936  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
        3937  {
        3938  (*it)->m_Allocations.push_back(allocInfo);
        3939  }
        3940  else
        3941  {
        3942  VMA_ASSERT(0);
        3943  }
        3944  }
        3945  m_Allocations.clear();
        3946 
        3947  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        3948  {
        3949  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
        3950  pBlockInfo->CalcHasNonMovableAllocations();
        3951  pBlockInfo->SortAllocationsBySizeDescecnding();
        3952  }
        3953 
        3954  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
        3955  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
        3956 
        3957  // Execute defragmentation round (the main part).
        3958  VkResult result = VK_SUCCESS;
        3959  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
        3960  {
        3961  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
        3962  }
        3963 
        3964  // Unmap blocks that were mapped for defragmentation.
        3965  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        3966  {
        3967  m_Blocks[blockIndex]->Unmap(m_hDevice);
        3968  }
        3969 
        3970  return result;
        3971 }
        3972 
        3973 bool VmaDefragmentator::MoveMakesSense(
        3974  size_t dstBlockIndex, VkDeviceSize dstOffset,
        3975  size_t srcBlockIndex, VkDeviceSize srcOffset)
        3976 {
        3977  if(dstBlockIndex < srcBlockIndex)
        3978  {
        3979  return true;
        3980  }
        3981  if(dstBlockIndex > srcBlockIndex)
        3982  {
        3983  return false;
        3984  }
        3985  if(dstOffset < srcOffset)
        3986  {
        3987  return true;
        3988  }
        3989  return false;
        3990 }
        3991 
        3993 // VmaAllocator_T
        3994 
        3995 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
        3996  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
        3997  m_PhysicalDevice(pCreateInfo->physicalDevice),
        3998  m_hDevice(pCreateInfo->device),
        3999  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
        4000  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
        4001  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
        4002  m_PreferredLargeHeapBlockSize(0),
        4003  m_PreferredSmallHeapBlockSize(0),
        4004  m_UnmapPersistentlyMappedMemoryCounter(0)
        4005 {
        4006  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
        4007 
        4008  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
        4009  memset(&m_MemProps, 0, sizeof(m_MemProps));
        4010  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
        4011 
        4012  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
        4013  memset(&m_HasEmptyBlock, 0, sizeof(m_HasEmptyBlock));
        4014  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
        4015 
        4016  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
        4017  {
        4018  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
        4019  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
        4020  }
        4021 
        4022  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
        4023  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
        4024  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
        4025  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
        4026 
        4027  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
        4028  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
        4029 
        4030  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
        4031  {
        4032  for(size_t j = 0; j < VMA_BLOCK_VECTOR_TYPE_COUNT; ++j)
        4033  {
        4034  m_pBlockVectors[i][j] = vma_new(this, VmaBlockVector)(this);
        4035  m_pOwnAllocations[i][j] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
        4036  }
        4037  }
        4038 }
        4039 
        4040 VmaAllocator_T::~VmaAllocator_T()
        4041 {
        4042  for(size_t i = GetMemoryTypeCount(); i--; )
        4043  {
        4044  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
        4045  {
        4046  vma_delete(this, m_pOwnAllocations[i][j]);
        4047  vma_delete(this, m_pBlockVectors[i][j]);
        4048  }
        4049  }
        4050 }
        4051 
        4052 VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const
        4053 {
        4054  VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
        4055  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
        4056  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
        4057 }
        4058 
        4059 VkResult VmaAllocator_T::AllocateMemoryOfType(
        4060  const VkMemoryRequirements& vkMemReq,
        4061  const VmaMemoryRequirements& vmaMemReq,
        4062  uint32_t memTypeIndex,
        4063  VmaSuballocationType suballocType,
        4064  VmaAllocation* pAllocation)
        4065 {
        4066  VMA_ASSERT(pAllocation != VMA_NULL);
        4067  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
        4068 
        4069  const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex);
        4070  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
        4071  const bool ownMemory =
        4072  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 ||
        4073  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
        4074  ((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) == 0 &&
        4075  vkMemReq.size > preferredBlockSize / 2);
        4076 
        4077  if(ownMemory)
        4078  {
        4079  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
        4080  {
        4081  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        4082  }
        4083  else
        4084  {
        4085  return AllocateOwnMemory(
        4086  vkMemReq.size,
        4087  suballocType,
        4088  memTypeIndex,
        4089  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
        4090  vmaMemReq.pUserData,
        4091  pAllocation);
        4092  }
        4093  }
        4094  else
        4095  {
        4096  uint32_t blockVectorType = VmaMemoryRequirementFlagsToBlockVectorType(vmaMemReq.flags);
        4097 
        4098  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4099  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        4100  VMA_ASSERT(blockVector);
        4101 
        4102  // 1. Search existing allocations.
        4103  // Forward order - prefer blocks with smallest amount of free space.
        4104  for(size_t allocIndex = 0; allocIndex < blockVector->m_Blocks.size(); ++allocIndex )
        4105  {
        4106  VmaBlock* const pBlock = blockVector->m_Blocks[allocIndex];
        4107  VMA_ASSERT(pBlock);
        4108  VmaAllocationRequest allocRequest = {};
        4109  // Check if can allocate from pBlock.
        4110  if(pBlock->CreateAllocationRequest(
        4111  GetBufferImageGranularity(),
        4112  vkMemReq.size,
        4113  vkMemReq.alignment,
        4114  suballocType,
        4115  &allocRequest))
        4116  {
        4117  // We no longer have an empty Allocation.
        4118  if(pBlock->IsEmpty())
        4119  {
        4120  m_HasEmptyBlock[memTypeIndex] = false;
        4121  }
        4122  // Allocate from this pBlock.
        4123  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
        4124  *pAllocation = vma_new(this, VmaAllocation_T)();
        4125  (*pAllocation)->InitBlockAllocation(
        4126  pBlock,
        4127  allocRequest.offset,
        4128  vkMemReq.alignment,
        4129  vkMemReq.size,
        4130  suballocType,
        4131  vmaMemReq.pUserData);
        4132  VMA_HEAVY_ASSERT(pBlock->Validate());
        4133  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex);
        4134  return VK_SUCCESS;
        4135  }
        4136  }
        4137 
        4138  // 2. Create new Allocation.
        4139  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
        4140  {
        4141  VMA_DEBUG_LOG(" FAILED due to VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT");
        4142  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        4143  }
        4144  else
        4145  {
        4146  // Start with full preferredBlockSize.
        4147  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
        4148  allocInfo.memoryTypeIndex = memTypeIndex;
        4149  allocInfo.allocationSize = preferredBlockSize;
        4150  VkDeviceMemory mem = VK_NULL_HANDLE;
        4151  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
        4152  if(res < 0)
        4153  {
        4154  // 3. Try half the size.
        4155  allocInfo.allocationSize /= 2;
        4156  if(allocInfo.allocationSize >= vkMemReq.size)
        4157  {
        4158  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
        4159  if(res < 0)
        4160  {
        4161  // 4. Try quarter the size.
        4162  allocInfo.allocationSize /= 2;
        4163  if(allocInfo.allocationSize >= vkMemReq.size)
        4164  {
        4165  res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem);
        4166  }
        4167  }
        4168  }
        4169  }
        4170  if(res < 0)
        4171  {
        4172  // 5. Try OwnAlloc.
        4173  res = AllocateOwnMemory(
        4174  vkMemReq.size,
        4175  suballocType,
        4176  memTypeIndex,
        4177  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0,
        4178  vmaMemReq.pUserData,
        4179  pAllocation);
        4180  if(res == VK_SUCCESS)
        4181  {
        4182  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
        4183  VMA_DEBUG_LOG(" Allocated as OwnMemory");
        4184  return VK_SUCCESS;
        4185  }
        4186  else
        4187  {
        4188  // Everything failed: Return error code.
        4189  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
        4190  return res;
        4191  }
        4192  }
        4193 
        4194  // New VkDeviceMemory successfully created.
        4195 
        4196  // Map memory if needed.
        4197  void* pMappedData = VMA_NULL;
        4198  const bool persistentMap = (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0;
        4199  if(persistentMap && m_UnmapPersistentlyMappedMemoryCounter == 0)
        4200  {
        4201  res = vkMapMemory(m_hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData);
        4202  if(res < 0)
        4203  {
        4204  VMA_DEBUG_LOG(" vkMapMemory FAILED");
        4205  vkFreeMemory(m_hDevice, mem, GetAllocationCallbacks());
        4206  return res;
        4207  }
        4208  }
        4209 
        4210  // Callback.
        4211  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
        4212  {
        4213  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, mem, allocInfo.allocationSize);
        4214  }
        4215 
        4216  // Create new Allocation for it.
        4217  VmaBlock* const pBlock = vma_new(this, VmaBlock)(this);
        4218  pBlock->Init(
        4219  memTypeIndex,
        4220  (VMA_BLOCK_VECTOR_TYPE)blockVectorType,
        4221  mem,
        4222  allocInfo.allocationSize,
        4223  persistentMap,
        4224  pMappedData);
        4225 
        4226  blockVector->m_Blocks.push_back(pBlock);
        4227 
        4228  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
        4229  VmaAllocationRequest allocRequest = {};
        4230  allocRequest.freeSuballocationItem = pBlock->m_Suballocations.begin();
        4231  allocRequest.offset = 0;
        4232  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size);
        4233  *pAllocation = vma_new(this, VmaAllocation_T)();
        4234  (*pAllocation)->InitBlockAllocation(
        4235  pBlock,
        4236  allocRequest.offset,
        4237  vkMemReq.alignment,
        4238  vkMemReq.size,
        4239  suballocType,
        4240  vmaMemReq.pUserData);
        4241  VMA_HEAVY_ASSERT(pBlock->Validate());
        4242  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
        4243  return VK_SUCCESS;
        4244  }
        4245  }
        4246 }
        4247 
        4248 VkResult VmaAllocator_T::AllocateOwnMemory(
        4249  VkDeviceSize size,
        4250  VmaSuballocationType suballocType,
        4251  uint32_t memTypeIndex,
        4252  bool map,
        4253  void* pUserData,
        4254  VmaAllocation* pAllocation)
        4255 {
        4256  VMA_ASSERT(pAllocation);
        4257 
        4258  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
        4259  allocInfo.memoryTypeIndex = memTypeIndex;
        4260  allocInfo.allocationSize = size;
        4261 
        4262  // Allocate VkDeviceMemory.
        4263  VkDeviceMemory hMemory = VK_NULL_HANDLE;
        4264  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &hMemory);
        4265  if(res < 0)
        4266  {
        4267  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
        4268  return res;
        4269  }
        4270 
        4271  void* pMappedData = nullptr;
        4272  if(map)
        4273  {
        4274  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
        4275  {
        4276  res = vkMapMemory(m_hDevice, hMemory, 0, VK_WHOLE_SIZE, 0, &pMappedData);
        4277  if(res < 0)
        4278  {
        4279  VMA_DEBUG_LOG(" vkMapMemory FAILED");
        4280  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
        4281  return res;
        4282  }
        4283  }
        4284  }
        4285 
        4286  // Callback.
        4287  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
        4288  {
        4289  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, hMemory, size);
        4290  }
        4291 
        4292  *pAllocation = vma_new(this, VmaAllocation_T)();
        4293  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
        4294 
        4295  // Register it in m_pOwnAllocations.
        4296  {
        4297  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        4298  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
        4299  VMA_ASSERT(pOwnAllocations);
        4300  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
        4301  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
        4302  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
        4303  pOwnAllocationsBeg,
        4304  pOwnAllocationsEnd,
        4305  *pAllocation,
        4306  VmaPointerLess()) - pOwnAllocationsBeg;
        4307  VectorInsert(*pOwnAllocations, indexToInsert, *pAllocation);
        4308  }
        4309 
        4310  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
        4311 
        4312  return VK_SUCCESS;
        4313 }
        4314 
        4315 VkResult VmaAllocator_T::AllocateMemory(
        4316  const VkMemoryRequirements& vkMemReq,
        4317  const VmaMemoryRequirements& vmaMemReq,
        4318  VmaSuballocationType suballocType,
        4319  VmaAllocation* pAllocation)
        4320 {
        4321  if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 &&
        4322  (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0)
        4323  {
        4324  VMA_ASSERT(0 && "Specifying VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT together with VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT makes no sense.");
        4325  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        4326  }
        4327 
        4328  // Bit mask of memory Vulkan types acceptable for this allocation.
        4329  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
        4330  uint32_t memTypeIndex = UINT32_MAX;
        4331  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
        4332  if(res == VK_SUCCESS)
        4333  {
        4334  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
        4335  // Succeeded on first try.
        4336  if(res == VK_SUCCESS)
        4337  {
        4338  return res;
        4339  }
        4340  // Allocation from this memory type failed. Try other compatible memory types.
        4341  else
        4342  {
        4343  for(;;)
        4344  {
        4345  // Remove old memTypeIndex from list of possibilities.
        4346  memoryTypeBits &= ~(1u << memTypeIndex);
        4347  // Find alternative memTypeIndex.
        4348  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex);
        4349  if(res == VK_SUCCESS)
        4350  {
        4351  res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation);
        4352  // Allocation from this alternative memory type succeeded.
        4353  if(res == VK_SUCCESS)
        4354  {
        4355  return res;
        4356  }
        4357  // else: Allocation from this memory type failed. Try next one - next loop iteration.
        4358  }
        4359  // No other matching memory type index could be found.
        4360  else
        4361  {
        4362  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
        4363  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        4364  }
        4365  }
        4366  }
        4367  }
        4368  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
        4369  else
        4370  return res;
        4371 }
        4372 
        4373 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
        4374 {
        4375  VMA_ASSERT(allocation);
        4376 
        4377  if(allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
        4378  {
        4379  VmaBlock* pBlockToDelete = VMA_NULL;
        4380 
        4381  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
        4382  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
        4383  {
        4384  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4385 
        4386  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        4387  VmaBlock* pBlock = allocation->GetBlock();
        4388 
        4389  pBlock->Free(allocation);
        4390  VMA_HEAVY_ASSERT(pBlock->Validate());
        4391 
        4392  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
        4393 
        4394  // pBlock became empty after this deallocation.
        4395  if(pBlock->IsEmpty())
        4396  {
        4397  // Already has empty Allocation. We don't want to have two, so delete this one.
        4398  if(m_HasEmptyBlock[memTypeIndex])
        4399  {
        4400  pBlockToDelete = pBlock;
        4401  pBlockVector->Remove(pBlock);
        4402  }
        4403  // We now have first empty Allocation.
        4404  else
        4405  {
        4406  m_HasEmptyBlock[memTypeIndex] = true;
        4407  }
        4408  }
        4409  // Must be called after srcBlockIndex is used, because later it may become invalid!
        4410  pBlockVector->IncrementallySortBlocks();
        4411  }
        4412  // Destruction of a free Allocation. Deferred until this point, outside of mutex
        4413  // lock, for performance reason.
        4414  if(pBlockToDelete != VMA_NULL)
        4415  {
        4416  VMA_DEBUG_LOG(" Deleted empty allocation");
        4417  pBlockToDelete->Destroy(this);
        4418  vma_delete(this, pBlockToDelete);
        4419  }
        4420 
        4421  vma_delete(this, allocation);
        4422  }
        4423  else // VmaAllocation_T::ALLOCATION_TYPE_OWN
        4424  {
        4425  FreeOwnMemory(allocation);
        4426  }
        4427 }
        4428 
        4429 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
        4430 {
        4431  InitStatInfo(pStats->total);
        4432  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
        4433  InitStatInfo(pStats->memoryType[i]);
        4434  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
        4435  InitStatInfo(pStats->memoryHeap[i]);
        4436 
        4437  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        4438  {
        4439  VmaMutexLock allocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4440  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
        4441  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        4442  {
        4443  const VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        4444  VMA_ASSERT(pBlockVector);
        4445  pBlockVector->AddStats(pStats, memTypeIndex, heapIndex);
        4446  }
        4447  }
        4448 
        4449  VmaPostprocessCalcStatInfo(pStats->total);
        4450  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
        4451  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
        4452  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
        4453  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
        4454 }
        4455 
        4456 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
        4457 
        4458 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
        4459 {
        4460  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
        4461  {
        4462  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
        4463  {
        4464  for(size_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
        4465  {
        4466  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        4467  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
        4468  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        4469  {
        4470  // Process OwnAllocations.
        4471  {
        4472  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        4473  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        4474  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
        4475  {
        4476  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
        4477  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(m_hDevice);
        4478  }
        4479  }
        4480 
        4481  // Process normal Allocations.
        4482  {
        4483  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4484  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        4485  pBlockVector->UnmapPersistentlyMappedMemory();
        4486  }
        4487  }
        4488  }
        4489  }
        4490  }
        4491 }
        4492 
        4493 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
        4494 {
        4495  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
        4496  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
        4497  {
        4498  VkResult finalResult = VK_SUCCESS;
        4499  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
        4500  {
        4501  for(size_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
        4502  {
        4503  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        4504  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
        4505  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        4506  {
        4507  // Process OwnAllocations.
        4508  {
        4509  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        4510  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        4511  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
        4512  {
        4513  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
        4514  hAlloc->OwnAllocMapPersistentlyMappedMemory(m_hDevice);
        4515  }
        4516  }
        4517 
        4518  // Process normal Allocations.
        4519  {
        4520  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4521  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        4522  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
        4523  if(localResult != VK_SUCCESS)
        4524  {
        4525  finalResult = localResult;
        4526  }
        4527  }
        4528  }
        4529  }
        4530  }
        4531  return finalResult;
        4532  }
        4533  else
        4534  return VK_SUCCESS;
        4535 }
        4536 
        4537 VkResult VmaAllocator_T::Defragment(
        4538  VmaAllocation* pAllocations,
        4539  size_t allocationCount,
        4540  VkBool32* pAllocationsChanged,
        4541  const VmaDefragmentationInfo* pDefragmentationInfo,
        4542  VmaDefragmentationStats* pDefragmentationStats)
        4543 {
        4544  if(pAllocationsChanged != VMA_NULL)
        4545  {
        4546  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
        4547  }
        4548  if(pDefragmentationStats != VMA_NULL)
        4549  {
        4550  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
        4551  }
        4552 
        4553  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
        4554  {
        4555  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
        4556  return VK_ERROR_MEMORY_MAP_FAILED;
        4557  }
        4558 
        4559  // Initialize defragmentators per memory type.
        4560  const VkDeviceSize bufferImageGranularity = GetBufferImageGranularity();
        4561  VmaDefragmentator* pDefragmentators[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
        4562  memset(pDefragmentators, 0, sizeof(pDefragmentators));
        4563  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        4564  {
        4565  // Only HOST_VISIBLE memory types can be defragmented.
        4566  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        4567  {
        4568  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        4569  {
        4570  pDefragmentators[memTypeIndex][blockVectorType] = vma_new(this, VmaDefragmentator)(
        4571  m_hDevice,
        4572  GetAllocationCallbacks(),
        4573  bufferImageGranularity,
        4574  memTypeIndex,
        4575  (VMA_BLOCK_VECTOR_TYPE)blockVectorType);
        4576  }
        4577  }
        4578  }
        4579 
        4580  // Dispatch pAllocations among defragmentators.
        4581  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
        4582  {
        4583  VmaAllocation hAlloc = pAllocations[allocIndex];
        4584  VMA_ASSERT(hAlloc);
        4585  if(hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK)
        4586  {
        4587  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
        4588  // Only HOST_VISIBLE memory types can be defragmented.
        4589  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        4590  {
        4591  const VMA_BLOCK_VECTOR_TYPE blockVectorType = hAlloc->GetBlockVectorType();
        4592  VkBool32* pChanged = (pAllocationsChanged != VMA_NULL) ?
        4593  &pAllocationsChanged[allocIndex] : VMA_NULL;
        4594  pDefragmentators[memTypeIndex][blockVectorType]->AddAllocation(hAlloc, pChanged);
        4595  }
        4596  // else: skip this allocation, cannot move it.
        4597  }
        4598  // else ALLOCATION_TYPE_OWN: skip this allocation, nothing to defragment.
        4599  }
        4600 
        4601  VkResult result = VK_SUCCESS;
        4602 
        4603  // Main processing.
        4604  VkDeviceSize maxBytesToMove = SIZE_MAX;
        4605  uint32_t maxAllocationsToMove = UINT32_MAX;
        4606  if(pDefragmentationInfo != VMA_NULL)
        4607  {
        4608  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
        4609  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
        4610  }
        4611  for(uint32_t memTypeIndex = 0;
        4612  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
        4613  ++memTypeIndex)
        4614  {
        4615  // Only HOST_VISIBLE memory types can be defragmented.
        4616  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        4617  {
        4618  VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4619 
        4620  for(uint32_t blockVectorType = 0;
        4621  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
        4622  ++blockVectorType)
        4623  {
        4624  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        4625 
        4626  // Defragment.
        4627  result = pDefragmentators[memTypeIndex][blockVectorType]->Defragment(pBlockVector, maxBytesToMove, maxAllocationsToMove);
        4628 
        4629  // Accumulate statistics.
        4630  if(pDefragmentationStats != VMA_NULL)
        4631  {
        4632  const VkDeviceSize bytesMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetBytesMoved();
        4633  const uint32_t allocationsMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetAllocationsMoved();
        4634  pDefragmentationStats->bytesMoved += bytesMoved;
        4635  pDefragmentationStats->allocationsMoved += allocationsMoved;
        4636  VMA_ASSERT(bytesMoved <= maxBytesToMove);
        4637  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
        4638  maxBytesToMove -= bytesMoved;
        4639  maxAllocationsToMove -= allocationsMoved;
        4640  }
        4641 
        4642  // Free empty blocks.
        4643  for(size_t blockIndex = pBlockVector->m_Blocks.size(); blockIndex--; )
        4644  {
        4645  VmaBlock* pBlock = pBlockVector->m_Blocks[blockIndex];
        4646  if(pBlock->IsEmpty())
        4647  {
        4648  if(pDefragmentationStats != VMA_NULL)
        4649  {
        4650  ++pDefragmentationStats->deviceMemoryBlocksFreed;
        4651  pDefragmentationStats->bytesFreed += pBlock->m_Size;
        4652  }
        4653 
        4654  VectorRemove(pBlockVector->m_Blocks, blockIndex);
        4655  pBlock->Destroy(this);
        4656  vma_delete(this, pBlock);
        4657  }
        4658  }
        4659 
        4660  // All block vector types processed: we can be sure that all empty allocations have been freed.
        4661  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_COUNT - 1)
        4662  {
        4663  m_HasEmptyBlock[memTypeIndex] = false;
        4664  }
        4665  }
        4666  }
        4667  }
        4668 
        4669  // Destroy defragmentators.
        4670  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
        4671  {
        4672  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
        4673  {
        4674  vma_delete(this, pDefragmentators[memTypeIndex][blockVectorType]);
        4675  }
        4676  }
        4677 
        4678  return result;
        4679 }
        4680 
        4681 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
        4682 {
        4683  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
        4684  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
        4685  pAllocationInfo->offset = hAllocation->GetOffset();
        4686  pAllocationInfo->size = hAllocation->GetSize();
        4687  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
        4688  pAllocationInfo->pUserData = hAllocation->GetUserData();
        4689 }
        4690 
        4691 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
        4692 {
        4693  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
        4694 
        4695  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
        4696  {
        4697  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        4698  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
        4699  VMA_ASSERT(pOwnAllocations);
        4700  VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data();
        4701  VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size();
        4702  VmaAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess(
        4703  pOwnAllocationsBeg,
        4704  pOwnAllocationsEnd,
        4705  allocation,
        4706  VmaPointerLess());
        4707  if(pOwnAllocationIt != pOwnAllocationsEnd)
        4708  {
        4709  const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg;
        4710  VectorRemove(*pOwnAllocations, ownAllocationIndex);
        4711  }
        4712  else
        4713  {
        4714  VMA_ASSERT(0);
        4715  }
        4716  }
        4717 
        4718  VkDeviceMemory hMemory = allocation->GetMemory();
        4719 
        4720  // Callback.
        4721  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
        4722  {
        4723  (*m_DeviceMemoryCallbacks.pfnFree)(this, memTypeIndex, hMemory, allocation->GetSize());
        4724  }
        4725 
        4726  if(allocation->GetMappedData() != VMA_NULL)
        4727  {
        4728  vkUnmapMemory(m_hDevice, hMemory);
        4729  }
        4730 
        4731  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
        4732 
        4733  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
        4734 
        4735  vma_delete(this, allocation);
        4736 }
        4737 
        4738 #if VMA_STATS_STRING_ENABLED
        4739 
        4740 void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb)
        4741 {
        4742  bool ownAllocationsStarted = false;
        4743  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        4744  {
        4745  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        4746  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        4747  {
        4748  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
        4749  VMA_ASSERT(pOwnAllocVector);
        4750  if(pOwnAllocVector->empty() == false)
        4751  {
        4752  if(ownAllocationsStarted)
        4753  {
        4754  sb.Add(",\n\t\"Type ");
        4755  }
        4756  else
        4757  {
        4758  sb.Add(",\n\"OwnAllocations\": {\n\t\"Type ");
        4759  ownAllocationsStarted = true;
        4760  }
        4761  sb.AddNumber(memTypeIndex);
        4762  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
        4763  {
        4764  sb.Add(" Mapped");
        4765  }
        4766  sb.Add("\": [");
        4767 
        4768  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
        4769  {
        4770  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
        4771  if(i > 0)
        4772  {
        4773  sb.Add(",\n\t\t{ \"Size\": ");
        4774  }
        4775  else
        4776  {
        4777  sb.Add("\n\t\t{ \"Size\": ");
        4778  }
        4779  sb.AddNumber(hAlloc->GetSize());
        4780  sb.Add(", \"Type\": ");
        4781  sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
        4782  sb.Add(" }");
        4783  }
        4784 
        4785  sb.Add("\n\t]");
        4786  }
        4787  }
        4788  }
        4789  if(ownAllocationsStarted)
        4790  {
        4791  sb.Add("\n}");
        4792  }
        4793 
        4794  {
        4795  bool allocationsStarted = false;
        4796  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        4797  {
        4798  VmaMutexLock globalAllocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex);
        4799  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        4800  {
        4801  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
        4802  {
        4803  if(allocationsStarted)
        4804  {
        4805  sb.Add(",\n\t\"Type ");
        4806  }
        4807  else
        4808  {
        4809  sb.Add(",\n\"Allocations\": {\n\t\"Type ");
        4810  allocationsStarted = true;
        4811  }
        4812  sb.AddNumber(memTypeIndex);
        4813  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
        4814  {
        4815  sb.Add(" Mapped");
        4816  }
        4817  sb.Add("\": [");
        4818 
        4819  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(sb);
        4820 
        4821  sb.Add("\n\t]");
        4822  }
        4823  }
        4824  }
        4825  if(allocationsStarted)
        4826  {
        4827  sb.Add("\n}");
        4828  }
        4829  }
        4830 }
        4831 
        4832 #endif // #if VMA_STATS_STRING_ENABLED
        4833 
        4834 static VkResult AllocateMemoryForImage(
        4835  VmaAllocator allocator,
        4836  VkImage image,
        4837  const VmaMemoryRequirements* pMemoryRequirements,
        4838  VmaSuballocationType suballocType,
        4839  VmaAllocation* pAllocation)
        4840 {
        4841  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pMemoryRequirements && pAllocation);
        4842 
        4843  VkMemoryRequirements vkMemReq = {};
        4844  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
        4845 
        4846  return allocator->AllocateMemory(
        4847  vkMemReq,
        4848  *pMemoryRequirements,
        4849  suballocType,
        4850  pAllocation);
        4851 }
        4852 
        4854 // Public interface
        4855 
        4856 VkResult vmaCreateAllocator(
        4857  const VmaAllocatorCreateInfo* pCreateInfo,
        4858  VmaAllocator* pAllocator)
        4859 {
        4860  VMA_ASSERT(pCreateInfo && pAllocator);
        4861  VMA_DEBUG_LOG("vmaCreateAllocator");
        4862  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
        4863  return VK_SUCCESS;
        4864 }
        4865 
        4866 void vmaDestroyAllocator(
        4867  VmaAllocator allocator)
        4868 {
        4869  if(allocator != VK_NULL_HANDLE)
        4870  {
        4871  VMA_DEBUG_LOG("vmaDestroyAllocator");
        4872  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
        4873  vma_delete(&allocationCallbacks, allocator);
        4874  }
        4875 }
        4876 
        4877 void vmaGetPhysicalDeviceProperties(
        4878  VmaAllocator allocator,
        4879  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
        4880 {
        4881  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
        4882  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
        4883 }
        4884 
        4885 void vmaGetMemoryProperties(
        4886  VmaAllocator allocator,
        4887  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
        4888 {
        4889  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
        4890  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
        4891 }
        4892 
        4893 void vmaGetMemoryTypeProperties(
        4894  VmaAllocator allocator,
        4895  uint32_t memoryTypeIndex,
        4896  VkMemoryPropertyFlags* pFlags)
        4897 {
        4898  VMA_ASSERT(allocator && pFlags);
        4899  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
        4900  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
        4901 }
        4902 
        4903 void vmaCalculateStats(
        4904  VmaAllocator allocator,
        4905  VmaStats* pStats)
        4906 {
        4907  VMA_ASSERT(allocator && pStats);
        4908  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        4909  allocator->CalculateStats(pStats);
        4910 }
        4911 
        4912 #if VMA_STATS_STRING_ENABLED
        4913 
        4914 void vmaBuildStatsString(
        4915  VmaAllocator allocator,
        4916  char** ppStatsString,
        4917  VkBool32 detailedMap)
        4918 {
        4919  VMA_ASSERT(allocator && ppStatsString);
        4920  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        4921 
        4922  VmaStringBuilder sb(allocator);
        4923  {
        4924  VmaStats stats;
        4925  allocator->CalculateStats(&stats);
        4926 
        4927  sb.Add("{\n\"Total\": ");
        4928  VmaPrintStatInfo(sb, stats.total);
        4929 
        4930  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
        4931  {
        4932  sb.Add(",\n\"Heap ");
        4933  sb.AddNumber(heapIndex);
        4934  sb.Add("\": {\n\t\"Size\": ");
        4935  sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
        4936  sb.Add(",\n\t\"Flags\": ");
        4937  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
        4938  {
        4939  sb.AddString("DEVICE_LOCAL");
        4940  }
        4941  else
        4942  {
        4943  sb.AddString("");
        4944  }
        4945  if(stats.memoryHeap[heapIndex].AllocationCount > 0)
        4946  {
        4947  sb.Add(",\n\t\"Stats:\": ");
        4948  VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]);
        4949  }
        4950 
        4951  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
        4952  {
        4953  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
        4954  {
        4955  sb.Add(",\n\t\"Type ");
        4956  sb.AddNumber(typeIndex);
        4957  sb.Add("\": {\n\t\t\"Flags\": \"");
        4958  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
        4959  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        4960  {
        4961  sb.Add(" DEVICE_LOCAL");
        4962  }
        4963  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        4964  {
        4965  sb.Add(" HOST_VISIBLE");
        4966  }
        4967  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
        4968  {
        4969  sb.Add(" HOST_COHERENT");
        4970  }
        4971  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
        4972  {
        4973  sb.Add(" HOST_CACHED");
        4974  }
        4975  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
        4976  {
        4977  sb.Add(" LAZILY_ALLOCATED");
        4978  }
        4979  sb.Add("\"");
        4980  if(stats.memoryType[typeIndex].AllocationCount > 0)
        4981  {
        4982  sb.Add(",\n\t\t\"Stats\": ");
        4983  VmaPrintStatInfo(sb, stats.memoryType[typeIndex]);
        4984  }
        4985  sb.Add("\n\t}");
        4986  }
        4987  }
        4988  sb.Add("\n}");
        4989  }
        4990  if(detailedMap == VK_TRUE)
        4991  {
        4992  allocator->PrintDetailedMap(sb);
        4993  }
        4994  sb.Add("\n}\n");
        4995  }
        4996 
        4997  const size_t len = sb.GetLength();
        4998  char* const pChars = vma_new_array(allocator, char, len + 1);
        4999  if(len > 0)
        5000  {
        5001  memcpy(pChars, sb.GetData(), len);
        5002  }
        5003  pChars[len] = '\0';
        5004  *ppStatsString = pChars;
        5005 }
        5006 
        5007 void vmaFreeStatsString(
        5008  VmaAllocator allocator,
        5009  char* pStatsString)
        5010 {
        5011  if(pStatsString != VMA_NULL)
        5012  {
        5013  VMA_ASSERT(allocator);
        5014  size_t len = strlen(pStatsString);
        5015  vma_delete_array(allocator, pStatsString, len + 1);
        5016  }
        5017 }
        5018 
        5019 #endif // #if VMA_STATS_STRING_ENABLED
        5020 
        5023 VkResult vmaFindMemoryTypeIndex(
        5024  VmaAllocator allocator,
        5025  uint32_t memoryTypeBits,
        5026  const VmaMemoryRequirements* pMemoryRequirements,
        5027  uint32_t* pMemoryTypeIndex)
        5028 {
        5029  VMA_ASSERT(allocator != VK_NULL_HANDLE);
        5030  VMA_ASSERT(pMemoryRequirements != VMA_NULL);
        5031  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
        5032 
        5033  uint32_t requiredFlags = pMemoryRequirements->requiredFlags;
        5034  uint32_t preferredFlags = pMemoryRequirements->preferredFlags;
        5035  if(preferredFlags == 0)
        5036  {
        5037  preferredFlags = requiredFlags;
        5038  }
        5039  // preferredFlags, if not 0, must be a superset of requiredFlags.
        5040  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
        5041 
        5042  // Convert usage to requiredFlags and preferredFlags.
        5043  switch(pMemoryRequirements->usage)
        5044  {
        5045  case VMA_MEMORY_USAGE_UNKNOWN:
        5046  break;
        5047  case VMA_MEMORY_USAGE_GPU_ONLY:
        5048  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
        5049  break;
        5050  case VMA_MEMORY_USAGE_CPU_ONLY:
        5051  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
        5052  break;
        5053  case VMA_MEMORY_USAGE_CPU_TO_GPU:
        5054  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        5055  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
        5056  break;
        5057  case VMA_MEMORY_USAGE_GPU_TO_CPU:
        5058  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        5059  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
        5060  break;
        5061  default:
        5062  break;
        5063  }
        5064 
        5065  if((pMemoryRequirements->flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0)
        5066  {
        5067  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        5068  }
        5069 
        5070  *pMemoryTypeIndex = UINT32_MAX;
        5071  uint32_t minCost = UINT32_MAX;
        5072  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
        5073  memTypeIndex < allocator->GetMemoryTypeCount();
        5074  ++memTypeIndex, memTypeBit <<= 1)
        5075  {
        5076  // This memory type is acceptable according to memoryTypeBits bitmask.
        5077  if((memTypeBit & memoryTypeBits) != 0)
        5078  {
        5079  const VkMemoryPropertyFlags currFlags =
        5080  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        5081  // This memory type contains requiredFlags.
        5082  if((requiredFlags & ~currFlags) == 0)
        5083  {
        5084  // Calculate cost as number of bits from preferredFlags not present in this memory type.
        5085  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
        5086  // Remember memory type with lowest cost.
        5087  if(currCost < minCost)
        5088  {
        5089  *pMemoryTypeIndex = memTypeIndex;
        5090  if(currCost == 0)
        5091  {
        5092  return VK_SUCCESS;
        5093  }
        5094  minCost = currCost;
        5095  }
        5096  }
        5097  }
        5098  }
        5099  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
        5100 }
        5101 
        5102 VkResult vmaAllocateMemory(
        5103  VmaAllocator allocator,
        5104  const VkMemoryRequirements* pVkMemoryRequirements,
        5105  const VmaMemoryRequirements* pVmaMemoryRequirements,
        5106  VmaAllocation* pAllocation,
        5107  VmaAllocationInfo* pAllocationInfo)
        5108 {
        5109  VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pAllocation);
        5110 
        5111  VMA_DEBUG_LOG("vmaAllocateMemory");
        5112 
        5113  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5114 
        5115  return allocator->AllocateMemory(
        5116  *pVkMemoryRequirements,
        5117  *pVmaMemoryRequirements,
        5118  VMA_SUBALLOCATION_TYPE_UNKNOWN,
        5119  pAllocation);
        5120 
        5121  if(pAllocationInfo)
        5122  {
        5123  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        5124  }
        5125 }
        5126 
        5127 VkResult vmaAllocateMemoryForBuffer(
        5128  VmaAllocator allocator,
        5129  VkBuffer buffer,
        5130  const VmaMemoryRequirements* pMemoryRequirements,
        5131  VmaAllocation* pAllocation,
        5132  VmaAllocationInfo* pAllocationInfo)
        5133 {
        5134  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
        5135 
        5136  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
        5137 
        5138  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5139 
        5140  VkMemoryRequirements vkMemReq = {};
        5141  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
        5142 
        5143  return allocator->AllocateMemory(
        5144  vkMemReq,
        5145  *pMemoryRequirements,
        5146  VMA_SUBALLOCATION_TYPE_BUFFER,
        5147  pAllocation);
        5148 
        5149  if(pAllocationInfo)
        5150  {
        5151  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        5152  }
        5153 }
        5154 
        5155 VkResult vmaAllocateMemoryForImage(
        5156  VmaAllocator allocator,
        5157  VkImage image,
        5158  const VmaMemoryRequirements* pMemoryRequirements,
        5159  VmaAllocation* pAllocation,
        5160  VmaAllocationInfo* pAllocationInfo)
        5161 {
        5162  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pAllocation);
        5163 
        5164  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
        5165 
        5166  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5167 
        5168  return AllocateMemoryForImage(
        5169  allocator,
        5170  image,
        5171  pMemoryRequirements,
        5172  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
        5173  pAllocation);
        5174 
        5175  if(pAllocationInfo)
        5176  {
        5177  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        5178  }
        5179 }
        5180 
        5181 void vmaFreeMemory(
        5182  VmaAllocator allocator,
        5183  VmaAllocation allocation)
        5184 {
        5185  VMA_ASSERT(allocator && allocation);
        5186 
        5187  VMA_DEBUG_LOG("vmaFreeMemory");
        5188 
        5189  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5190 
        5191  allocator->FreeMemory(allocation);
        5192 }
        5193 
        5194 void vmaGetAllocationInfo(
        5195  VmaAllocator allocator,
        5196  VmaAllocation allocation,
        5197  VmaAllocationInfo* pAllocationInfo)
        5198 {
        5199  VMA_ASSERT(allocator && allocation && pAllocationInfo);
        5200 
        5201  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5202 
        5203  allocator->GetAllocationInfo(allocation, pAllocationInfo);
        5204 }
        5205 
        5206 void vmaSetAllocationUserData(
        5207  VmaAllocator allocator,
        5208  VmaAllocation allocation,
        5209  void* pUserData)
        5210 {
        5211  VMA_ASSERT(allocator && allocation);
        5212 
        5213  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5214 
        5215  allocation->SetUserData(pUserData);
        5216 }
        5217 
        5218 VkResult vmaMapMemory(
        5219  VmaAllocator allocator,
        5220  VmaAllocation allocation,
        5221  void** ppData)
        5222 {
        5223  VMA_ASSERT(allocator && allocation && ppData);
        5224 
        5225  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5226 
        5227  return vkMapMemory(allocator->m_hDevice, allocation->GetMemory(),
        5228  allocation->GetOffset(), allocation->GetSize(), 0, ppData);
        5229 }
        5230 
        5231 void vmaUnmapMemory(
        5232  VmaAllocator allocator,
        5233  VmaAllocation allocation)
        5234 {
        5235  VMA_ASSERT(allocator && allocation);
        5236 
        5237  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5238 
        5239  vkUnmapMemory(allocator->m_hDevice, allocation->GetMemory());
        5240 }
        5241 
        5242 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
        5243 {
        5244  VMA_ASSERT(allocator);
        5245 
        5246  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5247 
        5248  allocator->UnmapPersistentlyMappedMemory();
        5249 }
        5250 
        5251 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
        5252 {
        5253  VMA_ASSERT(allocator);
        5254 
        5255  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5256 
        5257  return allocator->MapPersistentlyMappedMemory();
        5258 }
        5259 
        5260 VkResult vmaDefragment(
        5261  VmaAllocator allocator,
        5262  VmaAllocation* pAllocations,
        5263  size_t allocationCount,
        5264  VkBool32* pAllocationsChanged,
        5265  const VmaDefragmentationInfo *pDefragmentationInfo,
        5266  VmaDefragmentationStats* pDefragmentationStats)
        5267 {
        5268  VMA_ASSERT(allocator && pAllocations);
        5269 
        5270  VMA_DEBUG_LOG("vmaDefragment");
        5271 
        5272  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5273 
        5274  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
        5275 }
        5276 
        5277 VkResult vmaCreateBuffer(
        5278  VmaAllocator allocator,
        5279  const VkBufferCreateInfo* pCreateInfo,
        5280  const VmaMemoryRequirements* pMemoryRequirements,
        5281  VkBuffer* pBuffer,
        5282  VmaAllocation* pAllocation,
        5283  VmaAllocationInfo* pAllocationInfo)
        5284 {
        5285  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pBuffer && pAllocation);
        5286 
        5287  VMA_DEBUG_LOG("vmaCreateBuffer");
        5288 
        5289  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5290 
        5291  *pBuffer = VK_NULL_HANDLE;
        5292  *pAllocation = VK_NULL_HANDLE;
        5293 
        5294  // 1. Create VkBuffer.
        5295  VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
        5296  if(res >= 0)
        5297  {
        5298  // 2. vkGetBufferMemoryRequirements.
        5299  VkMemoryRequirements vkMemReq = {};
        5300  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
        5301 
        5302  // 3. Allocate memory using allocator.
        5303  res = allocator->AllocateMemory(
        5304  vkMemReq,
        5305  *pMemoryRequirements,
        5306  VMA_SUBALLOCATION_TYPE_BUFFER,
        5307  pAllocation);
        5308  if(res >= 0)
        5309  {
        5310  // 3. Bind buffer with memory.
        5311  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
        5312  if(res >= 0)
        5313  {
        5314  // All steps succeeded.
        5315  if(pAllocationInfo != VMA_NULL)
        5316  {
        5317  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        5318  }
        5319  return VK_SUCCESS;
        5320  }
        5321  allocator->FreeMemory(*pAllocation);
        5322  *pAllocation = VK_NULL_HANDLE;
        5323  return res;
        5324  }
        5325  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
        5326  *pBuffer = VK_NULL_HANDLE;
        5327  return res;
        5328  }
        5329  return res;
        5330 }
        5331 
        5332 void vmaDestroyBuffer(
        5333  VmaAllocator allocator,
        5334  VkBuffer buffer,
        5335  VmaAllocation allocation)
        5336 {
        5337  if(buffer != VK_NULL_HANDLE)
        5338  {
        5339  VMA_ASSERT(allocator);
        5340 
        5341  VMA_DEBUG_LOG("vmaDestroyBuffer");
        5342 
        5343  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5344 
        5345  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
        5346 
        5347  allocator->FreeMemory(allocation);
        5348  }
        5349 }
        5350 
        5351 VkResult vmaCreateImage(
        5352  VmaAllocator allocator,
        5353  const VkImageCreateInfo* pCreateInfo,
        5354  const VmaMemoryRequirements* pMemoryRequirements,
        5355  VkImage* pImage,
        5356  VmaAllocation* pAllocation,
        5357  VmaAllocationInfo* pAllocationInfo)
        5358 {
        5359  VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pImage && pAllocation);
        5360 
        5361  VMA_DEBUG_LOG("vmaCreateImage");
        5362 
        5363  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5364 
        5365  // 1. Create VkImage.
        5366  VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage);
        5367  if(res >= 0)
        5368  {
        5369  VkMappedMemoryRange mem = {};
        5370  VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
        5371  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
        5372  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
        5373 
        5374  // 2. Allocate memory using allocator.
        5375  res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, pAllocation);
        5376  if(res >= 0)
        5377  {
        5378  // 3. Bind image with memory.
        5379  res = vkBindImageMemory(allocator->m_hDevice, *pImage, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
        5380  if(res >= 0)
        5381  {
        5382  // All steps succeeded.
        5383  if(pAllocationInfo != VMA_NULL)
        5384  {
        5385  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        5386  }
        5387  return VK_SUCCESS;
        5388  }
        5389  allocator->FreeMemory(*pAllocation);
        5390  *pAllocation = VK_NULL_HANDLE;
        5391  return res;
        5392  }
        5393  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
        5394  *pImage = VK_NULL_HANDLE;
        5395  return res;
        5396  }
        5397  return res;
        5398 }
        5399 
        5400 void vmaDestroyImage(
        5401  VmaAllocator allocator,
        5402  VkImage image,
        5403  VmaAllocation allocation)
        5404 {
        5405  if(image != VK_NULL_HANDLE)
        5406  {
        5407  VMA_ASSERT(allocator);
        5408 
        5409  VMA_DEBUG_LOG("vmaDestroyImage");
        5410 
        5411  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        5412 
        5413  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
        5414 
        5415  allocator->FreeMemory(allocation);
        5416  }
        5417 }
        5418 
        5419 #endif // #ifdef VMA_IMPLEMENTATION
        VmaMemoryRequirementFlagBits
        VmaMemoryRequirementFlagBits
        Flags to be passed as VmaMemoryRequirements::flags.
        Definition: vk_mem_alloc.h:336
        -
        VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT
        Set this flag if the allocation should have its own memory block.
        Definition: vk_mem_alloc.h:345
        -
        VmaMemoryRequirements
        struct VmaMemoryRequirements VmaMemoryRequirements
        -
        VmaAllocatorCreateInfo::physicalDevice
        VkPhysicalDevice physicalDevice
        Vulkan physical device.
        Definition: vk_mem_alloc.h:214
        -
        vmaCreateBuffer
        VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        -
        VMA_MEMORY_USAGE_GPU_TO_CPU
        Memory will be used for writing on device and readback on host.
        Definition: vk_mem_alloc.h:331
        -
        VmaMemoryRequirements::usage
        VmaMemoryUsage usage
        Intended usage of memory.
        Definition: vk_mem_alloc.h:374
        -
        VmaAllocationInfo::deviceMemory
        VkDeviceMemory deviceMemory
        Handle to Vulkan memory object.
        Definition: vk_mem_alloc.h:431
        -
        VmaDefragmentationInfo::maxAllocationsToMove
        uint32_t maxAllocationsToMove
        Maximum number of allocations that can be moved to different place.
        Definition: vk_mem_alloc.h:567
        +Go to the documentation of this file.
        1 //
        2 // Copyright (c) 2017 Advanced Micro Devices, Inc. All rights reserved.
        3 //
        4 // Permission is hereby granted, free of charge, to any person obtaining a copy
        5 // of this software and associated documentation files (the "Software"), to deal
        6 // in the Software without restriction, including without limitation the rights
        7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
        8 // copies of the Software, and to permit persons to whom the Software is
        9 // furnished to do so, subject to the following conditions:
        10 //
        11 // The above copyright notice and this permission notice shall be included in
        12 // all copies or substantial portions of the Software.
        13 //
        14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
        15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
        16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
        17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
        18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
        19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
        20 // THE SOFTWARE.
        21 //
        22 
        23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
        24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
        25 
        404 #include <vulkan/vulkan.h>
        405 
        407 
        411 VK_DEFINE_HANDLE(VmaAllocator)
        412 
        413 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
        415  VmaAllocator allocator,
        416  uint32_t memoryType,
        417  VkDeviceMemory memory,
        418  VkDeviceSize size);
        420 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
        421  VmaAllocator allocator,
        422  uint32_t memoryType,
        423  VkDeviceMemory memory,
        424  VkDeviceSize size);
        425 
        431 typedef struct VmaDeviceMemoryCallbacks {
        433  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
        435  PFN_vmaFreeDeviceMemoryFunction pfnFree;
        436 } VmaDeviceMemoryCallbacks;
        437 
        439 typedef enum VmaAllocatorFlagBits {
        444  VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
        445 
        446  VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
        447 } VmaAllocatorFlagBits;
        448 typedef VkFlags VmaAllocatorFlags;
        449 
        451 typedef struct VmaAllocatorCreateInfo
        452 {
        454  VmaAllocatorFlags flags;
        456 
        457  VkPhysicalDevice physicalDevice;
        459 
        460  VkDevice device;
        462 
        463  VkDeviceSize preferredLargeHeapBlockSize;
        465 
        466  VkDeviceSize preferredSmallHeapBlockSize;
        468 
        469  const VkAllocationCallbacks* pAllocationCallbacks;
        471 
        472  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
        486  uint32_t frameInUseCount;
        487 } VmaAllocatorCreateInfo;
        488 
        490 VkResult vmaCreateAllocator(
        491  const VmaAllocatorCreateInfo* pCreateInfo,
        492  VmaAllocator* pAllocator);
        493 
        495 void vmaDestroyAllocator(
        496  VmaAllocator allocator);
        497 
        502 void vmaGetPhysicalDeviceProperties(
        503  VmaAllocator allocator,
        504  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
        505 
        510 void vmaGetMemoryProperties(
        511  VmaAllocator allocator,
        512  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
        513 
        520 void vmaGetMemoryTypeProperties(
        521  VmaAllocator allocator,
        522  uint32_t memoryTypeIndex,
        523  VkMemoryPropertyFlags* pFlags);
        524 
        533 void vmaSetCurrentFrameIndex(
        534  VmaAllocator allocator,
        535  uint32_t frameIndex);
        536 
        537 typedef struct VmaStatInfo
        538 {
        540  uint32_t BlockCount;
        542  uint32_t AllocationCount;
        544  uint32_t UnusedRangeCount;
        546  VkDeviceSize UsedBytes;
        548  VkDeviceSize UnusedBytes;
        549  VkDeviceSize AllocationSizeMin, AllocationSizeAvg, AllocationSizeMax;
        550  VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax;
        551 } VmaStatInfo;
        552 
        554 struct VmaStats
        555 {
        556  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
        557  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
        558  VmaStatInfo total;
        559 };
        560 
        562 void vmaCalculateStats(
        563  VmaAllocator allocator,
        564  VmaStats* pStats);
        565 
        566 #define VMA_STATS_STRING_ENABLED 1
        567 
        568 #if VMA_STATS_STRING_ENABLED
        569 
        571 
        573 void vmaBuildStatsString(
        574  VmaAllocator allocator,
        575  char** ppStatsString,
        576  VkBool32 detailedMap);
        577 
        578 void vmaFreeStatsString(
        579  VmaAllocator allocator,
        580  char* pStatsString);
        581 
        582 #endif // #if VMA_STATS_STRING_ENABLED
        583 
        586 
        591 VK_DEFINE_HANDLE(VmaPool)
        592 
        593 typedef enum VmaMemoryUsage
        594 {
        596  VMA_MEMORY_USAGE_UNKNOWN = 0,
        598  VMA_MEMORY_USAGE_GPU_ONLY = 1,
        600 
        601  VMA_MEMORY_USAGE_CPU_ONLY = 2,
        603 
        604  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
        606 
        607  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
        608  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
        609 } VmaMemoryUsage;
        610 
        612 typedef enum VmaAllocationCreateFlagBits {
        624  VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001,
        625 
        635  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
        645  VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004,
        656  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
        663  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
        664 
        665  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
        666 } VmaAllocationCreateFlagBits;
        667 typedef VkFlags VmaAllocationCreateFlags;
        668 
        669 typedef struct VmaAllocationCreateInfo
        670 {
        672  VmaAllocationCreateFlags flags;
        678  VmaMemoryUsage usage;
        683  VkMemoryPropertyFlags requiredFlags;
        689  VkMemoryPropertyFlags preferredFlags;
        691  void* pUserData;
        696  VmaPool pool;
        697 } VmaAllocationCreateInfo;
        698 
        713 VkResult vmaFindMemoryTypeIndex(
        714  VmaAllocator allocator,
        715  uint32_t memoryTypeBits,
        716  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        717  uint32_t* pMemoryTypeIndex);
        718 
        721 
        726 typedef enum VmaPoolCreateFlagBits {
        736  VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001,
        754  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
        755 
        756  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
        757 } VmaPoolCreateFlagBits;
        758 typedef VkFlags VmaPoolCreateFlags;
        759 
        762 typedef struct VmaPoolCreateInfo {
        765  uint32_t memoryTypeIndex;
        768  VmaPoolCreateFlags flags;
        773  VkDeviceSize blockSize;
        778  size_t minBlockCount;
        786  size_t maxBlockCount;
        800  uint32_t frameInUseCount;
        801 } VmaPoolCreateInfo;
        802 
        805 typedef struct VmaPoolStats {
        808  VkDeviceSize size;
        811  VkDeviceSize unusedSize;
        814  size_t allocationCount;
        817  size_t unusedRangeCount;
        818 } VmaPoolStats;
        819 
        826 VkResult vmaCreatePool(
        827  VmaAllocator allocator,
        828  const VmaPoolCreateInfo* pCreateInfo,
        829  VmaPool* pPool);
        830 
        833 void vmaDestroyPool(
        834  VmaAllocator allocator,
        835  VmaPool pool);
        836 
        843 void vmaGetPoolStats(
        844  VmaAllocator allocator,
        845  VmaPool pool,
        846  VmaPoolStats* pPoolStats);
        847 
        854 void vmaMakePoolAllocationsLost(
        855  VmaAllocator allocator,
        856  VmaPool pool,
        857  size_t* pLostAllocationCount);
        858 
        859 VK_DEFINE_HANDLE(VmaAllocation)
        860 
        861 
        863 typedef struct VmaAllocationInfo {
        868  uint32_t memoryType;
        877  VkDeviceMemory deviceMemory;
        882  VkDeviceSize offset;
        887  VkDeviceSize size;
        893  void* pMappedData;
        898  void* pUserData;
        899 } VmaAllocationInfo;
        900 
        911 VkResult vmaAllocateMemory(
        912  VmaAllocator allocator,
        913  const VkMemoryRequirements* pVkMemoryRequirements,
        914  const VmaAllocationCreateInfo* pCreateInfo,
        915  VmaAllocation* pAllocation,
        916  VmaAllocationInfo* pAllocationInfo);
        917 
        924 VkResult vmaAllocateMemoryForBuffer(
        925  VmaAllocator allocator,
        926  VkBuffer buffer,
        927  const VmaAllocationCreateInfo* pCreateInfo,
        928  VmaAllocation* pAllocation,
        929  VmaAllocationInfo* pAllocationInfo);
        930 
        932 VkResult vmaAllocateMemoryForImage(
        933  VmaAllocator allocator,
        934  VkImage image,
        935  const VmaAllocationCreateInfo* pCreateInfo,
        936  VmaAllocation* pAllocation,
        937  VmaAllocationInfo* pAllocationInfo);
        938 
        940 void vmaFreeMemory(
        941  VmaAllocator allocator,
        942  VmaAllocation allocation);
        943 
        945 void vmaGetAllocationInfo(
        946  VmaAllocator allocator,
        947  VmaAllocation allocation,
        948  VmaAllocationInfo* pAllocationInfo);
        949 
        951 void vmaSetAllocationUserData(
        952  VmaAllocator allocator,
        953  VmaAllocation allocation,
        954  void* pUserData);
        955 
        966 void vmaCreateLostAllocation(
        967  VmaAllocator allocator,
        968  VmaAllocation* pAllocation);
        969 
        978 VkResult vmaMapMemory(
        979  VmaAllocator allocator,
        980  VmaAllocation allocation,
        981  void** ppData);
        982 
        983 void vmaUnmapMemory(
        984  VmaAllocator allocator,
        985  VmaAllocation allocation);
        986 
        1005 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator);
        1006 
        1014 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator);
        1015 
        1017 typedef struct VmaDefragmentationInfo {
        1022  VkDeviceSize maxBytesToMove;
        1027  uint32_t maxAllocationsToMove;
        1028 } VmaDefragmentationInfo;
        1029 
        1031 typedef struct VmaDefragmentationStats {
        1033  VkDeviceSize bytesMoved;
        1035  VkDeviceSize bytesFreed;
        1037  uint32_t allocationsMoved;
        1039  uint32_t deviceMemoryBlocksFreed;
        1040 } VmaDefragmentationStats;
        1041 
        1112 VkResult vmaDefragment(
        1113  VmaAllocator allocator,
        1114  VmaAllocation* pAllocations,
        1115  size_t allocationCount,
        1116  VkBool32* pAllocationsChanged,
        1117  const VmaDefragmentationInfo *pDefragmentationInfo,
        1118  VmaDefragmentationStats* pDefragmentationStats);
        1119 
        1122 
        1145 VkResult vmaCreateBuffer(
        1146  VmaAllocator allocator,
        1147  const VkBufferCreateInfo* pBufferCreateInfo,
        1148  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        1149  VkBuffer* pBuffer,
        1150  VmaAllocation* pAllocation,
        1151  VmaAllocationInfo* pAllocationInfo);
        1152 
        1153 void vmaDestroyBuffer(
        1154  VmaAllocator allocator,
        1155  VkBuffer buffer,
        1156  VmaAllocation allocation);
        1157 
        1159 VkResult vmaCreateImage(
        1160  VmaAllocator allocator,
        1161  const VkImageCreateInfo* pImageCreateInfo,
        1162  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        1163  VkImage* pImage,
        1164  VmaAllocation* pAllocation,
        1165  VmaAllocationInfo* pAllocationInfo);
        1166 
        1167 void vmaDestroyImage(
        1168  VmaAllocator allocator,
        1169  VkImage image,
        1170  VmaAllocation allocation);
        1171 
        1174 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
        1175 
        1176 // For Visual Studio IntelliSense.
        1177 #ifdef __INTELLISENSE__
        1178 #define VMA_IMPLEMENTATION
        1179 #endif
        1180 
        1181 #ifdef VMA_IMPLEMENTATION
        1182 #undef VMA_IMPLEMENTATION
        1183 
        1184 #include <cstdint>
        1185 #include <cstdlib>
        1186 #include <cstring>
        1187 
        1188 /*******************************************************************************
        1189 CONFIGURATION SECTION
        1190 
        1191 Define some of these macros before each #include of this header or change them
        1192 here if you need other then default behavior depending on your environment.
        1193 */
        1194 
        1195 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
        1196 //#define VMA_USE_STL_CONTAINERS 1
        1197 
        1198 /* Set this macro to 1 to make the library including and using STL containers:
        1199 std::pair, std::vector, std::list, std::unordered_map.
        1200 
        1201 Set it to 0 or undefined to make the library using its own implementation of
        1202 the containers.
        1203 */
        1204 #if VMA_USE_STL_CONTAINERS
        1205  #define VMA_USE_STL_VECTOR 1
        1206  #define VMA_USE_STL_UNORDERED_MAP 1
        1207  #define VMA_USE_STL_LIST 1
        1208 #endif
        1209 
        1210 #if VMA_USE_STL_VECTOR
        1211  #include <vector>
        1212 #endif
        1213 
        1214 #if VMA_USE_STL_UNORDERED_MAP
        1215  #include <unordered_map>
        1216 #endif
        1217 
        1218 #if VMA_USE_STL_LIST
        1219  #include <list>
        1220 #endif
        1221 
        1222 /*
        1223 Following headers are used in this CONFIGURATION section only, so feel free to
        1224 remove them if not needed.
        1225 */
        1226 #include <cassert> // for assert
        1227 #include <algorithm> // for min, max
        1228 #include <mutex> // for std::mutex
        1229 #include <atomic> // for std::atomic_int64_t
        1230 
        1231 #if !defined(_WIN32)
        1232  #include <malloc.h> // for aligned_alloc()
        1233 #endif
        1234 
        1235 // Normal assert to check for programmer's errors, especially in Debug configuration.
        1236 #ifndef VMA_ASSERT
        1237  #ifdef _DEBUG
        1238  #define VMA_ASSERT(expr) assert(expr)
        1239  #else
        1240  #define VMA_ASSERT(expr)
        1241  #endif
        1242 #endif
        1243 
        1244 // Assert that will be called very often, like inside data structures e.g. operator[].
        1245 // Making it non-empty can make program slow.
        1246 #ifndef VMA_HEAVY_ASSERT
        1247  #ifdef _DEBUG
        1248  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
        1249  #else
        1250  #define VMA_HEAVY_ASSERT(expr)
        1251  #endif
        1252 #endif
        1253 
        1254 #ifndef VMA_NULL
        1255  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
        1256  #define VMA_NULL nullptr
        1257 #endif
        1258 
        1259 #ifndef VMA_ALIGN_OF
        1260  #define VMA_ALIGN_OF(type) (__alignof(type))
        1261 #endif
        1262 
        1263 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
        1264  #if defined(_WIN32)
        1265  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
        1266  #else
        1267  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
        1268  #endif
        1269 #endif
        1270 
        1271 #ifndef VMA_SYSTEM_FREE
        1272  #if defined(_WIN32)
        1273  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
        1274  #else
        1275  #define VMA_SYSTEM_FREE(ptr) free(ptr)
        1276  #endif
        1277 #endif
        1278 
        1279 #ifndef VMA_MIN
        1280  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
        1281 #endif
        1282 
        1283 #ifndef VMA_MAX
        1284  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
        1285 #endif
        1286 
        1287 #ifndef VMA_SWAP
        1288  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
        1289 #endif
        1290 
        1291 #ifndef VMA_SORT
        1292  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
        1293 #endif
        1294 
        1295 #ifndef VMA_DEBUG_LOG
        1296  #define VMA_DEBUG_LOG(format, ...)
        1297  /*
        1298  #define VMA_DEBUG_LOG(format, ...) do { \
        1299  printf(format, __VA_ARGS__); \
        1300  printf("\n"); \
        1301  } while(false)
        1302  */
        1303 #endif
        1304 
        1305 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
        1306 #if VMA_STATS_STRING_ENABLED
        1307  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
        1308  {
        1309  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
        1310  }
        1311  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
        1312  {
        1313  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
        1314  }
        1315  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
        1316  {
        1317  snprintf(outStr, strLen, "%p", ptr);
        1318  }
        1319 #endif
        1320 
        1321 #ifndef VMA_MUTEX
        1322  class VmaMutex
        1323  {
        1324  public:
        1325  VmaMutex() { }
        1326  ~VmaMutex() { }
        1327  void Lock() { m_Mutex.lock(); }
        1328  void Unlock() { m_Mutex.unlock(); }
        1329  private:
        1330  std::mutex m_Mutex;
        1331  };
        1332  #define VMA_MUTEX VmaMutex
        1333 #endif
        1334 
        1335 /*
        1336 If providing your own implementation, you need to implement a subset of std::atomic:
        1337 
        1338 - Constructor(uint32_t desired)
        1339 - uint32_t load() const
        1340 - void store(uint32_t desired)
        1341 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
        1342 */
        1343 #ifndef VMA_ATOMIC_UINT32
        1344  #define VMA_ATOMIC_UINT32 std::atomic_uint32_t
        1345 #endif
        1346 
        1347 #ifndef VMA_BEST_FIT
        1348 
        1360  #define VMA_BEST_FIT (1)
        1361 #endif
        1362 
        1363 #ifndef VMA_DEBUG_ALWAYS_OWN_MEMORY
        1364 
        1368  #define VMA_DEBUG_ALWAYS_OWN_MEMORY (0)
        1369 #endif
        1370 
        1371 #ifndef VMA_DEBUG_ALIGNMENT
        1372 
        1376  #define VMA_DEBUG_ALIGNMENT (1)
        1377 #endif
        1378 
        1379 #ifndef VMA_DEBUG_MARGIN
        1380 
        1384  #define VMA_DEBUG_MARGIN (0)
        1385 #endif
        1386 
        1387 #ifndef VMA_DEBUG_GLOBAL_MUTEX
        1388 
        1392  #define VMA_DEBUG_GLOBAL_MUTEX (0)
        1393 #endif
        1394 
        1395 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
        1396 
        1400  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
        1401 #endif
        1402 
        1403 #ifndef VMA_SMALL_HEAP_MAX_SIZE
        1404  #define VMA_SMALL_HEAP_MAX_SIZE (512 * 1024 * 1024)
        1406 #endif
        1407 
        1408 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
        1409  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256 * 1024 * 1024)
        1411 #endif
        1412 
        1413 #ifndef VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE
        1414  #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024)
        1416 #endif
        1417 
        1418 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
        1419 
        1420 /*******************************************************************************
        1421 END OF CONFIGURATION
        1422 */
        1423 
        1424 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
        1425  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
        1426 
        1427 // Returns number of bits set to 1 in (v).
        1428 static inline uint32_t CountBitsSet(uint32_t v)
        1429 {
        1430  uint32_t c = v - ((v >> 1) & 0x55555555);
        1431  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
        1432  c = ((c >> 4) + c) & 0x0F0F0F0F;
        1433  c = ((c >> 8) + c) & 0x00FF00FF;
        1434  c = ((c >> 16) + c) & 0x0000FFFF;
        1435  return c;
        1436 }
        1437 
        1438 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
        1439 // Use types like uint32_t, uint64_t as T.
        1440 template <typename T>
        1441 static inline T VmaAlignUp(T val, T align)
        1442 {
        1443  return (val + align - 1) / align * align;
        1444 }
        1445 
        1446 // Division with mathematical rounding to nearest number.
        1447 template <typename T>
        1448 inline T VmaRoundDiv(T x, T y)
        1449 {
        1450  return (x + (y / (T)2)) / y;
        1451 }
        1452 
        1453 #ifndef VMA_SORT
        1454 
        1455 template<typename Iterator, typename Compare>
        1456 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
        1457 {
        1458  Iterator centerValue = end; --centerValue;
        1459  Iterator insertIndex = beg;
        1460  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
        1461  {
        1462  if(cmp(*memTypeIndex, *centerValue))
        1463  {
        1464  if(insertIndex != memTypeIndex)
        1465  {
        1466  VMA_SWAP(*memTypeIndex, *insertIndex);
        1467  }
        1468  ++insertIndex;
        1469  }
        1470  }
        1471  if(insertIndex != centerValue)
        1472  {
        1473  VMA_SWAP(*insertIndex, *centerValue);
        1474  }
        1475  return insertIndex;
        1476 }
        1477 
        1478 template<typename Iterator, typename Compare>
        1479 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
        1480 {
        1481  if(beg < end)
        1482  {
        1483  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
        1484  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
        1485  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
        1486  }
        1487 }
        1488 
        1489 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
        1490 
        1491 #endif // #ifndef VMA_SORT
        1492 
        1493 /*
        1494 Returns true if two memory blocks occupy overlapping pages.
        1495 ResourceA must be in less memory offset than ResourceB.
        1496 
        1497 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
        1498 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
        1499 */
        1500 static inline bool VmaBlocksOnSamePage(
        1501  VkDeviceSize resourceAOffset,
        1502  VkDeviceSize resourceASize,
        1503  VkDeviceSize resourceBOffset,
        1504  VkDeviceSize pageSize)
        1505 {
        1506  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
        1507  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
        1508  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
        1509  VkDeviceSize resourceBStart = resourceBOffset;
        1510  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
        1511  return resourceAEndPage == resourceBStartPage;
        1512 }
        1513 
        1514 enum VmaSuballocationType
        1515 {
        1516  VMA_SUBALLOCATION_TYPE_FREE = 0,
        1517  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
        1518  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
        1519  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
        1520  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
        1521  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
        1522  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
        1523 };
        1524 
        1525 /*
        1526 Returns true if given suballocation types could conflict and must respect
        1527 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
        1528 or linear image and another one is optimal image. If type is unknown, behave
        1529 conservatively.
        1530 */
        1531 static inline bool VmaIsBufferImageGranularityConflict(
        1532  VmaSuballocationType suballocType1,
        1533  VmaSuballocationType suballocType2)
        1534 {
        1535  if(suballocType1 > suballocType2)
        1536  {
        1537  VMA_SWAP(suballocType1, suballocType2);
        1538  }
        1539 
        1540  switch(suballocType1)
        1541  {
        1542  case VMA_SUBALLOCATION_TYPE_FREE:
        1543  return false;
        1544  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
        1545  return true;
        1546  case VMA_SUBALLOCATION_TYPE_BUFFER:
        1547  return
        1548  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
        1549  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1550  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
        1551  return
        1552  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
        1553  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
        1554  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1555  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
        1556  return
        1557  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
        1558  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
        1559  return false;
        1560  default:
        1561  VMA_ASSERT(0);
        1562  return true;
        1563  }
        1564 }
        1565 
        1566 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
        1567 struct VmaMutexLock
        1568 {
        1569 public:
        1570  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
        1571  m_pMutex(useMutex ? &mutex : VMA_NULL)
        1572  {
        1573  if(m_pMutex)
        1574  {
        1575  m_pMutex->Lock();
        1576  }
        1577  }
        1578 
        1579  ~VmaMutexLock()
        1580  {
        1581  if(m_pMutex)
        1582  {
        1583  m_pMutex->Unlock();
        1584  }
        1585  }
        1586 
        1587 private:
        1588  VMA_MUTEX* m_pMutex;
        1589 };
        1590 
        1591 #if VMA_DEBUG_GLOBAL_MUTEX
        1592  static VMA_MUTEX gDebugGlobalMutex;
        1593  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
        1594 #else
        1595  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
        1596 #endif
        1597 
        1598 // Minimum size of a free suballocation to register it in the free suballocation collection.
        1599 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
        1600 
        1601 /*
        1602 Performs binary search and returns iterator to first element that is greater or
        1603 equal to (key), according to comparison (cmp).
        1604 
        1605 Cmp should return true if first argument is less than second argument.
        1606 
        1607 Returned value is the found element, if present in the collection or place where
        1608 new element with value (key) should be inserted.
        1609 */
        1610 template <typename IterT, typename KeyT, typename CmpT>
        1611 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
        1612 {
        1613  size_t down = 0, up = (end - beg);
        1614  while(down < up)
        1615  {
        1616  const size_t mid = (down + up) / 2;
        1617  if(cmp(*(beg+mid), key))
        1618  {
        1619  down = mid + 1;
        1620  }
        1621  else
        1622  {
        1623  up = mid;
        1624  }
        1625  }
        1626  return beg + down;
        1627 }
        1628 
        1630 // Memory allocation
        1631 
        1632 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
        1633 {
        1634  if((pAllocationCallbacks != VMA_NULL) &&
        1635  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
        1636  {
        1637  return (*pAllocationCallbacks->pfnAllocation)(
        1638  pAllocationCallbacks->pUserData,
        1639  size,
        1640  alignment,
        1641  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
        1642  }
        1643  else
        1644  {
        1645  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
        1646  }
        1647 }
        1648 
        1649 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
        1650 {
        1651  if((pAllocationCallbacks != VMA_NULL) &&
        1652  (pAllocationCallbacks->pfnFree != VMA_NULL))
        1653  {
        1654  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
        1655  }
        1656  else
        1657  {
        1658  VMA_SYSTEM_FREE(ptr);
        1659  }
        1660 }
        1661 
        1662 template<typename T>
        1663 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
        1664 {
        1665  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
        1666 }
        1667 
        1668 template<typename T>
        1669 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
        1670 {
        1671  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
        1672 }
        1673 
        1674 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
        1675 
        1676 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
        1677 
        1678 template<typename T>
        1679 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
        1680 {
        1681  ptr->~T();
        1682  VmaFree(pAllocationCallbacks, ptr);
        1683 }
        1684 
        1685 template<typename T>
        1686 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
        1687 {
        1688  if(ptr != VMA_NULL)
        1689  {
        1690  for(size_t i = count; i--; )
        1691  {
        1692  ptr[i].~T();
        1693  }
        1694  VmaFree(pAllocationCallbacks, ptr);
        1695  }
        1696 }
        1697 
        1698 // STL-compatible allocator.
        1699 template<typename T>
        1700 class VmaStlAllocator
        1701 {
        1702 public:
        1703  const VkAllocationCallbacks* const m_pCallbacks;
        1704  typedef T value_type;
        1705 
        1706  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
        1707  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
        1708 
        1709  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
        1710  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
        1711 
        1712  template<typename U>
        1713  bool operator==(const VmaStlAllocator<U>& rhs) const
        1714  {
        1715  return m_pCallbacks == rhs.m_pCallbacks;
        1716  }
        1717  template<typename U>
        1718  bool operator!=(const VmaStlAllocator<U>& rhs) const
        1719  {
        1720  return m_pCallbacks != rhs.m_pCallbacks;
        1721  }
        1722 
        1723  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
        1724 };
        1725 
        1726 #if VMA_USE_STL_VECTOR
        1727 
        1728 #define VmaVector std::vector
        1729 
        1730 template<typename T, typename allocatorT>
        1731 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
        1732 {
        1733  vec.insert(vec.begin() + index, item);
        1734 }
        1735 
        1736 template<typename T, typename allocatorT>
        1737 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
        1738 {
        1739  vec.erase(vec.begin() + index);
        1740 }
        1741 
        1742 #else // #if VMA_USE_STL_VECTOR
        1743 
        1744 /* Class with interface compatible with subset of std::vector.
        1745 T must be POD because constructors and destructors are not called and memcpy is
        1746 used for these objects. */
        1747 template<typename T, typename AllocatorT>
        1748 class VmaVector
        1749 {
        1750 public:
        1751  typedef T value_type;
        1752 
        1753  VmaVector(const AllocatorT& allocator) :
        1754  m_Allocator(allocator),
        1755  m_pArray(VMA_NULL),
        1756  m_Count(0),
        1757  m_Capacity(0)
        1758  {
        1759  }
        1760 
        1761  VmaVector(size_t count, const AllocatorT& allocator) :
        1762  m_Allocator(allocator),
        1763  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
        1764  m_Count(count),
        1765  m_Capacity(count)
        1766  {
        1767  }
        1768 
        1769  VmaVector(const VmaVector<T, AllocatorT>& src) :
        1770  m_Allocator(src.m_Allocator),
        1771  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
        1772  m_Count(src.m_Count),
        1773  m_Capacity(src.m_Count)
        1774  {
        1775  if(m_Count != 0)
        1776  {
        1777  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
        1778  }
        1779  }
        1780 
        1781  ~VmaVector()
        1782  {
        1783  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1784  }
        1785 
        1786  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
        1787  {
        1788  if(&rhs != this)
        1789  {
        1790  resize(rhs.m_Count);
        1791  if(m_Count != 0)
        1792  {
        1793  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
        1794  }
        1795  }
        1796  return *this;
        1797  }
        1798 
        1799  bool empty() const { return m_Count == 0; }
        1800  size_t size() const { return m_Count; }
        1801  T* data() { return m_pArray; }
        1802  const T* data() const { return m_pArray; }
        1803 
        1804  T& operator[](size_t index)
        1805  {
        1806  VMA_HEAVY_ASSERT(index < m_Count);
        1807  return m_pArray[index];
        1808  }
        1809  const T& operator[](size_t index) const
        1810  {
        1811  VMA_HEAVY_ASSERT(index < m_Count);
        1812  return m_pArray[index];
        1813  }
        1814 
        1815  T& front()
        1816  {
        1817  VMA_HEAVY_ASSERT(m_Count > 0);
        1818  return m_pArray[0];
        1819  }
        1820  const T& front() const
        1821  {
        1822  VMA_HEAVY_ASSERT(m_Count > 0);
        1823  return m_pArray[0];
        1824  }
        1825  T& back()
        1826  {
        1827  VMA_HEAVY_ASSERT(m_Count > 0);
        1828  return m_pArray[m_Count - 1];
        1829  }
        1830  const T& back() const
        1831  {
        1832  VMA_HEAVY_ASSERT(m_Count > 0);
        1833  return m_pArray[m_Count - 1];
        1834  }
        1835 
        1836  void reserve(size_t newCapacity, bool freeMemory = false)
        1837  {
        1838  newCapacity = VMA_MAX(newCapacity, m_Count);
        1839 
        1840  if((newCapacity < m_Capacity) && !freeMemory)
        1841  {
        1842  newCapacity = m_Capacity;
        1843  }
        1844 
        1845  if(newCapacity != m_Capacity)
        1846  {
        1847  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
        1848  if(m_Count != 0)
        1849  {
        1850  memcpy(newArray, m_pArray, m_Count * sizeof(T));
        1851  }
        1852  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1853  m_Capacity = newCapacity;
        1854  m_pArray = newArray;
        1855  }
        1856  }
        1857 
        1858  void resize(size_t newCount, bool freeMemory = false)
        1859  {
        1860  size_t newCapacity = m_Capacity;
        1861  if(newCount > m_Capacity)
        1862  {
        1863  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
        1864  }
        1865  else if(freeMemory)
        1866  {
        1867  newCapacity = newCount;
        1868  }
        1869 
        1870  if(newCapacity != m_Capacity)
        1871  {
        1872  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
        1873  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
        1874  if(elementsToCopy != 0)
        1875  {
        1876  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
        1877  }
        1878  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
        1879  m_Capacity = newCapacity;
        1880  m_pArray = newArray;
        1881  }
        1882 
        1883  m_Count = newCount;
        1884  }
        1885 
        1886  void clear(bool freeMemory = false)
        1887  {
        1888  resize(0, freeMemory);
        1889  }
        1890 
        1891  void insert(size_t index, const T& src)
        1892  {
        1893  VMA_HEAVY_ASSERT(index <= m_Count);
        1894  const size_t oldCount = size();
        1895  resize(oldCount + 1);
        1896  if(index < oldCount)
        1897  {
        1898  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
        1899  }
        1900  m_pArray[index] = src;
        1901  }
        1902 
        1903  void remove(size_t index)
        1904  {
        1905  VMA_HEAVY_ASSERT(index < m_Count);
        1906  const size_t oldCount = size();
        1907  if(index < oldCount - 1)
        1908  {
        1909  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
        1910  }
        1911  resize(oldCount - 1);
        1912  }
        1913 
        1914  void push_back(const T& src)
        1915  {
        1916  const size_t newIndex = size();
        1917  resize(newIndex + 1);
        1918  m_pArray[newIndex] = src;
        1919  }
        1920 
        1921  void pop_back()
        1922  {
        1923  VMA_HEAVY_ASSERT(m_Count > 0);
        1924  resize(size() - 1);
        1925  }
        1926 
        1927  void push_front(const T& src)
        1928  {
        1929  insert(0, src);
        1930  }
        1931 
        1932  void pop_front()
        1933  {
        1934  VMA_HEAVY_ASSERT(m_Count > 0);
        1935  remove(0);
        1936  }
        1937 
        1938  typedef T* iterator;
        1939 
        1940  iterator begin() { return m_pArray; }
        1941  iterator end() { return m_pArray + m_Count; }
        1942 
        1943 private:
        1944  AllocatorT m_Allocator;
        1945  T* m_pArray;
        1946  size_t m_Count;
        1947  size_t m_Capacity;
        1948 };
        1949 
        1950 template<typename T, typename allocatorT>
        1951 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
        1952 {
        1953  vec.insert(index, item);
        1954 }
        1955 
        1956 template<typename T, typename allocatorT>
        1957 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
        1958 {
        1959  vec.remove(index);
        1960 }
        1961 
        1962 #endif // #if VMA_USE_STL_VECTOR
        1963 
        1964 template<typename CmpLess, typename VectorT>
        1965 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
        1966 {
        1967  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
        1968  vector.data(),
        1969  vector.data() + vector.size(),
        1970  value,
        1971  CmpLess()) - vector.data();
        1972  VmaVectorInsert(vector, indexToInsert, value);
        1973  return indexToInsert;
        1974 }
        1975 
        1976 template<typename CmpLess, typename VectorT>
        1977 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
        1978 {
        1979  CmpLess comparator;
        1980  VectorT::iterator it = VmaBinaryFindFirstNotLess(
        1981  vector.data(),
        1982  vector.data() + vector.size(),
        1983  value,
        1984  comparator);
        1985  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
        1986  {
        1987  size_t indexToRemove = it - vector.begin();
        1988  VmaVectorRemove(vector, indexToRemove);
        1989  return true;
        1990  }
        1991  return false;
        1992 }
        1993 
        1994 template<typename CmpLess, typename VectorT>
        1995 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
        1996 {
        1997  CmpLess comparator;
        1998  VectorT::iterator it = VmaBinaryFindFirstNotLess(
        1999  vector.data(),
        2000  vector.data() + vector.size(),
        2001  value,
        2002  comparator);
        2003  if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
        2004  {
        2005  return it - vector.begin();
        2006  }
        2007  else
        2008  {
        2009  return vector.size();
        2010  }
        2011 }
        2012 
        2014 // class VmaPoolAllocator
        2015 
        2016 /*
        2017 Allocator for objects of type T using a list of arrays (pools) to speed up
        2018 allocation. Number of elements that can be allocated is not bounded because
        2019 allocator can create multiple blocks.
        2020 */
        2021 template<typename T>
        2022 class VmaPoolAllocator
        2023 {
        2024 public:
        2025  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
        2026  ~VmaPoolAllocator();
        2027  void Clear();
        2028  T* Alloc();
        2029  void Free(T* ptr);
        2030 
        2031 private:
        2032  union Item
        2033  {
        2034  uint32_t NextFreeIndex;
        2035  T Value;
        2036  };
        2037 
        2038  struct ItemBlock
        2039  {
        2040  Item* pItems;
        2041  uint32_t FirstFreeIndex;
        2042  };
        2043 
        2044  const VkAllocationCallbacks* m_pAllocationCallbacks;
        2045  size_t m_ItemsPerBlock;
        2046  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
        2047 
        2048  ItemBlock& CreateNewBlock();
        2049 };
        2050 
        2051 template<typename T>
        2052 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
        2053  m_pAllocationCallbacks(pAllocationCallbacks),
        2054  m_ItemsPerBlock(itemsPerBlock),
        2055  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
        2056 {
        2057  VMA_ASSERT(itemsPerBlock > 0);
        2058 }
        2059 
        2060 template<typename T>
        2061 VmaPoolAllocator<T>::~VmaPoolAllocator()
        2062 {
        2063  Clear();
        2064 }
        2065 
        2066 template<typename T>
        2067 void VmaPoolAllocator<T>::Clear()
        2068 {
        2069  for(size_t i = m_ItemBlocks.size(); i--; )
        2070  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
        2071  m_ItemBlocks.clear();
        2072 }
        2073 
        2074 template<typename T>
        2075 T* VmaPoolAllocator<T>::Alloc()
        2076 {
        2077  for(size_t i = m_ItemBlocks.size(); i--; )
        2078  {
        2079  ItemBlock& block = m_ItemBlocks[i];
        2080  // This block has some free items: Use first one.
        2081  if(block.FirstFreeIndex != UINT32_MAX)
        2082  {
        2083  Item* const pItem = &block.pItems[block.FirstFreeIndex];
        2084  block.FirstFreeIndex = pItem->NextFreeIndex;
        2085  return &pItem->Value;
        2086  }
        2087  }
        2088 
        2089  // No block has free item: Create new one and use it.
        2090  ItemBlock& newBlock = CreateNewBlock();
        2091  Item* const pItem = &newBlock.pItems[0];
        2092  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
        2093  return &pItem->Value;
        2094 }
        2095 
        2096 template<typename T>
        2097 void VmaPoolAllocator<T>::Free(T* ptr)
        2098 {
        2099  // Search all memory blocks to find ptr.
        2100  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
        2101  {
        2102  ItemBlock& block = m_ItemBlocks[i];
        2103 
        2104  // Casting to union.
        2105  Item* pItemPtr;
        2106  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
        2107 
        2108  // Check if pItemPtr is in address range of this block.
        2109  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
        2110  {
        2111  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
        2112  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
        2113  block.FirstFreeIndex = index;
        2114  return;
        2115  }
        2116  }
        2117  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
        2118 }
        2119 
        2120 template<typename T>
        2121 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
        2122 {
        2123  ItemBlock newBlock = {
        2124  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
        2125 
        2126  m_ItemBlocks.push_back(newBlock);
        2127 
        2128  // Setup singly-linked list of all free items in this block.
        2129  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
        2130  newBlock.pItems[i].NextFreeIndex = i + 1;
        2131  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
        2132  return m_ItemBlocks.back();
        2133 }
        2134 
        2136 // class VmaRawList, VmaList
        2137 
        2138 #if VMA_USE_STL_LIST
        2139 
        2140 #define VmaList std::list
        2141 
        2142 #else // #if VMA_USE_STL_LIST
        2143 
        2144 template<typename T>
        2145 struct VmaListItem
        2146 {
        2147  VmaListItem* pPrev;
        2148  VmaListItem* pNext;
        2149  T Value;
        2150 };
        2151 
        2152 // Doubly linked list.
        2153 template<typename T>
        2154 class VmaRawList
        2155 {
        2156 public:
        2157  typedef VmaListItem<T> ItemType;
        2158 
        2159  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
        2160  ~VmaRawList();
        2161  void Clear();
        2162 
        2163  size_t GetCount() const { return m_Count; }
        2164  bool IsEmpty() const { return m_Count == 0; }
        2165 
        2166  ItemType* Front() { return m_pFront; }
        2167  const ItemType* Front() const { return m_pFront; }
        2168  ItemType* Back() { return m_pBack; }
        2169  const ItemType* Back() const { return m_pBack; }
        2170 
        2171  ItemType* PushBack();
        2172  ItemType* PushFront();
        2173  ItemType* PushBack(const T& value);
        2174  ItemType* PushFront(const T& value);
        2175  void PopBack();
        2176  void PopFront();
        2177 
        2178  // Item can be null - it means PushBack.
        2179  ItemType* InsertBefore(ItemType* pItem);
        2180  // Item can be null - it means PushFront.
        2181  ItemType* InsertAfter(ItemType* pItem);
        2182 
        2183  ItemType* InsertBefore(ItemType* pItem, const T& value);
        2184  ItemType* InsertAfter(ItemType* pItem, const T& value);
        2185 
        2186  void Remove(ItemType* pItem);
        2187 
        2188 private:
        2189  const VkAllocationCallbacks* const m_pAllocationCallbacks;
        2190  VmaPoolAllocator<ItemType> m_ItemAllocator;
        2191  ItemType* m_pFront;
        2192  ItemType* m_pBack;
        2193  size_t m_Count;
        2194 
        2195  // Declared not defined, to block copy constructor and assignment operator.
        2196  VmaRawList(const VmaRawList<T>& src);
        2197  VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
        2198 };
        2199 
        2200 template<typename T>
        2201 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
        2202  m_pAllocationCallbacks(pAllocationCallbacks),
        2203  m_ItemAllocator(pAllocationCallbacks, 128),
        2204  m_pFront(VMA_NULL),
        2205  m_pBack(VMA_NULL),
        2206  m_Count(0)
        2207 {
        2208 }
        2209 
        2210 template<typename T>
        2211 VmaRawList<T>::~VmaRawList()
        2212 {
        2213  // Intentionally not calling Clear, because that would be unnecessary
        2214  // computations to return all items to m_ItemAllocator as free.
        2215 }
        2216 
        2217 template<typename T>
        2218 void VmaRawList<T>::Clear()
        2219 {
        2220  if(IsEmpty() == false)
        2221  {
        2222  ItemType* pItem = m_pBack;
        2223  while(pItem != VMA_NULL)
        2224  {
        2225  ItemType* const pPrevItem = pItem->pPrev;
        2226  m_ItemAllocator.Free(pItem);
        2227  pItem = pPrevItem;
        2228  }
        2229  m_pFront = VMA_NULL;
        2230  m_pBack = VMA_NULL;
        2231  m_Count = 0;
        2232  }
        2233 }
        2234 
        2235 template<typename T>
        2236 VmaListItem<T>* VmaRawList<T>::PushBack()
        2237 {
        2238  ItemType* const pNewItem = m_ItemAllocator.Alloc();
        2239  pNewItem->pNext = VMA_NULL;
        2240  if(IsEmpty())
        2241  {
        2242  pNewItem->pPrev = VMA_NULL;
        2243  m_pFront = pNewItem;
        2244  m_pBack = pNewItem;
        2245  m_Count = 1;
        2246  }
        2247  else
        2248  {
        2249  pNewItem->pPrev = m_pBack;
        2250  m_pBack->pNext = pNewItem;
        2251  m_pBack = pNewItem;
        2252  ++m_Count;
        2253  }
        2254  return pNewItem;
        2255 }
        2256 
        2257 template<typename T>
        2258 VmaListItem<T>* VmaRawList<T>::PushFront()
        2259 {
        2260  ItemType* const pNewItem = m_ItemAllocator.Alloc();
        2261  pNewItem->pPrev = VMA_NULL;
        2262  if(IsEmpty())
        2263  {
        2264  pNewItem->pNext = VMA_NULL;
        2265  m_pFront = pNewItem;
        2266  m_pBack = pNewItem;
        2267  m_Count = 1;
        2268  }
        2269  else
        2270  {
        2271  pNewItem->pNext = m_pFront;
        2272  m_pFront->pPrev = pNewItem;
        2273  m_pFront = pNewItem;
        2274  ++m_Count;
        2275  }
        2276  return pNewItem;
        2277 }
        2278 
        2279 template<typename T>
        2280 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
        2281 {
        2282  ItemType* const pNewItem = PushBack();
        2283  pNewItem->Value = value;
        2284  return pNewItem;
        2285 }
        2286 
        2287 template<typename T>
        2288 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
        2289 {
        2290  ItemType* const pNewItem = PushFront();
        2291  pNewItem->Value = value;
        2292  return pNewItem;
        2293 }
        2294 
        2295 template<typename T>
        2296 void VmaRawList<T>::PopBack()
        2297 {
        2298  VMA_HEAVY_ASSERT(m_Count > 0);
        2299  ItemType* const pBackItem = m_pBack;
        2300  ItemType* const pPrevItem = pBackItem->pPrev;
        2301  if(pPrevItem != VMA_NULL)
        2302  {
        2303  pPrevItem->pNext = VMA_NULL;
        2304  }
        2305  m_pBack = pPrevItem;
        2306  m_ItemAllocator.Free(pBackItem);
        2307  --m_Count;
        2308 }
        2309 
        2310 template<typename T>
        2311 void VmaRawList<T>::PopFront()
        2312 {
        2313  VMA_HEAVY_ASSERT(m_Count > 0);
        2314  ItemType* const pFrontItem = m_pFront;
        2315  ItemType* const pNextItem = pFrontItem->pNext;
        2316  if(pNextItem != VMA_NULL)
        2317  {
        2318  pNextItem->pPrev = VMA_NULL;
        2319  }
        2320  m_pFront = pNextItem;
        2321  m_ItemAllocator.Free(pFrontItem);
        2322  --m_Count;
        2323 }
        2324 
        2325 template<typename T>
        2326 void VmaRawList<T>::Remove(ItemType* pItem)
        2327 {
        2328  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
        2329  VMA_HEAVY_ASSERT(m_Count > 0);
        2330 
        2331  if(pItem->pPrev != VMA_NULL)
        2332  {
        2333  pItem->pPrev->pNext = pItem->pNext;
        2334  }
        2335  else
        2336  {
        2337  VMA_HEAVY_ASSERT(m_pFront == pItem);
        2338  m_pFront = pItem->pNext;
        2339  }
        2340 
        2341  if(pItem->pNext != VMA_NULL)
        2342  {
        2343  pItem->pNext->pPrev = pItem->pPrev;
        2344  }
        2345  else
        2346  {
        2347  VMA_HEAVY_ASSERT(m_pBack == pItem);
        2348  m_pBack = pItem->pPrev;
        2349  }
        2350 
        2351  m_ItemAllocator.Free(pItem);
        2352  --m_Count;
        2353 }
        2354 
        2355 template<typename T>
        2356 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
        2357 {
        2358  if(pItem != VMA_NULL)
        2359  {
        2360  ItemType* const prevItem = pItem->pPrev;
        2361  ItemType* const newItem = m_ItemAllocator.Alloc();
        2362  newItem->pPrev = prevItem;
        2363  newItem->pNext = pItem;
        2364  pItem->pPrev = newItem;
        2365  if(prevItem != VMA_NULL)
        2366  {
        2367  prevItem->pNext = newItem;
        2368  }
        2369  else
        2370  {
        2371  VMA_HEAVY_ASSERT(m_pFront == pItem);
        2372  m_pFront = newItem;
        2373  }
        2374  ++m_Count;
        2375  return newItem;
        2376  }
        2377  else
        2378  return PushBack();
        2379 }
        2380 
        2381 template<typename T>
        2382 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
        2383 {
        2384  if(pItem != VMA_NULL)
        2385  {
        2386  ItemType* const nextItem = pItem->pNext;
        2387  ItemType* const newItem = m_ItemAllocator.Alloc();
        2388  newItem->pNext = nextItem;
        2389  newItem->pPrev = pItem;
        2390  pItem->pNext = newItem;
        2391  if(nextItem != VMA_NULL)
        2392  {
        2393  nextItem->pPrev = newItem;
        2394  }
        2395  else
        2396  {
        2397  VMA_HEAVY_ASSERT(m_pBack == pItem);
        2398  m_pBack = newItem;
        2399  }
        2400  ++m_Count;
        2401  return newItem;
        2402  }
        2403  else
        2404  return PushFront();
        2405 }
        2406 
        2407 template<typename T>
        2408 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
        2409 {
        2410  ItemType* const newItem = InsertBefore(pItem);
        2411  newItem->Value = value;
        2412  return newItem;
        2413 }
        2414 
        2415 template<typename T>
        2416 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
        2417 {
        2418  ItemType* const newItem = InsertAfter(pItem);
        2419  newItem->Value = value;
        2420  return newItem;
        2421 }
        2422 
        2423 template<typename T, typename AllocatorT>
        2424 class VmaList
        2425 {
        2426 public:
        2427  class iterator
        2428  {
        2429  public:
        2430  iterator() :
        2431  m_pList(VMA_NULL),
        2432  m_pItem(VMA_NULL)
        2433  {
        2434  }
        2435 
        2436  T& operator*() const
        2437  {
        2438  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2439  return m_pItem->Value;
        2440  }
        2441  T* operator->() const
        2442  {
        2443  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2444  return &m_pItem->Value;
        2445  }
        2446 
        2447  iterator& operator++()
        2448  {
        2449  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2450  m_pItem = m_pItem->pNext;
        2451  return *this;
        2452  }
        2453  iterator& operator--()
        2454  {
        2455  if(m_pItem != VMA_NULL)
        2456  {
        2457  m_pItem = m_pItem->pPrev;
        2458  }
        2459  else
        2460  {
        2461  VMA_HEAVY_ASSERT(!m_pList.IsEmpty());
        2462  m_pItem = m_pList->Back();
        2463  }
        2464  return *this;
        2465  }
        2466 
        2467  iterator operator++(int)
        2468  {
        2469  iterator result = *this;
        2470  ++*this;
        2471  return result;
        2472  }
        2473  iterator operator--(int)
        2474  {
        2475  iterator result = *this;
        2476  --*this;
        2477  return result;
        2478  }
        2479 
        2480  bool operator==(const iterator& rhs) const
        2481  {
        2482  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2483  return m_pItem == rhs.m_pItem;
        2484  }
        2485  bool operator!=(const iterator& rhs) const
        2486  {
        2487  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2488  return m_pItem != rhs.m_pItem;
        2489  }
        2490 
        2491  private:
        2492  VmaRawList<T>* m_pList;
        2493  VmaListItem<T>* m_pItem;
        2494 
        2495  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
        2496  m_pList(pList),
        2497  m_pItem(pItem)
        2498  {
        2499  }
        2500 
        2501  friend class VmaList<T, AllocatorT>;
        2502  };
        2503 
        2504  class const_iterator
        2505  {
        2506  public:
        2507  const_iterator() :
        2508  m_pList(VMA_NULL),
        2509  m_pItem(VMA_NULL)
        2510  {
        2511  }
        2512 
        2513  const_iterator(const iterator& src) :
        2514  m_pList(src.m_pList),
        2515  m_pItem(src.m_pItem)
        2516  {
        2517  }
        2518 
        2519  const T& operator*() const
        2520  {
        2521  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2522  return m_pItem->Value;
        2523  }
        2524  const T* operator->() const
        2525  {
        2526  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2527  return &m_pItem->Value;
        2528  }
        2529 
        2530  const_iterator& operator++()
        2531  {
        2532  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
        2533  m_pItem = m_pItem->pNext;
        2534  return *this;
        2535  }
        2536  const_iterator& operator--()
        2537  {
        2538  if(m_pItem != VMA_NULL)
        2539  {
        2540  m_pItem = m_pItem->pPrev;
        2541  }
        2542  else
        2543  {
        2544  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
        2545  m_pItem = m_pList->Back();
        2546  }
        2547  return *this;
        2548  }
        2549 
        2550  const_iterator operator++(int)
        2551  {
        2552  const_iterator result = *this;
        2553  ++*this;
        2554  return result;
        2555  }
        2556  const_iterator operator--(int)
        2557  {
        2558  const_iterator result = *this;
        2559  --*this;
        2560  return result;
        2561  }
        2562 
        2563  bool operator==(const const_iterator& rhs) const
        2564  {
        2565  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2566  return m_pItem == rhs.m_pItem;
        2567  }
        2568  bool operator!=(const const_iterator& rhs) const
        2569  {
        2570  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
        2571  return m_pItem != rhs.m_pItem;
        2572  }
        2573 
        2574  private:
        2575  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
        2576  m_pList(pList),
        2577  m_pItem(pItem)
        2578  {
        2579  }
        2580 
        2581  const VmaRawList<T>* m_pList;
        2582  const VmaListItem<T>* m_pItem;
        2583 
        2584  friend class VmaList<T, AllocatorT>;
        2585  };
        2586 
        2587  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
        2588 
        2589  bool empty() const { return m_RawList.IsEmpty(); }
        2590  size_t size() const { return m_RawList.GetCount(); }
        2591 
        2592  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
        2593  iterator end() { return iterator(&m_RawList, VMA_NULL); }
        2594 
        2595  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
        2596  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
        2597 
        2598  void clear() { m_RawList.Clear(); }
        2599  void push_back(const T& value) { m_RawList.PushBack(value); }
        2600  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
        2601  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
        2602 
        2603 private:
        2604  VmaRawList<T> m_RawList;
        2605 };
        2606 
        2607 #endif // #if VMA_USE_STL_LIST
        2608 
        2610 // class VmaMap
        2611 
        2612 // Unused in this version.
        2613 #if 0
        2614 
        2615 #if VMA_USE_STL_UNORDERED_MAP
        2616 
        2617 #define VmaPair std::pair
        2618 
        2619 #define VMA_MAP_TYPE(KeyT, ValueT) \
        2620  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
        2621 
        2622 #else // #if VMA_USE_STL_UNORDERED_MAP
        2623 
        2624 template<typename T1, typename T2>
        2625 struct VmaPair
        2626 {
        2627  T1 first;
        2628  T2 second;
        2629 
        2630  VmaPair() : first(), second() { }
        2631  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
        2632 };
        2633 
        2634 /* Class compatible with subset of interface of std::unordered_map.
        2635 KeyT, ValueT must be POD because they will be stored in VmaVector.
        2636 */
        2637 template<typename KeyT, typename ValueT>
        2638 class VmaMap
        2639 {
        2640 public:
        2641  typedef VmaPair<KeyT, ValueT> PairType;
        2642  typedef PairType* iterator;
        2643 
        2644  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
        2645 
        2646  iterator begin() { return m_Vector.begin(); }
        2647  iterator end() { return m_Vector.end(); }
        2648 
        2649  void insert(const PairType& pair);
        2650  iterator find(const KeyT& key);
        2651  void erase(iterator it);
        2652 
        2653 private:
        2654  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
        2655 };
        2656 
        2657 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
        2658 
        2659 template<typename FirstT, typename SecondT>
        2660 struct VmaPairFirstLess
        2661 {
        2662  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
        2663  {
        2664  return lhs.first < rhs.first;
        2665  }
        2666  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
        2667  {
        2668  return lhs.first < rhsFirst;
        2669  }
        2670 };
        2671 
        2672 template<typename KeyT, typename ValueT>
        2673 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
        2674 {
        2675  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
        2676  m_Vector.data(),
        2677  m_Vector.data() + m_Vector.size(),
        2678  pair,
        2679  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
        2680  VmaVectorInsert(m_Vector, indexToInsert, pair);
        2681 }
        2682 
        2683 template<typename KeyT, typename ValueT>
        2684 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
        2685 {
        2686  PairType* it = VmaBinaryFindFirstNotLess(
        2687  m_Vector.data(),
        2688  m_Vector.data() + m_Vector.size(),
        2689  key,
        2690  VmaPairFirstLess<KeyT, ValueT>());
        2691  if((it != m_Vector.end()) && (it->first == key))
        2692  {
        2693  return it;
        2694  }
        2695  else
        2696  {
        2697  return m_Vector.end();
        2698  }
        2699 }
        2700 
        2701 template<typename KeyT, typename ValueT>
        2702 void VmaMap<KeyT, ValueT>::erase(iterator it)
        2703 {
        2704  VmaVectorRemove(m_Vector, it - m_Vector.begin());
        2705 }
        2706 
        2707 #endif // #if VMA_USE_STL_UNORDERED_MAP
        2708 
        2709 #endif // #if 0
        2710 
        2712 
        2713 class VmaDeviceMemoryBlock;
        2714 
        2715 enum VMA_BLOCK_VECTOR_TYPE
        2716 {
        2717  VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
        2718  VMA_BLOCK_VECTOR_TYPE_MAPPED,
        2719  VMA_BLOCK_VECTOR_TYPE_COUNT
        2720 };
        2721 
        2722 static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags)
        2723 {
        2724  return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ?
        2725  VMA_BLOCK_VECTOR_TYPE_MAPPED :
        2726  VMA_BLOCK_VECTOR_TYPE_UNMAPPED;
        2727 }
        2728 
        2729 struct VmaAllocation_T
        2730 {
        2731 public:
        2732  enum ALLOCATION_TYPE
        2733  {
        2734  ALLOCATION_TYPE_NONE,
        2735  ALLOCATION_TYPE_BLOCK,
        2736  ALLOCATION_TYPE_OWN,
        2737  };
        2738 
        2739  VmaAllocation_T(uint32_t currentFrameIndex) :
        2740  m_Alignment(1),
        2741  m_Size(0),
        2742  m_pUserData(VMA_NULL),
        2743  m_Type(ALLOCATION_TYPE_NONE),
        2744  m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN),
        2745  m_LastUseFrameIndex(currentFrameIndex)
        2746  {
        2747  }
        2748 
        2749  void InitBlockAllocation(
        2750  VmaPool hPool,
        2751  VmaDeviceMemoryBlock* block,
        2752  VkDeviceSize offset,
        2753  VkDeviceSize alignment,
        2754  VkDeviceSize size,
        2755  VmaSuballocationType suballocationType,
        2756  void* pUserData,
        2757  bool canBecomeLost)
        2758  {
        2759  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
        2760  VMA_ASSERT(block != VMA_NULL);
        2761  m_Type = ALLOCATION_TYPE_BLOCK;
        2762  m_Alignment = alignment;
        2763  m_Size = size;
        2764  m_pUserData = pUserData;
        2765  m_SuballocationType = suballocationType;
        2766  m_BlockAllocation.m_hPool = hPool;
        2767  m_BlockAllocation.m_Block = block;
        2768  m_BlockAllocation.m_Offset = offset;
        2769  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
        2770  }
        2771 
        2772  void InitLost()
        2773  {
        2774  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
        2775  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
        2776  m_Type = ALLOCATION_TYPE_BLOCK;
        2777  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
        2778  m_BlockAllocation.m_Block = VMA_NULL;
        2779  m_BlockAllocation.m_Offset = 0;
        2780  m_BlockAllocation.m_CanBecomeLost = true;
        2781  }
        2782 
        2783  void ChangeBlockAllocation(
        2784  VmaDeviceMemoryBlock* block,
        2785  VkDeviceSize offset)
        2786  {
        2787  VMA_ASSERT(block != VMA_NULL);
        2788  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
        2789  m_BlockAllocation.m_Block = block;
        2790  m_BlockAllocation.m_Offset = offset;
        2791  }
        2792 
        2793  void InitOwnAllocation(
        2794  uint32_t memoryTypeIndex,
        2795  VkDeviceMemory hMemory,
        2796  VmaSuballocationType suballocationType,
        2797  bool persistentMap,
        2798  void* pMappedData,
        2799  VkDeviceSize size,
        2800  void* pUserData)
        2801  {
        2802  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
        2803  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
        2804  m_Type = ALLOCATION_TYPE_OWN;
        2805  m_Alignment = 0;
        2806  m_Size = size;
        2807  m_pUserData = pUserData;
        2808  m_SuballocationType = suballocationType;
        2809  m_OwnAllocation.m_MemoryTypeIndex = memoryTypeIndex;
        2810  m_OwnAllocation.m_hMemory = hMemory;
        2811  m_OwnAllocation.m_PersistentMap = persistentMap;
        2812  m_OwnAllocation.m_pMappedData = pMappedData;
        2813  }
        2814 
        2815  ALLOCATION_TYPE GetType() const { return m_Type; }
        2816  VkDeviceSize GetAlignment() const { return m_Alignment; }
        2817  VkDeviceSize GetSize() const { return m_Size; }
        2818  void* GetUserData() const { return m_pUserData; }
        2819  void SetUserData(void* pUserData) { m_pUserData = pUserData; }
        2820  VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; }
        2821 
        2822  VmaDeviceMemoryBlock* GetBlock() const
        2823  {
        2824  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
        2825  return m_BlockAllocation.m_Block;
        2826  }
        2827  VkDeviceSize GetOffset() const;
        2828  VkDeviceMemory GetMemory() const;
        2829  uint32_t GetMemoryTypeIndex() const;
        2830  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const;
        2831  void* GetMappedData() const;
        2832  bool CanBecomeLost() const;
        2833  VmaPool GetPool() const;
        2834 
        2835  VkResult OwnAllocMapPersistentlyMappedMemory(VkDevice hDevice)
        2836  {
        2837  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
        2838  if(m_OwnAllocation.m_PersistentMap)
        2839  {
        2840  return vkMapMemory(hDevice, m_OwnAllocation.m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_OwnAllocation.m_pMappedData);
        2841  }
        2842  return VK_SUCCESS;
        2843  }
        2844  void OwnAllocUnmapPersistentlyMappedMemory(VkDevice hDevice)
        2845  {
        2846  VMA_ASSERT(m_Type == ALLOCATION_TYPE_OWN);
        2847  if(m_OwnAllocation.m_pMappedData)
        2848  {
        2849  VMA_ASSERT(m_OwnAllocation.m_PersistentMap);
        2850  vkUnmapMemory(hDevice, m_OwnAllocation.m_hMemory);
        2851  m_OwnAllocation.m_pMappedData = VMA_NULL;
        2852  }
        2853  }
        2854 
        2855  uint32_t GetLastUseFrameIndex() const
        2856  {
        2857  return m_LastUseFrameIndex.load();
        2858  }
        2859  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
        2860  {
        2861  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
        2862  }
        2863  /*
        2864  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
        2865  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
        2866  - Else, returns false.
        2867 
        2868  If hAllocation is already lost, assert - you should not call it then.
        2869  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
        2870  */
        2871  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
        2872 
        2873 private:
        2874  VkDeviceSize m_Alignment;
        2875  VkDeviceSize m_Size;
        2876  void* m_pUserData;
        2877  ALLOCATION_TYPE m_Type;
        2878  VmaSuballocationType m_SuballocationType;
        2879  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
        2880 
        2881  // Allocation out of VmaDeviceMemoryBlock.
        2882  struct BlockAllocation
        2883  {
        2884  VmaPool m_hPool; // Null if belongs to general memory.
        2885  VmaDeviceMemoryBlock* m_Block;
        2886  VkDeviceSize m_Offset;
        2887  bool m_CanBecomeLost;
        2888  };
        2889 
        2890  // Allocation for an object that has its own private VkDeviceMemory.
        2891  struct OwnAllocation
        2892  {
        2893  uint32_t m_MemoryTypeIndex;
        2894  VkDeviceMemory m_hMemory;
        2895  bool m_PersistentMap;
        2896  void* m_pMappedData;
        2897  };
        2898 
        2899  union
        2900  {
        2901  // Allocation out of VmaDeviceMemoryBlock.
        2902  BlockAllocation m_BlockAllocation;
        2903  // Allocation for an object that has its own private VkDeviceMemory.
        2904  OwnAllocation m_OwnAllocation;
        2905  };
        2906 };
        2907 
        2908 /*
        2909 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
        2910 allocated memory block or free.
        2911 */
        2912 struct VmaSuballocation
        2913 {
        2914  VkDeviceSize offset;
        2915  VkDeviceSize size;
        2916  VmaAllocation hAllocation;
        2917  VmaSuballocationType type;
        2918 };
        2919 
        2920 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
        2921 
        2922 // Cost of one additional allocation lost, as equivalent in bytes.
        2923 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
        2924 
        2925 /*
        2926 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
        2927 
        2928 If canMakeOtherLost was false:
        2929 - item points to a FREE suballocation.
        2930 - itemsToMakeLostCount is 0.
        2931 
        2932 If canMakeOtherLost was true:
        2933 - item points to first of sequence of suballocations, which are either FREE,
        2934  or point to VmaAllocations that can become lost.
        2935 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
        2936  the requested allocation to succeed.
        2937 */
        2938 struct VmaAllocationRequest
        2939 {
        2940  VkDeviceSize offset;
        2941  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
        2942  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
        2943  VmaSuballocationList::iterator item;
        2944  size_t itemsToMakeLostCount;
        2945 
        2946  VkDeviceSize CalcCost() const
        2947  {
        2948  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
        2949  }
        2950 };
        2951 
        2952 /*
        2953 Represents a single block of device memory (VkDeviceMemory ) with all the
        2954 data about its regions (aka suballocations, VmaAllocation), assigned and free.
        2955 
        2956 Thread-safety: This class must be externally synchronized.
        2957 */
        2958 class VmaDeviceMemoryBlock
        2959 {
        2960 public:
        2961  uint32_t m_MemoryTypeIndex;
        2962  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
        2963  VkDeviceMemory m_hMemory;
        2964  VkDeviceSize m_Size;
        2965  bool m_PersistentMap;
        2966  void* m_pMappedData;
        2967  uint32_t m_FreeCount;
        2968  VkDeviceSize m_SumFreeSize;
        2969  VmaSuballocationList m_Suballocations;
        2970  // Suballocations that are free and have size greater than certain threshold.
        2971  // Sorted by size, ascending.
        2972  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
        2973 
        2974  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
        2975 
        2976  ~VmaDeviceMemoryBlock()
        2977  {
        2978  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
        2979  }
        2980 
        2981  // Always call after construction.
        2982  void Init(
        2983  uint32_t newMemoryTypeIndex,
        2984  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
        2985  VkDeviceMemory newMemory,
        2986  VkDeviceSize newSize,
        2987  bool persistentMap,
        2988  void* pMappedData);
        2989  // Always call before destruction.
        2990  void Destroy(VmaAllocator allocator);
        2991 
        2992  // Validates all data structures inside this object. If not valid, returns false.
        2993  bool Validate() const;
        2994 
        2995  // Tries to find a place for suballocation with given parameters inside this allocation.
        2996  // If succeeded, fills pAllocationRequest and returns true.
        2997  // If failed, returns false.
        2998  bool CreateAllocationRequest(
        2999  uint32_t currentFrameIndex,
        3000  uint32_t frameInUseCount,
        3001  VkDeviceSize bufferImageGranularity,
        3002  VkDeviceSize allocSize,
        3003  VkDeviceSize allocAlignment,
        3004  VmaSuballocationType allocType,
        3005  bool canMakeOtherLost,
        3006  VmaAllocationRequest* pAllocationRequest);
        3007 
        3008  bool MakeRequestedAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest);
        3009 
        3010  uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
        3011 
        3012  // Returns true if this allocation is empty - contains only single free suballocation.
        3013  bool IsEmpty() const;
        3014 
        3015  // Makes actual allocation based on request. Request must already be checked
        3016  // and valid.
        3017  void Alloc(
        3018  const VmaAllocationRequest& request,
        3019  VmaSuballocationType type,
        3020  VkDeviceSize allocSize,
        3021  VmaAllocation hAllocation);
        3022 
        3023  // Frees suballocation assigned to given memory region.
        3024  void Free(const VmaAllocation allocation);
        3025 
        3026 #if VMA_STATS_STRING_ENABLED
        3027  void PrintDetailedMap(class VmaJsonWriter& json) const;
        3028 #endif
        3029 
        3030 private:
        3031  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
        3032  // If yes, fills pOffset and returns true. If no, returns false.
        3033  bool CheckAllocation(
        3034  uint32_t currentFrameIndex,
        3035  uint32_t frameInUseCount,
        3036  VkDeviceSize bufferImageGranularity,
        3037  VkDeviceSize allocSize,
        3038  VkDeviceSize allocAlignment,
        3039  VmaSuballocationType allocType,
        3040  VmaSuballocationList::const_iterator suballocItem,
        3041  bool canMakeOtherLost,
        3042  VkDeviceSize* pOffset,
        3043  size_t* itemsToMakeLostCount,
        3044  VkDeviceSize* pSumFreeSize,
        3045  VkDeviceSize* pSumItemSize) const;
        3046 
        3047  // Given free suballocation, it merges it with following one, which must also be free.
        3048  void MergeFreeWithNext(VmaSuballocationList::iterator item);
        3049  // Releases given suballocation, making it free.
        3050  // Merges it with adjacent free suballocations if applicable.
        3051  // Returns iterator to new free suballocation at this place.
        3052  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
        3053  // Given free suballocation, it inserts it into sorted list of
        3054  // m_FreeSuballocationsBySize if it's suitable.
        3055  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
        3056  // Given free suballocation, it removes it from sorted list of
        3057  // m_FreeSuballocationsBySize if it's suitable.
        3058  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
        3059 
        3060  bool ValidateFreeSuballocationList() const;
        3061 };
        3062 
        3063 struct VmaPointerLess
        3064 {
        3065  bool operator()(const void* lhs, const void* rhs) const
        3066  {
        3067  return lhs < rhs;
        3068  }
        3069 };
        3070 
        3071 class VmaDefragmentator;
        3072 
        3073 /*
        3074 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
        3075 Vulkan memory type.
        3076 
        3077 Synchronized internally with a mutex.
        3078 */
        3079 struct VmaBlockVector
        3080 {
        3081  VmaBlockVector(
        3082  VmaAllocator hAllocator,
        3083  uint32_t memoryTypeIndex,
        3084  VMA_BLOCK_VECTOR_TYPE blockVectorType,
        3085  VkDeviceSize preferredBlockSize,
        3086  size_t minBlockCount,
        3087  size_t maxBlockCount,
        3088  VkDeviceSize bufferImageGranularity,
        3089  uint32_t frameInUseCount,
        3090  bool isCustomPool);
        3091  ~VmaBlockVector();
        3092 
        3093  VkResult CreateMinBlocks();
        3094 
        3095  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
        3096  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
        3097  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
        3098  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
        3099  VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; }
        3100 
        3101  void GetPoolStats(VmaPoolStats* pStats);
        3102 
        3103  bool IsEmpty() const { return m_Blocks.empty(); }
        3104 
        3105  VkResult Allocate(
        3106  VmaPool hCurrentPool,
        3107  uint32_t currentFrameIndex,
        3108  const VkMemoryRequirements& vkMemReq,
        3109  const VmaAllocationCreateInfo& createInfo,
        3110  VmaSuballocationType suballocType,
        3111  VmaAllocation* pAllocation);
        3112 
        3113  void Free(
        3114  VmaAllocation hAllocation);
        3115 
        3116  // Adds statistics of this BlockVector to pStats.
        3117  void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex);
        3118 
        3119 #if VMA_STATS_STRING_ENABLED
        3120  void PrintDetailedMap(class VmaJsonWriter& json);
        3121 #endif
        3122 
        3123  void UnmapPersistentlyMappedMemory();
        3124  VkResult MapPersistentlyMappedMemory();
        3125 
        3126  void MakePoolAllocationsLost(
        3127  uint32_t currentFrameIndex,
        3128  size_t* pLostAllocationCount);
        3129 
        3130  VmaDefragmentator* EnsureDefragmentator(
        3131  VkDevice hDevice,
        3132  const VkAllocationCallbacks* pAllocationCallbacks,
        3133  uint32_t currentFrameIndex);
        3134 
        3135  VkResult Defragment(
        3136  VmaDefragmentationStats* pDefragmentationStats,
        3137  VkDeviceSize& maxBytesToMove,
        3138  uint32_t& maxAllocationsToMove);
        3139 
        3140  void DestroyDefragmentator();
        3141 
        3142 private:
        3143  friend class VmaDefragmentator;
        3144 
        3145  const VmaAllocator m_hAllocator;
        3146  const uint32_t m_MemoryTypeIndex;
        3147  const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
        3148  const VkDeviceSize m_PreferredBlockSize;
        3149  const size_t m_MinBlockCount;
        3150  const size_t m_MaxBlockCount;
        3151  const VkDeviceSize m_BufferImageGranularity;
        3152  const uint32_t m_FrameInUseCount;
        3153  const bool m_IsCustomPool;
        3154  VMA_MUTEX m_Mutex;
        3155  // Incrementally sorted by sumFreeSize, ascending.
        3156  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
        3157  /* There can be at most one allocation that is completely empty - a
        3158  hysteresis to avoid pessimistic case of alternating creation and destruction
        3159  of a VkDeviceMemory. */
        3160  bool m_HasEmptyBlock;
        3161  VmaDefragmentator* m_pDefragmentator;
        3162 
        3163  // Finds and removes given block from vector.
        3164  void Remove(VmaDeviceMemoryBlock* pBlock);
        3165 
        3166  // Performs single step in sorting m_Blocks. They may not be fully sorted
        3167  // after this call.
        3168  void IncrementallySortBlocks();
        3169 
        3170  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
        3171 };
        3172 
        3173 struct VmaPool_T
        3174 {
        3175 public:
        3176  VmaBlockVector m_BlockVector;
        3177 
        3178  // Takes ownership.
        3179  VmaPool_T(
        3180  VmaAllocator hAllocator,
        3181  const VmaPoolCreateInfo& createInfo);
        3182  ~VmaPool_T();
        3183 
        3184  VmaBlockVector& GetBlockVector() { return m_BlockVector; }
        3185 
        3186 #if VMA_STATS_STRING_ENABLED
        3187  //void PrintDetailedMap(class VmaStringBuilder& sb);
        3188 #endif
        3189 };
        3190 
        3191 class VmaDefragmentator
        3192 {
        3193  const VkDevice m_hDevice;
        3194  const VkAllocationCallbacks* const m_pAllocationCallbacks;
        3195  VmaBlockVector* const m_pBlockVector;
        3196  uint32_t m_CurrentFrameIndex;
        3197  VMA_BLOCK_VECTOR_TYPE m_BlockVectorType;
        3198  VkDeviceSize m_BytesMoved;
        3199  uint32_t m_AllocationsMoved;
        3200 
        3201  struct AllocationInfo
        3202  {
        3203  VmaAllocation m_hAllocation;
        3204  VkBool32* m_pChanged;
        3205 
        3206  AllocationInfo() :
        3207  m_hAllocation(VK_NULL_HANDLE),
        3208  m_pChanged(VMA_NULL)
        3209  {
        3210  }
        3211  };
        3212 
        3213  struct AllocationInfoSizeGreater
        3214  {
        3215  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
        3216  {
        3217  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
        3218  }
        3219  };
        3220 
        3221  // Used between AddAllocation and Defragment.
        3222  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
        3223 
        3224  struct BlockInfo
        3225  {
        3226  VmaDeviceMemoryBlock* m_pBlock;
        3227  bool m_HasNonMovableAllocations;
        3228  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
        3229 
        3230  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
        3231  m_pBlock(VMA_NULL),
        3232  m_HasNonMovableAllocations(true),
        3233  m_Allocations(pAllocationCallbacks),
        3234  m_pMappedDataForDefragmentation(VMA_NULL)
        3235  {
        3236  }
        3237 
        3238  void CalcHasNonMovableAllocations()
        3239  {
        3240  const size_t blockAllocCount =
        3241  m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount;
        3242  const size_t defragmentAllocCount = m_Allocations.size();
        3243  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
        3244  }
        3245 
        3246  void SortAllocationsBySizeDescecnding()
        3247  {
        3248  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
        3249  }
        3250 
        3251  VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData)
        3252  {
        3253  // It has already been mapped for defragmentation.
        3254  if(m_pMappedDataForDefragmentation)
        3255  {
        3256  *ppMappedData = m_pMappedDataForDefragmentation;
        3257  return VK_SUCCESS;
        3258  }
        3259 
        3260  // It is persistently mapped.
        3261  if(m_pBlock->m_PersistentMap)
        3262  {
        3263  VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL);
        3264  *ppMappedData = m_pBlock->m_pMappedData;
        3265  return VK_SUCCESS;
        3266  }
        3267 
        3268  // Map on first usage.
        3269  VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation);
        3270  *ppMappedData = m_pMappedDataForDefragmentation;
        3271  return res;
        3272  }
        3273 
        3274  void Unmap(VkDevice hDevice)
        3275  {
        3276  if(m_pMappedDataForDefragmentation != VMA_NULL)
        3277  {
        3278  vkUnmapMemory(hDevice, m_pBlock->m_hMemory);
        3279  }
        3280  }
        3281 
        3282  private:
        3283  // Not null if mapped for defragmentation only, not persistently mapped.
        3284  void* m_pMappedDataForDefragmentation;
        3285  };
        3286 
        3287  struct BlockPointerLess
        3288  {
        3289  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
        3290  {
        3291  return pLhsBlockInfo->m_pBlock < pRhsBlock;
        3292  }
        3293  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
        3294  {
        3295  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
        3296  }
        3297  };
        3298 
        3299  // 1. Blocks with some non-movable allocations go first.
        3300  // 2. Blocks with smaller sumFreeSize go first.
        3301  struct BlockInfoCompareMoveDestination
        3302  {
        3303  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
        3304  {
        3305  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
        3306  {
        3307  return true;
        3308  }
        3309  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
        3310  {
        3311  return false;
        3312  }
        3313  if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize)
        3314  {
        3315  return true;
        3316  }
        3317  return false;
        3318  }
        3319  };
        3320 
        3321  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
        3322  BlockInfoVector m_Blocks;
        3323 
        3324  VkResult DefragmentRound(
        3325  VkDeviceSize maxBytesToMove,
        3326  uint32_t maxAllocationsToMove);
        3327 
        3328  static bool MoveMakesSense(
        3329  size_t dstBlockIndex, VkDeviceSize dstOffset,
        3330  size_t srcBlockIndex, VkDeviceSize srcOffset);
        3331 
        3332 public:
        3333  VmaDefragmentator(
        3334  VkDevice hDevice,
        3335  const VkAllocationCallbacks* pAllocationCallbacks,
        3336  VmaBlockVector* pBlockVector,
        3337  uint32_t currentFrameIndex);
        3338 
        3339  ~VmaDefragmentator();
        3340 
        3341  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
        3342  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
        3343 
        3344  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
        3345 
        3346  VkResult Defragment(
        3347  VkDeviceSize maxBytesToMove,
        3348  uint32_t maxAllocationsToMove);
        3349 };
        3350 
        3351 // Main allocator object.
        3352 struct VmaAllocator_T
        3353 {
        3354  bool m_UseMutex;
        3355  VkDevice m_hDevice;
        3356  bool m_AllocationCallbacksSpecified;
        3357  VkAllocationCallbacks m_AllocationCallbacks;
        3358  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
        3359  // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory.
        3360  // Counter to allow nested calls to these functions.
        3361  uint32_t m_UnmapPersistentlyMappedMemoryCounter;
        3362 
        3363  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
        3364  VkPhysicalDeviceMemoryProperties m_MemProps;
        3365 
        3366  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
        3367 
        3368  // Each vector is sorted by memory (handle value).
        3369  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
        3370  AllocationVectorType* m_pOwnAllocations[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT];
        3371  VMA_MUTEX m_OwnAllocationsMutex[VK_MAX_MEMORY_TYPES];
        3372 
        3373  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
        3374  ~VmaAllocator_T();
        3375 
        3376  const VkAllocationCallbacks* GetAllocationCallbacks() const
        3377  {
        3378  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
        3379  }
        3380 
        3381  VkDeviceSize GetBufferImageGranularity() const
        3382  {
        3383  return VMA_MAX(
        3384  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
        3385  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
        3386  }
        3387 
        3388  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
        3389  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
        3390 
        3391  // Main allocation function.
        3392  VkResult AllocateMemory(
        3393  const VkMemoryRequirements& vkMemReq,
        3394  const VmaAllocationCreateInfo& createInfo,
        3395  VmaSuballocationType suballocType,
        3396  VmaAllocation* pAllocation);
        3397 
        3398  // Main deallocation function.
        3399  void FreeMemory(const VmaAllocation allocation);
        3400 
        3401  void CalculateStats(VmaStats* pStats);
        3402 
        3403 #if VMA_STATS_STRING_ENABLED
        3404  void PrintDetailedMap(class VmaJsonWriter& json);
        3405 #endif
        3406 
        3407  void UnmapPersistentlyMappedMemory();
        3408  VkResult MapPersistentlyMappedMemory();
        3409 
        3410  VkResult Defragment(
        3411  VmaAllocation* pAllocations,
        3412  size_t allocationCount,
        3413  VkBool32* pAllocationsChanged,
        3414  const VmaDefragmentationInfo* pDefragmentationInfo,
        3415  VmaDefragmentationStats* pDefragmentationStats);
        3416 
        3417  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
        3418 
        3419  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
        3420  void DestroyPool(VmaPool pool);
        3421  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
        3422 
        3423  void SetCurrentFrameIndex(uint32_t frameIndex);
        3424 
        3425  void MakePoolAllocationsLost(
        3426  VmaPool hPool,
        3427  size_t* pLostAllocationCount);
        3428 
        3429  void CreateLostAllocation(VmaAllocation* pAllocation);
        3430 
        3431 private:
        3432  VkDeviceSize m_PreferredLargeHeapBlockSize;
        3433  VkDeviceSize m_PreferredSmallHeapBlockSize;
        3434 
        3435  VkPhysicalDevice m_PhysicalDevice;
        3436  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
        3437 
        3438  VMA_MUTEX m_PoolsMutex;
        3439  // Protected by m_PoolsMutex. Sorted by pointer value.
        3440  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
        3441 
        3442  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
        3443 
        3444  VkResult AllocateMemoryOfType(
        3445  const VkMemoryRequirements& vkMemReq,
        3446  const VmaAllocationCreateInfo& createInfo,
        3447  uint32_t memTypeIndex,
        3448  VmaSuballocationType suballocType,
        3449  VmaAllocation* pAllocation);
        3450 
        3451  // Allocates and registers new VkDeviceMemory specifically for single allocation.
        3452  VkResult AllocateOwnMemory(
        3453  VkDeviceSize size,
        3454  VmaSuballocationType suballocType,
        3455  uint32_t memTypeIndex,
        3456  bool map,
        3457  void* pUserData,
        3458  VmaAllocation* pAllocation);
        3459 
        3460  // Tries to free pMemory as Own Memory. Returns true if found and freed.
        3461  void FreeOwnMemory(VmaAllocation allocation);
        3462 };
        3463 
        3465 // Memory allocation #2 after VmaAllocator_T definition
        3466 
        3467 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
        3468 {
        3469  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
        3470 }
        3471 
        3472 static void VmaFree(VmaAllocator hAllocator, void* ptr)
        3473 {
        3474  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
        3475 }
        3476 
        3477 template<typename T>
        3478 static T* VmaAllocate(VmaAllocator hAllocator)
        3479 {
        3480  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
        3481 }
        3482 
        3483 template<typename T>
        3484 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
        3485 {
        3486  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
        3487 }
        3488 
        3489 template<typename T>
        3490 static void vma_delete(VmaAllocator hAllocator, T* ptr)
        3491 {
        3492  if(ptr != VMA_NULL)
        3493  {
        3494  ptr->~T();
        3495  VmaFree(hAllocator, ptr);
        3496  }
        3497 }
        3498 
        3499 template<typename T>
        3500 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
        3501 {
        3502  if(ptr != VMA_NULL)
        3503  {
        3504  for(size_t i = count; i--; )
        3505  ptr[i].~T();
        3506  VmaFree(hAllocator, ptr);
        3507  }
        3508 }
        3509 
        3511 // VmaStringBuilder
        3512 
        3513 #if VMA_STATS_STRING_ENABLED
        3514 
        3515 class VmaStringBuilder
        3516 {
        3517 public:
        3518  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
        3519  size_t GetLength() const { return m_Data.size(); }
        3520  const char* GetData() const { return m_Data.data(); }
        3521 
        3522  void Add(char ch) { m_Data.push_back(ch); }
        3523  void Add(const char* pStr);
        3524  void AddNewLine() { Add('\n'); }
        3525  void AddNumber(uint32_t num);
        3526  void AddNumber(uint64_t num);
        3527  void AddPointer(const void* ptr);
        3528 
        3529 private:
        3530  VmaVector< char, VmaStlAllocator<char> > m_Data;
        3531 };
        3532 
        3533 void VmaStringBuilder::Add(const char* pStr)
        3534 {
        3535  const size_t strLen = strlen(pStr);
        3536  if(strLen > 0)
        3537  {
        3538  const size_t oldCount = m_Data.size();
        3539  m_Data.resize(oldCount + strLen);
        3540  memcpy(m_Data.data() + oldCount, pStr, strLen);
        3541  }
        3542 }
        3543 
        3544 void VmaStringBuilder::AddNumber(uint32_t num)
        3545 {
        3546  char buf[11];
        3547  VmaUint32ToStr(buf, sizeof(buf), num);
        3548  Add(buf);
        3549 }
        3550 
        3551 void VmaStringBuilder::AddNumber(uint64_t num)
        3552 {
        3553  char buf[21];
        3554  VmaUint64ToStr(buf, sizeof(buf), num);
        3555  Add(buf);
        3556 }
        3557 
        3558 void VmaStringBuilder::AddPointer(const void* ptr)
        3559 {
        3560  char buf[21];
        3561  VmaPtrToStr(buf, sizeof(buf), ptr);
        3562  Add(buf);
        3563 }
        3564 
        3565 #endif // #if VMA_STATS_STRING_ENABLED
        3566 
        3568 // VmaJsonWriter
        3569 
        3570 #if VMA_STATS_STRING_ENABLED
        3571 
        3572 class VmaJsonWriter
        3573 {
        3574 public:
        3575  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
        3576  ~VmaJsonWriter();
        3577 
        3578  void BeginObject(bool singleLine = false);
        3579  void EndObject();
        3580 
        3581  void BeginArray(bool singleLine = false);
        3582  void EndArray();
        3583 
        3584  void WriteString(const char* pStr);
        3585  void BeginString(const char* pStr = VMA_NULL);
        3586  void ContinueString(const char* pStr);
        3587  void ContinueString(uint32_t n);
        3588  void ContinueString(uint64_t n);
        3589  void EndString(const char* pStr = VMA_NULL);
        3590 
        3591  void WriteNumber(uint32_t n);
        3592  void WriteNumber(uint64_t n);
        3593  void WriteBool(bool b);
        3594  void WriteNull();
        3595 
        3596 private:
        3597  static const char* const INDENT;
        3598 
        3599  enum COLLECTION_TYPE
        3600  {
        3601  COLLECTION_TYPE_OBJECT,
        3602  COLLECTION_TYPE_ARRAY,
        3603  };
        3604  struct StackItem
        3605  {
        3606  COLLECTION_TYPE type;
        3607  uint32_t valueCount;
        3608  bool singleLineMode;
        3609  };
        3610 
        3611  VmaStringBuilder& m_SB;
        3612  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
        3613  bool m_InsideString;
        3614 
        3615  void BeginValue(bool isString);
        3616  void WriteIndent(bool oneLess = false);
        3617 };
        3618 
        3619 const char* const VmaJsonWriter::INDENT = " ";
        3620 
        3621 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
        3622  m_SB(sb),
        3623  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
        3624  m_InsideString(false)
        3625 {
        3626 }
        3627 
        3628 VmaJsonWriter::~VmaJsonWriter()
        3629 {
        3630  VMA_ASSERT(!m_InsideString);
        3631  VMA_ASSERT(m_Stack.empty());
        3632 }
        3633 
        3634 void VmaJsonWriter::BeginObject(bool singleLine)
        3635 {
        3636  VMA_ASSERT(!m_InsideString);
        3637 
        3638  BeginValue(false);
        3639  m_SB.Add('{');
        3640 
        3641  StackItem item;
        3642  item.type = COLLECTION_TYPE_OBJECT;
        3643  item.valueCount = 0;
        3644  item.singleLineMode = singleLine;
        3645  m_Stack.push_back(item);
        3646 }
        3647 
        3648 void VmaJsonWriter::EndObject()
        3649 {
        3650  VMA_ASSERT(!m_InsideString);
        3651 
        3652  WriteIndent(true);
        3653  m_SB.Add('}');
        3654 
        3655  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
        3656  m_Stack.pop_back();
        3657 }
        3658 
        3659 void VmaJsonWriter::BeginArray(bool singleLine)
        3660 {
        3661  VMA_ASSERT(!m_InsideString);
        3662 
        3663  BeginValue(false);
        3664  m_SB.Add('[');
        3665 
        3666  StackItem item;
        3667  item.type = COLLECTION_TYPE_ARRAY;
        3668  item.valueCount = 0;
        3669  item.singleLineMode = singleLine;
        3670  m_Stack.push_back(item);
        3671 }
        3672 
        3673 void VmaJsonWriter::EndArray()
        3674 {
        3675  VMA_ASSERT(!m_InsideString);
        3676 
        3677  WriteIndent(true);
        3678  m_SB.Add(']');
        3679 
        3680  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
        3681  m_Stack.pop_back();
        3682 }
        3683 
        3684 void VmaJsonWriter::WriteString(const char* pStr)
        3685 {
        3686  BeginString(pStr);
        3687  EndString();
        3688 }
        3689 
        3690 void VmaJsonWriter::BeginString(const char* pStr)
        3691 {
        3692  VMA_ASSERT(!m_InsideString);
        3693 
        3694  BeginValue(true);
        3695  m_SB.Add('"');
        3696  m_InsideString = true;
        3697  if(pStr != VMA_NULL && pStr[0] != '\0')
        3698  {
        3699  ContinueString(pStr);
        3700  }
        3701 }
        3702 
        3703 void VmaJsonWriter::ContinueString(const char* pStr)
        3704 {
        3705  VMA_ASSERT(m_InsideString);
        3706 
        3707  const size_t strLen = strlen(pStr);
        3708  for(size_t i = 0; i < strLen; ++i)
        3709  {
        3710  char ch = pStr[i];
        3711  if(ch == '\'')
        3712  {
        3713  m_SB.Add("\\\\");
        3714  }
        3715  else if(ch == '"')
        3716  {
        3717  m_SB.Add("\\\"");
        3718  }
        3719  else if(ch >= 32)
        3720  {
        3721  m_SB.Add(ch);
        3722  }
        3723  else switch(ch)
        3724  {
        3725  case '\n':
        3726  m_SB.Add("\\n");
        3727  break;
        3728  case '\r':
        3729  m_SB.Add("\\r");
        3730  break;
        3731  case '\t':
        3732  m_SB.Add("\\t");
        3733  break;
        3734  default:
        3735  VMA_ASSERT(0 && "Character not currently supported.");
        3736  break;
        3737  }
        3738  }
        3739 }
        3740 
        3741 void VmaJsonWriter::ContinueString(uint32_t n)
        3742 {
        3743  VMA_ASSERT(m_InsideString);
        3744  m_SB.AddNumber(n);
        3745 }
        3746 
        3747 void VmaJsonWriter::ContinueString(uint64_t n)
        3748 {
        3749  VMA_ASSERT(m_InsideString);
        3750  m_SB.AddNumber(n);
        3751 }
        3752 
        3753 void VmaJsonWriter::EndString(const char* pStr)
        3754 {
        3755  VMA_ASSERT(m_InsideString);
        3756  if(pStr != VMA_NULL && pStr[0] != '\0')
        3757  {
        3758  ContinueString(pStr);
        3759  }
        3760  m_SB.Add('"');
        3761  m_InsideString = false;
        3762 }
        3763 
        3764 void VmaJsonWriter::WriteNumber(uint32_t n)
        3765 {
        3766  VMA_ASSERT(!m_InsideString);
        3767  BeginValue(false);
        3768  m_SB.AddNumber(n);
        3769 }
        3770 
        3771 void VmaJsonWriter::WriteNumber(uint64_t n)
        3772 {
        3773  VMA_ASSERT(!m_InsideString);
        3774  BeginValue(false);
        3775  m_SB.AddNumber(n);
        3776 }
        3777 
        3778 void VmaJsonWriter::WriteBool(bool b)
        3779 {
        3780  VMA_ASSERT(!m_InsideString);
        3781  BeginValue(false);
        3782  m_SB.Add(b ? "true" : "false");
        3783 }
        3784 
        3785 void VmaJsonWriter::WriteNull()
        3786 {
        3787  VMA_ASSERT(!m_InsideString);
        3788  BeginValue(false);
        3789  m_SB.Add("null");
        3790 }
        3791 
        3792 void VmaJsonWriter::BeginValue(bool isString)
        3793 {
        3794  if(!m_Stack.empty())
        3795  {
        3796  StackItem& currItem = m_Stack.back();
        3797  if(currItem.type == COLLECTION_TYPE_OBJECT &&
        3798  currItem.valueCount % 2 == 0)
        3799  {
        3800  VMA_ASSERT(isString);
        3801  }
        3802 
        3803  if(currItem.type == COLLECTION_TYPE_OBJECT &&
        3804  currItem.valueCount % 2 != 0)
        3805  {
        3806  m_SB.Add(": ");
        3807  }
        3808  else if(currItem.valueCount > 0)
        3809  {
        3810  m_SB.Add(", ");
        3811  WriteIndent();
        3812  }
        3813  else
        3814  {
        3815  WriteIndent();
        3816  }
        3817  ++currItem.valueCount;
        3818  }
        3819 }
        3820 
        3821 void VmaJsonWriter::WriteIndent(bool oneLess)
        3822 {
        3823  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
        3824  {
        3825  m_SB.AddNewLine();
        3826 
        3827  size_t count = m_Stack.size();
        3828  if(count > 0 && oneLess)
        3829  {
        3830  --count;
        3831  }
        3832  for(size_t i = 0; i < count; ++i)
        3833  {
        3834  m_SB.Add(INDENT);
        3835  }
        3836  }
        3837 }
        3838 
        3839 #endif // #if VMA_STATS_STRING_ENABLED
        3840 
        3842 
        3843 VkDeviceSize VmaAllocation_T::GetOffset() const
        3844 {
        3845  switch(m_Type)
        3846  {
        3847  case ALLOCATION_TYPE_BLOCK:
        3848  return m_BlockAllocation.m_Offset;
        3849  case ALLOCATION_TYPE_OWN:
        3850  return 0;
        3851  default:
        3852  VMA_ASSERT(0);
        3853  return 0;
        3854  }
        3855 }
        3856 
        3857 VkDeviceMemory VmaAllocation_T::GetMemory() const
        3858 {
        3859  switch(m_Type)
        3860  {
        3861  case ALLOCATION_TYPE_BLOCK:
        3862  return m_BlockAllocation.m_Block->m_hMemory;
        3863  case ALLOCATION_TYPE_OWN:
        3864  return m_OwnAllocation.m_hMemory;
        3865  default:
        3866  VMA_ASSERT(0);
        3867  return VK_NULL_HANDLE;
        3868  }
        3869 }
        3870 
        3871 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
        3872 {
        3873  switch(m_Type)
        3874  {
        3875  case ALLOCATION_TYPE_BLOCK:
        3876  return m_BlockAllocation.m_Block->m_MemoryTypeIndex;
        3877  case ALLOCATION_TYPE_OWN:
        3878  return m_OwnAllocation.m_MemoryTypeIndex;
        3879  default:
        3880  VMA_ASSERT(0);
        3881  return UINT32_MAX;
        3882  }
        3883 }
        3884 
        3885 VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const
        3886 {
        3887  switch(m_Type)
        3888  {
        3889  case ALLOCATION_TYPE_BLOCK:
        3890  return m_BlockAllocation.m_Block->m_BlockVectorType;
        3891  case ALLOCATION_TYPE_OWN:
        3892  return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED);
        3893  default:
        3894  VMA_ASSERT(0);
        3895  return VMA_BLOCK_VECTOR_TYPE_COUNT;
        3896  }
        3897 }
        3898 
        3899 void* VmaAllocation_T::GetMappedData() const
        3900 {
        3901  switch(m_Type)
        3902  {
        3903  case ALLOCATION_TYPE_BLOCK:
        3904  if(m_BlockAllocation.m_Block->m_pMappedData != VMA_NULL)
        3905  {
        3906  return (char*)m_BlockAllocation.m_Block->m_pMappedData + m_BlockAllocation.m_Offset;
        3907  }
        3908  else
        3909  {
        3910  return VMA_NULL;
        3911  }
        3912  break;
        3913  case ALLOCATION_TYPE_OWN:
        3914  return m_OwnAllocation.m_pMappedData;
        3915  default:
        3916  VMA_ASSERT(0);
        3917  return VMA_NULL;
        3918  }
        3919 }
        3920 
        3921 bool VmaAllocation_T::CanBecomeLost() const
        3922 {
        3923  switch(m_Type)
        3924  {
        3925  case ALLOCATION_TYPE_BLOCK:
        3926  return m_BlockAllocation.m_CanBecomeLost;
        3927  case ALLOCATION_TYPE_OWN:
        3928  return false;
        3929  default:
        3930  VMA_ASSERT(0);
        3931  return false;
        3932  }
        3933 }
        3934 
        3935 VmaPool VmaAllocation_T::GetPool() const
        3936 {
        3937  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
        3938  return m_BlockAllocation.m_hPool;
        3939 }
        3940 
        3941 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
        3942 {
        3943  VMA_ASSERT(CanBecomeLost());
        3944 
        3945  /*
        3946  Warning: This is a carefully designed algorithm.
        3947  Do not modify unless you really know what you're doing :)
        3948  */
        3949  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
        3950  for(;;)
        3951  {
        3952  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
        3953  {
        3954  VMA_ASSERT(0);
        3955  return false;
        3956  }
        3957  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
        3958  {
        3959  return false;
        3960  }
        3961  else // Last use time earlier than current time.
        3962  {
        3963  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
        3964  {
        3965  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
        3966  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
        3967  return true;
        3968  }
        3969  }
        3970  }
        3971 }
        3972 
        3973 #if VMA_STATS_STRING_ENABLED
        3974 
        3975 // Correspond to values of enum VmaSuballocationType.
        3976 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
        3977  "FREE",
        3978  "UNKNOWN",
        3979  "BUFFER",
        3980  "IMAGE_UNKNOWN",
        3981  "IMAGE_LINEAR",
        3982  "IMAGE_OPTIMAL",
        3983 };
        3984 
        3985 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
        3986 {
        3987  json.BeginObject();
        3988 
        3989  json.WriteString("Blocks");
        3990  json.WriteNumber(stat.BlockCount);
        3991 
        3992  json.WriteString("Allocations");
        3993  json.WriteNumber(stat.AllocationCount);
        3994 
        3995  json.WriteString("UnusedRanges");
        3996  json.WriteNumber(stat.UnusedRangeCount);
        3997 
        3998  json.WriteString("UsedBytes");
        3999  json.WriteNumber(stat.UsedBytes);
        4000 
        4001  json.WriteString("UnusedBytes");
        4002  json.WriteNumber(stat.UnusedBytes);
        4003 
        4004  json.WriteString("AllocationSize");
        4005  json.BeginObject(true);
        4006  json.WriteString("Min");
        4007  json.WriteNumber(stat.AllocationSizeMin);
        4008  json.WriteString("Avg");
        4009  json.WriteNumber(stat.AllocationSizeAvg);
        4010  json.WriteString("Max");
        4011  json.WriteNumber(stat.AllocationSizeMax);
        4012  json.EndObject();
        4013 
        4014  json.WriteString("UnusedRangeSize");
        4015  json.BeginObject(true);
        4016  json.WriteString("Min");
        4017  json.WriteNumber(stat.UnusedRangeSizeMin);
        4018  json.WriteString("Avg");
        4019  json.WriteNumber(stat.UnusedRangeSizeAvg);
        4020  json.WriteString("Max");
        4021  json.WriteNumber(stat.UnusedRangeSizeMax);
        4022  json.EndObject();
        4023 
        4024  json.EndObject();
        4025 }
        4026 
        4027 #endif // #if VMA_STATS_STRING_ENABLED
        4028 
        4029 struct VmaSuballocationItemSizeLess
        4030 {
        4031  bool operator()(
        4032  const VmaSuballocationList::iterator lhs,
        4033  const VmaSuballocationList::iterator rhs) const
        4034  {
        4035  return lhs->size < rhs->size;
        4036  }
        4037  bool operator()(
        4038  const VmaSuballocationList::iterator lhs,
        4039  VkDeviceSize rhsSize) const
        4040  {
        4041  return lhs->size < rhsSize;
        4042  }
        4043 };
        4044 
        4045 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
        4046  m_MemoryTypeIndex(UINT32_MAX),
        4047  m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT),
        4048  m_hMemory(VK_NULL_HANDLE),
        4049  m_Size(0),
        4050  m_PersistentMap(false),
        4051  m_pMappedData(VMA_NULL),
        4052  m_FreeCount(0),
        4053  m_SumFreeSize(0),
        4054  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
        4055  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
        4056 {
        4057 }
        4058 
        4059 void VmaDeviceMemoryBlock::Init(
        4060  uint32_t newMemoryTypeIndex,
        4061  VMA_BLOCK_VECTOR_TYPE newBlockVectorType,
        4062  VkDeviceMemory newMemory,
        4063  VkDeviceSize newSize,
        4064  bool persistentMap,
        4065  void* pMappedData)
        4066 {
        4067  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
        4068 
        4069  m_MemoryTypeIndex = newMemoryTypeIndex;
        4070  m_BlockVectorType = newBlockVectorType;
        4071  m_hMemory = newMemory;
        4072  m_Size = newSize;
        4073  m_PersistentMap = persistentMap;
        4074  m_pMappedData = pMappedData;
        4075  m_FreeCount = 1;
        4076  m_SumFreeSize = newSize;
        4077 
        4078  m_Suballocations.clear();
        4079  m_FreeSuballocationsBySize.clear();
        4080 
        4081  VmaSuballocation suballoc = {};
        4082  suballoc.offset = 0;
        4083  suballoc.size = newSize;
        4084  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        4085  suballoc.hAllocation = VK_NULL_HANDLE;
        4086 
        4087  m_Suballocations.push_back(suballoc);
        4088  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
        4089  --suballocItem;
        4090  m_FreeSuballocationsBySize.push_back(suballocItem);
        4091 }
        4092 
        4093 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
        4094 {
        4095  // This is the most important assert in the entire library.
        4096  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
        4097  VMA_ASSERT(IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
        4098 
        4099  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
        4100  if(m_pMappedData != VMA_NULL)
        4101  {
        4102  vkUnmapMemory(allocator->m_hDevice, m_hMemory);
        4103  m_pMappedData = VMA_NULL;
        4104  }
        4105 
        4106  // Callback.
        4107  if(allocator->m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
        4108  {
        4109  (*allocator->m_DeviceMemoryCallbacks.pfnFree)(allocator, m_MemoryTypeIndex, m_hMemory, m_Size);
        4110  }
        4111 
        4112  vkFreeMemory(allocator->m_hDevice, m_hMemory, allocator->GetAllocationCallbacks());
        4113  m_hMemory = VK_NULL_HANDLE;
        4114 }
        4115 
        4116 bool VmaDeviceMemoryBlock::Validate() const
        4117 {
        4118  if((m_hMemory == VK_NULL_HANDLE) ||
        4119  (m_Size == 0) ||
        4120  m_Suballocations.empty())
        4121  {
        4122  return false;
        4123  }
        4124 
        4125  // Expected offset of new suballocation as calculates from previous ones.
        4126  VkDeviceSize calculatedOffset = 0;
        4127  // Expected number of free suballocations as calculated from traversing their list.
        4128  uint32_t calculatedFreeCount = 0;
        4129  // Expected sum size of free suballocations as calculated from traversing their list.
        4130  VkDeviceSize calculatedSumFreeSize = 0;
        4131  // Expected number of free suballocations that should be registered in
        4132  // m_FreeSuballocationsBySize calculated from traversing their list.
        4133  size_t freeSuballocationsToRegister = 0;
        4134  // True if previous visisted suballocation was free.
        4135  bool prevFree = false;
        4136 
        4137  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
        4138  suballocItem != m_Suballocations.cend();
        4139  ++suballocItem)
        4140  {
        4141  const VmaSuballocation& subAlloc = *suballocItem;
        4142 
        4143  // Actual offset of this suballocation doesn't match expected one.
        4144  if(subAlloc.offset != calculatedOffset)
        4145  {
        4146  return false;
        4147  }
        4148 
        4149  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
        4150  // Two adjacent free suballocations are invalid. They should be merged.
        4151  if(prevFree && currFree)
        4152  {
        4153  return false;
        4154  }
        4155  prevFree = currFree;
        4156 
        4157  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
        4158  {
        4159  return false;
        4160  }
        4161 
        4162  if(currFree)
        4163  {
        4164  calculatedSumFreeSize += subAlloc.size;
        4165  ++calculatedFreeCount;
        4166  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        4167  {
        4168  ++freeSuballocationsToRegister;
        4169  }
        4170  }
        4171 
        4172  calculatedOffset += subAlloc.size;
        4173  }
        4174 
        4175  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
        4176  // match expected one.
        4177  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
        4178  {
        4179  return false;
        4180  }
        4181 
        4182  VkDeviceSize lastSize = 0;
        4183  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
        4184  {
        4185  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
        4186 
        4187  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
        4188  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
        4189  {
        4190  return false;
        4191  }
        4192  // They must be sorted by size ascending.
        4193  if(suballocItem->size < lastSize)
        4194  {
        4195  return false;
        4196  }
        4197 
        4198  lastSize = suballocItem->size;
        4199  }
        4200 
        4201  // Check if totals match calculacted values.
        4202  return
        4203  (calculatedOffset == m_Size) &&
        4204  (calculatedSumFreeSize == m_SumFreeSize) &&
        4205  (calculatedFreeCount == m_FreeCount);
        4206 }
        4207 
        4208 /*
        4209 How many suitable free suballocations to analyze before choosing best one.
        4210 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
        4211  be chosen.
        4212 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
        4213  suballocations will be analized and best one will be chosen.
        4214 - Any other value is also acceptable.
        4215 */
        4216 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
        4217 
        4218 bool VmaDeviceMemoryBlock::CreateAllocationRequest(
        4219  uint32_t currentFrameIndex,
        4220  uint32_t frameInUseCount,
        4221  VkDeviceSize bufferImageGranularity,
        4222  VkDeviceSize allocSize,
        4223  VkDeviceSize allocAlignment,
        4224  VmaSuballocationType allocType,
        4225  bool canMakeOtherLost,
        4226  VmaAllocationRequest* pAllocationRequest)
        4227 {
        4228  VMA_ASSERT(allocSize > 0);
        4229  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
        4230  VMA_ASSERT(pAllocationRequest != VMA_NULL);
        4231  VMA_HEAVY_ASSERT(Validate());
        4232 
        4233  // There is not enough total free space in this block to fullfill the request: Early return.
        4234  if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
        4235  {
        4236  return false;
        4237  }
        4238 
        4239  // New algorithm, efficiently searching freeSuballocationsBySize.
        4240  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
        4241  if(freeSuballocCount > 0)
        4242  {
        4243  if(VMA_BEST_FIT)
        4244  {
        4245  // Find first free suballocation with size not less than allocSize.
        4246  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
        4247  m_FreeSuballocationsBySize.data(),
        4248  m_FreeSuballocationsBySize.data() + freeSuballocCount,
        4249  allocSize,
        4250  VmaSuballocationItemSizeLess());
        4251  size_t index = it - m_FreeSuballocationsBySize.data();
        4252  for(; index < freeSuballocCount; ++index)
        4253  {
        4254  if(CheckAllocation(
        4255  currentFrameIndex,
        4256  frameInUseCount,
        4257  bufferImageGranularity,
        4258  allocSize,
        4259  allocAlignment,
        4260  allocType,
        4261  m_FreeSuballocationsBySize[index],
        4262  false, // canMakeOtherLost
        4263  &pAllocationRequest->offset,
        4264  &pAllocationRequest->itemsToMakeLostCount,
        4265  &pAllocationRequest->sumFreeSize,
        4266  &pAllocationRequest->sumItemSize))
        4267  {
        4268  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
        4269  return true;
        4270  }
        4271  }
        4272  }
        4273  else
        4274  {
        4275  // Search staring from biggest suballocations.
        4276  for(size_t index = freeSuballocCount; index--; )
        4277  {
        4278  if(CheckAllocation(
        4279  currentFrameIndex,
        4280  frameInUseCount,
        4281  bufferImageGranularity,
        4282  allocSize,
        4283  allocAlignment,
        4284  allocType,
        4285  m_FreeSuballocationsBySize[index],
        4286  false, // canMakeOtherLost
        4287  &pAllocationRequest->offset,
        4288  &pAllocationRequest->itemsToMakeLostCount,
        4289  &pAllocationRequest->sumFreeSize,
        4290  &pAllocationRequest->sumItemSize))
        4291  {
        4292  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
        4293  return true;
        4294  }
        4295  }
        4296  }
        4297  }
        4298 
        4299  if(canMakeOtherLost)
        4300  {
        4301  // Brute-force algorithm. TODO: Come up with something better.
        4302 
        4303  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
        4304  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
        4305 
        4306  VmaAllocationRequest tmpAllocRequest = {};
        4307  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
        4308  suballocIt != m_Suballocations.end();
        4309  ++suballocIt)
        4310  {
        4311  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
        4312  suballocIt->hAllocation->CanBecomeLost())
        4313  {
        4314  if(CheckAllocation(
        4315  currentFrameIndex,
        4316  frameInUseCount,
        4317  bufferImageGranularity,
        4318  allocSize,
        4319  allocAlignment,
        4320  allocType,
        4321  suballocIt,
        4322  canMakeOtherLost,
        4323  &tmpAllocRequest.offset,
        4324  &tmpAllocRequest.itemsToMakeLostCount,
        4325  &tmpAllocRequest.sumFreeSize,
        4326  &tmpAllocRequest.sumItemSize))
        4327  {
        4328  tmpAllocRequest.item = suballocIt;
        4329 
        4330  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
        4331  {
        4332  *pAllocationRequest = tmpAllocRequest;
        4333  }
        4334  }
        4335  }
        4336  }
        4337 
        4338  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
        4339  {
        4340  return true;
        4341  }
        4342  }
        4343 
        4344  return false;
        4345 }
        4346 
        4347 bool VmaDeviceMemoryBlock::MakeRequestedAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest)
        4348 {
        4349  while(pAllocationRequest->itemsToMakeLostCount > 0)
        4350  {
        4351  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
        4352  {
        4353  ++pAllocationRequest->item;
        4354  }
        4355  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
        4356  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
        4357  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
        4358  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
        4359  {
        4360  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
        4361  --pAllocationRequest->itemsToMakeLostCount;
        4362  }
        4363  else
        4364  {
        4365  return false;
        4366  }
        4367  }
        4368 
        4369  VMA_HEAVY_ASSERT(Validate());
        4370  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
        4371  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
        4372 
        4373  return true;
        4374 }
        4375 
        4376 uint32_t VmaDeviceMemoryBlock::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
        4377 {
        4378  uint32_t lostAllocationCount = 0;
        4379  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
        4380  it != m_Suballocations.end();
        4381  ++it)
        4382  {
        4383  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
        4384  it->hAllocation->CanBecomeLost() &&
        4385  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
        4386  {
        4387  it = FreeSuballocation(it);
        4388  ++lostAllocationCount;
        4389  }
        4390  }
        4391  return lostAllocationCount;
        4392 }
        4393 
        4394 bool VmaDeviceMemoryBlock::CheckAllocation(
        4395  uint32_t currentFrameIndex,
        4396  uint32_t frameInUseCount,
        4397  VkDeviceSize bufferImageGranularity,
        4398  VkDeviceSize allocSize,
        4399  VkDeviceSize allocAlignment,
        4400  VmaSuballocationType allocType,
        4401  VmaSuballocationList::const_iterator suballocItem,
        4402  bool canMakeOtherLost,
        4403  VkDeviceSize* pOffset,
        4404  size_t* itemsToMakeLostCount,
        4405  VkDeviceSize* pSumFreeSize,
        4406  VkDeviceSize* pSumItemSize) const
        4407 {
        4408  VMA_ASSERT(allocSize > 0);
        4409  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
        4410  VMA_ASSERT(suballocItem != m_Suballocations.cend());
        4411  VMA_ASSERT(pOffset != VMA_NULL);
        4412 
        4413  *itemsToMakeLostCount = 0;
        4414  *pSumFreeSize = 0;
        4415  *pSumItemSize = 0;
        4416 
        4417  if(canMakeOtherLost)
        4418  {
        4419  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
        4420  {
        4421  *pSumFreeSize = suballocItem->size;
        4422  }
        4423  else
        4424  {
        4425  if(suballocItem->hAllocation->CanBecomeLost() &&
        4426  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
        4427  {
        4428  ++*itemsToMakeLostCount;
        4429  *pSumItemSize = suballocItem->size;
        4430  }
        4431  else
        4432  {
        4433  return false;
        4434  }
        4435  }
        4436 
        4437  // Remaining size is too small for this request: Early return.
        4438  if(m_Size - suballocItem->offset < allocSize)
        4439  {
        4440  return false;
        4441  }
        4442 
        4443  // Start from offset equal to beginning of this suballocation.
        4444  *pOffset = suballocItem->offset;
        4445 
        4446  // Apply VMA_DEBUG_MARGIN at the beginning.
        4447  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
        4448  {
        4449  *pOffset += VMA_DEBUG_MARGIN;
        4450  }
        4451 
        4452  // Apply alignment.
        4453  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
        4454  *pOffset = VmaAlignUp(*pOffset, alignment);
        4455 
        4456  // Check previous suballocations for BufferImageGranularity conflicts.
        4457  // Make bigger alignment if necessary.
        4458  if(bufferImageGranularity > 1)
        4459  {
        4460  bool bufferImageGranularityConflict = false;
        4461  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
        4462  while(prevSuballocItem != m_Suballocations.cbegin())
        4463  {
        4464  --prevSuballocItem;
        4465  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
        4466  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
        4467  {
        4468  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
        4469  {
        4470  bufferImageGranularityConflict = true;
        4471  break;
        4472  }
        4473  }
        4474  else
        4475  // Already on previous page.
        4476  break;
        4477  }
        4478  if(bufferImageGranularityConflict)
        4479  {
        4480  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
        4481  }
        4482  }
        4483 
        4484  // Now that we have final *pOffset, check if we are past suballocItem.
        4485  // If yes, return false - this function should be called for another suballocItem as starting point.
        4486  if(*pOffset >= suballocItem->offset + suballocItem->size)
        4487  {
        4488  return false;
        4489  }
        4490 
        4491  // Calculate padding at the beginning based on current offset.
        4492  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
        4493 
        4494  // Calculate required margin at the end if this is not last suballocation.
        4495  VmaSuballocationList::const_iterator next = suballocItem;
        4496  ++next;
        4497  const VkDeviceSize requiredEndMargin =
        4498  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
        4499 
        4500  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
        4501  // Another early return check.
        4502  if(suballocItem->offset + totalSize > m_Size)
        4503  {
        4504  return false;
        4505  }
        4506 
        4507  // Advance lastSuballocItem until desired size is reached.
        4508  // Update itemsToMakeLostCount.
        4509  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
        4510  if(totalSize > suballocItem->size)
        4511  {
        4512  VkDeviceSize remainingSize = totalSize - suballocItem->size;
        4513  while(remainingSize > 0)
        4514  {
        4515  ++lastSuballocItem;
        4516  if(lastSuballocItem == m_Suballocations.cend())
        4517  {
        4518  return false;
        4519  }
        4520  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
        4521  {
        4522  *pSumFreeSize += lastSuballocItem->size;
        4523  }
        4524  else
        4525  {
        4526  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
        4527  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
        4528  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
        4529  {
        4530  ++*itemsToMakeLostCount;
        4531  *pSumItemSize += lastSuballocItem->size;
        4532  }
        4533  else
        4534  {
        4535  return false;
        4536  }
        4537  }
        4538  remainingSize = (lastSuballocItem->size < remainingSize) ?
        4539  remainingSize - lastSuballocItem->size : 0;
        4540  }
        4541  }
        4542 
        4543  // Check next suballocations for BufferImageGranularity conflicts.
        4544  // If conflict exists, we must mark more allocations lost or fail.
        4545  if(bufferImageGranularity > 1)
        4546  {
        4547  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
        4548  ++nextSuballocItem;
        4549  while(nextSuballocItem != m_Suballocations.cend())
        4550  {
        4551  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
        4552  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
        4553  {
        4554  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
        4555  {
        4556  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
        4557  if(nextSuballoc.hAllocation->CanBecomeLost() &&
        4558  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
        4559  {
        4560  ++*itemsToMakeLostCount;
        4561  }
        4562  else
        4563  {
        4564  return false;
        4565  }
        4566  }
        4567  }
        4568  else
        4569  {
        4570  // Already on next page.
        4571  break;
        4572  }
        4573  ++nextSuballocItem;
        4574  }
        4575  }
        4576  }
        4577  else
        4578  {
        4579  const VmaSuballocation& suballoc = *suballocItem;
        4580  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
        4581 
        4582  *pSumFreeSize = suballoc.size;
        4583 
        4584  // Size of this suballocation is too small for this request: Early return.
        4585  if(suballoc.size < allocSize)
        4586  {
        4587  return false;
        4588  }
        4589 
        4590  // Start from offset equal to beginning of this suballocation.
        4591  *pOffset = suballoc.offset;
        4592 
        4593  // Apply VMA_DEBUG_MARGIN at the beginning.
        4594  if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
        4595  {
        4596  *pOffset += VMA_DEBUG_MARGIN;
        4597  }
        4598 
        4599  // Apply alignment.
        4600  const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
        4601  *pOffset = VmaAlignUp(*pOffset, alignment);
        4602 
        4603  // Check previous suballocations for BufferImageGranularity conflicts.
        4604  // Make bigger alignment if necessary.
        4605  if(bufferImageGranularity > 1)
        4606  {
        4607  bool bufferImageGranularityConflict = false;
        4608  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
        4609  while(prevSuballocItem != m_Suballocations.cbegin())
        4610  {
        4611  --prevSuballocItem;
        4612  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
        4613  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
        4614  {
        4615  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
        4616  {
        4617  bufferImageGranularityConflict = true;
        4618  break;
        4619  }
        4620  }
        4621  else
        4622  // Already on previous page.
        4623  break;
        4624  }
        4625  if(bufferImageGranularityConflict)
        4626  {
        4627  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
        4628  }
        4629  }
        4630 
        4631  // Calculate padding at the beginning based on current offset.
        4632  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
        4633 
        4634  // Calculate required margin at the end if this is not last suballocation.
        4635  VmaSuballocationList::const_iterator next = suballocItem;
        4636  ++next;
        4637  const VkDeviceSize requiredEndMargin =
        4638  (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
        4639 
        4640  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
        4641  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
        4642  {
        4643  return false;
        4644  }
        4645 
        4646  // Check next suballocations for BufferImageGranularity conflicts.
        4647  // If conflict exists, allocation cannot be made here.
        4648  if(bufferImageGranularity > 1)
        4649  {
        4650  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
        4651  ++nextSuballocItem;
        4652  while(nextSuballocItem != m_Suballocations.cend())
        4653  {
        4654  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
        4655  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
        4656  {
        4657  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
        4658  {
        4659  return false;
        4660  }
        4661  }
        4662  else
        4663  {
        4664  // Already on next page.
        4665  break;
        4666  }
        4667  ++nextSuballocItem;
        4668  }
        4669  }
        4670  }
        4671 
        4672  // All tests passed: Success. pOffset is already filled.
        4673  return true;
        4674 }
        4675 
        4676 bool VmaDeviceMemoryBlock::IsEmpty() const
        4677 {
        4678  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
        4679 }
        4680 
        4681 void VmaDeviceMemoryBlock::Alloc(
        4682  const VmaAllocationRequest& request,
        4683  VmaSuballocationType type,
        4684  VkDeviceSize allocSize,
        4685  VmaAllocation hAllocation)
        4686 {
        4687  VMA_ASSERT(request.item != m_Suballocations.end());
        4688  VmaSuballocation& suballoc = *request.item;
        4689  // Given suballocation is a free block.
        4690  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
        4691  // Given offset is inside this suballocation.
        4692  VMA_ASSERT(request.offset >= suballoc.offset);
        4693  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
        4694  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
        4695  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
        4696 
        4697  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
        4698  // it to become used.
        4699  UnregisterFreeSuballocation(request.item);
        4700 
        4701  suballoc.offset = request.offset;
        4702  suballoc.size = allocSize;
        4703  suballoc.type = type;
        4704  suballoc.hAllocation = hAllocation;
        4705 
        4706  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
        4707  if(paddingEnd)
        4708  {
        4709  VmaSuballocation paddingSuballoc = {};
        4710  paddingSuballoc.offset = request.offset + allocSize;
        4711  paddingSuballoc.size = paddingEnd;
        4712  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        4713  VmaSuballocationList::iterator next = request.item;
        4714  ++next;
        4715  const VmaSuballocationList::iterator paddingEndItem =
        4716  m_Suballocations.insert(next, paddingSuballoc);
        4717  RegisterFreeSuballocation(paddingEndItem);
        4718  }
        4719 
        4720  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
        4721  if(paddingBegin)
        4722  {
        4723  VmaSuballocation paddingSuballoc = {};
        4724  paddingSuballoc.offset = request.offset - paddingBegin;
        4725  paddingSuballoc.size = paddingBegin;
        4726  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        4727  const VmaSuballocationList::iterator paddingBeginItem =
        4728  m_Suballocations.insert(request.item, paddingSuballoc);
        4729  RegisterFreeSuballocation(paddingBeginItem);
        4730  }
        4731 
        4732  // Update totals.
        4733  m_FreeCount = m_FreeCount - 1;
        4734  if(paddingBegin > 0)
        4735  {
        4736  ++m_FreeCount;
        4737  }
        4738  if(paddingEnd > 0)
        4739  {
        4740  ++m_FreeCount;
        4741  }
        4742  m_SumFreeSize -= allocSize;
        4743 }
        4744 
        4745 VmaSuballocationList::iterator VmaDeviceMemoryBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
        4746 {
        4747  // Change this suballocation to be marked as free.
        4748  VmaSuballocation& suballoc = *suballocItem;
        4749  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
        4750  suballoc.hAllocation = VK_NULL_HANDLE;
        4751 
        4752  // Update totals.
        4753  ++m_FreeCount;
        4754  m_SumFreeSize += suballoc.size;
        4755 
        4756  // Merge with previous and/or next suballocation if it's also free.
        4757  bool mergeWithNext = false;
        4758  bool mergeWithPrev = false;
        4759 
        4760  VmaSuballocationList::iterator nextItem = suballocItem;
        4761  ++nextItem;
        4762  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
        4763  {
        4764  mergeWithNext = true;
        4765  }
        4766 
        4767  VmaSuballocationList::iterator prevItem = suballocItem;
        4768  if(suballocItem != m_Suballocations.begin())
        4769  {
        4770  --prevItem;
        4771  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
        4772  {
        4773  mergeWithPrev = true;
        4774  }
        4775  }
        4776 
        4777  if(mergeWithNext)
        4778  {
        4779  UnregisterFreeSuballocation(nextItem);
        4780  MergeFreeWithNext(suballocItem);
        4781  }
        4782 
        4783  if(mergeWithPrev)
        4784  {
        4785  UnregisterFreeSuballocation(prevItem);
        4786  MergeFreeWithNext(prevItem);
        4787  RegisterFreeSuballocation(prevItem);
        4788  return prevItem;
        4789  }
        4790  else
        4791  {
        4792  RegisterFreeSuballocation(suballocItem);
        4793  return suballocItem;
        4794  }
        4795 }
        4796 
        4797 void VmaDeviceMemoryBlock::Free(const VmaAllocation allocation)
        4798 {
        4799  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
        4800  suballocItem != m_Suballocations.end();
        4801  ++suballocItem)
        4802  {
        4803  VmaSuballocation& suballoc = *suballocItem;
        4804  if(suballoc.hAllocation == allocation)
        4805  {
        4806  FreeSuballocation(suballocItem);
        4807  VMA_HEAVY_ASSERT(Validate());
        4808  return;
        4809  }
        4810  }
        4811  VMA_ASSERT(0 && "Not found!");
        4812 }
        4813 
        4814 #if VMA_STATS_STRING_ENABLED
        4815 
        4816 void VmaDeviceMemoryBlock::PrintDetailedMap(class VmaJsonWriter& json) const
        4817 {
        4818  json.BeginObject();
        4819 
        4820  json.WriteString("TotalBytes");
        4821  json.WriteNumber(m_Size);
        4822 
        4823  json.WriteString("UnusedBytes");
        4824  json.WriteNumber(m_SumFreeSize);
        4825 
        4826  json.WriteString("Allocations");
        4827  json.WriteNumber(m_Suballocations.size() - m_FreeCount);
        4828 
        4829  json.WriteString("UnusedRanges");
        4830  json.WriteNumber(m_FreeCount);
        4831 
        4832  json.WriteString("Suballocations");
        4833  json.BeginArray();
        4834  size_t i = 0;
        4835  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
        4836  suballocItem != m_Suballocations.cend();
        4837  ++suballocItem, ++i)
        4838  {
        4839  json.BeginObject(true);
        4840 
        4841  json.WriteString("Type");
        4842  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
        4843 
        4844  json.WriteString("Size");
        4845  json.WriteNumber(suballocItem->size);
        4846 
        4847  json.WriteString("Offset");
        4848  json.WriteNumber(suballocItem->offset);
        4849 
        4850  json.EndObject();
        4851  }
        4852  json.EndArray();
        4853 
        4854  json.EndObject();
        4855 }
        4856 
        4857 #endif // #if VMA_STATS_STRING_ENABLED
        4858 
        4859 void VmaDeviceMemoryBlock::MergeFreeWithNext(VmaSuballocationList::iterator item)
        4860 {
        4861  VMA_ASSERT(item != m_Suballocations.end());
        4862  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        4863 
        4864  VmaSuballocationList::iterator nextItem = item;
        4865  ++nextItem;
        4866  VMA_ASSERT(nextItem != m_Suballocations.end());
        4867  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
        4868 
        4869  item->size += nextItem->size;
        4870  --m_FreeCount;
        4871  m_Suballocations.erase(nextItem);
        4872 }
        4873 
        4874 void VmaDeviceMemoryBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
        4875 {
        4876  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        4877  VMA_ASSERT(item->size > 0);
        4878 
        4879  // You may want to enable this validation at the beginning or at the end of
        4880  // this function, depending on what do you want to check.
        4881  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
        4882 
        4883  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        4884  {
        4885  if(m_FreeSuballocationsBySize.empty())
        4886  {
        4887  m_FreeSuballocationsBySize.push_back(item);
        4888  }
        4889  else
        4890  {
        4891  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
        4892  }
        4893  }
        4894 
        4895  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
        4896 }
        4897 
        4898 
        4899 void VmaDeviceMemoryBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
        4900 {
        4901  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
        4902  VMA_ASSERT(item->size > 0);
        4903 
        4904  // You may want to enable this validation at the beginning or at the end of
        4905  // this function, depending on what do you want to check.
        4906  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
        4907 
        4908  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        4909  {
        4910  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
        4911  m_FreeSuballocationsBySize.data(),
        4912  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
        4913  item,
        4914  VmaSuballocationItemSizeLess());
        4915  for(size_t index = it - m_FreeSuballocationsBySize.data();
        4916  index < m_FreeSuballocationsBySize.size();
        4917  ++index)
        4918  {
        4919  if(m_FreeSuballocationsBySize[index] == item)
        4920  {
        4921  VmaVectorRemove(m_FreeSuballocationsBySize, index);
        4922  return;
        4923  }
        4924  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
        4925  }
        4926  VMA_ASSERT(0 && "Not found.");
        4927  }
        4928 
        4929  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
        4930 }
        4931 
        4932 bool VmaDeviceMemoryBlock::ValidateFreeSuballocationList() const
        4933 {
        4934  VkDeviceSize lastSize = 0;
        4935  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
        4936  {
        4937  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
        4938 
        4939  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
        4940  {
        4941  VMA_ASSERT(0);
        4942  return false;
        4943  }
        4944  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
        4945  {
        4946  VMA_ASSERT(0);
        4947  return false;
        4948  }
        4949  if(it->size < lastSize)
        4950  {
        4951  VMA_ASSERT(0);
        4952  return false;
        4953  }
        4954 
        4955  lastSize = it->size;
        4956  }
        4957  return true;
        4958 }
        4959 
        4960 static void InitStatInfo(VmaStatInfo& outInfo)
        4961 {
        4962  memset(&outInfo, 0, sizeof(outInfo));
        4963  outInfo.AllocationSizeMin = UINT64_MAX;
        4964  outInfo.UnusedRangeSizeMin = UINT64_MAX;
        4965 }
        4966 
        4967 static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaDeviceMemoryBlock& alloc)
        4968 {
        4969  outInfo.BlockCount = 1;
        4970 
        4971  const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size();
        4972  outInfo.AllocationCount = rangeCount - alloc.m_FreeCount;
        4973  outInfo.UnusedRangeCount = alloc.m_FreeCount;
        4974 
        4975  outInfo.UnusedBytes = alloc.m_SumFreeSize;
        4976  outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes;
        4977 
        4978  outInfo.AllocationSizeMin = UINT64_MAX;
        4979  outInfo.AllocationSizeMax = 0;
        4980  outInfo.UnusedRangeSizeMin = UINT64_MAX;
        4981  outInfo.UnusedRangeSizeMax = 0;
        4982 
        4983  for(VmaSuballocationList::const_iterator suballocItem = alloc.m_Suballocations.cbegin();
        4984  suballocItem != alloc.m_Suballocations.cend();
        4985  ++suballocItem)
        4986  {
        4987  const VmaSuballocation& suballoc = *suballocItem;
        4988  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
        4989  {
        4990  outInfo.AllocationSizeMin = VMA_MIN(outInfo.AllocationSizeMin, suballoc.size);
        4991  outInfo.AllocationSizeMax = VMA_MAX(outInfo.AllocationSizeMax, suballoc.size);
        4992  }
        4993  else
        4994  {
        4995  outInfo.UnusedRangeSizeMin = VMA_MIN(outInfo.UnusedRangeSizeMin, suballoc.size);
        4996  outInfo.UnusedRangeSizeMax = VMA_MAX(outInfo.UnusedRangeSizeMax, suballoc.size);
        4997  }
        4998  }
        4999 }
        5000 
        5001 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
        5002 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
        5003 {
        5004  inoutInfo.BlockCount += srcInfo.BlockCount;
        5005  inoutInfo.AllocationCount += srcInfo.AllocationCount;
        5006  inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount;
        5007  inoutInfo.UsedBytes += srcInfo.UsedBytes;
        5008  inoutInfo.UnusedBytes += srcInfo.UnusedBytes;
        5009  inoutInfo.AllocationSizeMin = VMA_MIN(inoutInfo.AllocationSizeMin, srcInfo.AllocationSizeMin);
        5010  inoutInfo.AllocationSizeMax = VMA_MAX(inoutInfo.AllocationSizeMax, srcInfo.AllocationSizeMax);
        5011  inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin);
        5012  inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax);
        5013 }
        5014 
        5015 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
        5016 {
        5017  inoutInfo.AllocationSizeAvg = (inoutInfo.AllocationCount > 0) ?
        5018  VmaRoundDiv<VkDeviceSize>(inoutInfo.UsedBytes, inoutInfo.AllocationCount) : 0;
        5019  inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ?
        5020  VmaRoundDiv<VkDeviceSize>(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0;
        5021 }
        5022 
        5023 VmaPool_T::VmaPool_T(
        5024  VmaAllocator hAllocator,
        5025  const VmaPoolCreateInfo& createInfo) :
        5026  m_BlockVector(
        5027  hAllocator,
        5028  createInfo.memoryTypeIndex,
        5029  (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ?
        5030  VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED,
        5031  createInfo.blockSize,
        5032  createInfo.minBlockCount,
        5033  createInfo.maxBlockCount,
        5034  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
        5035  createInfo.frameInUseCount,
        5036  true) // isCustomPool
        5037 {
        5038 }
        5039 
        5040 VmaPool_T::~VmaPool_T()
        5041 {
        5042 }
        5043 
        5044 #if VMA_STATS_STRING_ENABLED
        5045 
        5046 /*
        5047 void VmaPool_T::PrintDetailedMap(class VmaStringBuilder& sb)
        5048 {
        5049  sb.Add("\n{\n\"This\": \"");
        5050  sb.AddPointer(this);
        5051  sb.Add("\",\n\"Flags\": ");
        5052  sb.AddNumber(m_Flags);
        5053  sb.Add(",\n\"FrameInUseCount\": ");
        5054  sb.AddNumber(m_FrameInUseCount);
        5055  sb.Add(",\n\"DeviceMemoryBlock\": ");
        5056  m_pDeviceMemoryBlock->PrintDetailedMap(sb);
        5057  sb.Add("\n}");
        5058 }
        5059 */
        5060 
        5061 #endif // #if VMA_STATS_STRING_ENABLED
        5062 
        5063 VmaBlockVector::VmaBlockVector(
        5064  VmaAllocator hAllocator,
        5065  uint32_t memoryTypeIndex,
        5066  VMA_BLOCK_VECTOR_TYPE blockVectorType,
        5067  VkDeviceSize preferredBlockSize,
        5068  size_t minBlockCount,
        5069  size_t maxBlockCount,
        5070  VkDeviceSize bufferImageGranularity,
        5071  uint32_t frameInUseCount,
        5072  bool isCustomPool) :
        5073  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
        5074  m_hAllocator(hAllocator),
        5075  m_MemoryTypeIndex(memoryTypeIndex),
        5076  m_BlockVectorType(blockVectorType),
        5077  m_PreferredBlockSize(preferredBlockSize),
        5078  m_MinBlockCount(minBlockCount),
        5079  m_MaxBlockCount(maxBlockCount),
        5080  m_BufferImageGranularity(bufferImageGranularity),
        5081  m_FrameInUseCount(frameInUseCount),
        5082  m_IsCustomPool(isCustomPool),
        5083  m_HasEmptyBlock(false),
        5084  m_pDefragmentator(VMA_NULL)
        5085 {
        5086 }
        5087 
        5088 VmaBlockVector::~VmaBlockVector()
        5089 {
        5090  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
        5091 
        5092  for(size_t i = m_Blocks.size(); i--; )
        5093  {
        5094  m_Blocks[i]->Destroy(m_hAllocator);
        5095  vma_delete(m_hAllocator, m_Blocks[i]);
        5096  }
        5097 }
        5098 
        5099 VkResult VmaBlockVector::CreateMinBlocks()
        5100 {
        5101  for(size_t i = 0; i < m_MinBlockCount; ++i)
        5102  {
        5103  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
        5104  if(res != VK_SUCCESS)
        5105  {
        5106  return res;
        5107  }
        5108  }
        5109  return VK_SUCCESS;
        5110 }
        5111 
        5112 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
        5113 {
        5114  pStats->size = 0;
        5115  pStats->unusedSize = 0;
        5116  pStats->allocationCount = 0;
        5117  pStats->unusedRangeCount = 0;
        5118 
        5119  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5120 
        5121  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
        5122  {
        5123  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
        5124  VMA_ASSERT(pBlock);
        5125  VMA_HEAVY_ASSERT(pBlock->Validate());
        5126 
        5127  const uint32_t rangeCount = (uint32_t)pBlock->m_Suballocations.size();
        5128 
        5129  pStats->size += pBlock->m_Size;
        5130  pStats->unusedSize += pBlock->m_SumFreeSize;
        5131  pStats->allocationCount += rangeCount - pBlock->m_FreeCount;
        5132  pStats->unusedRangeCount += pBlock->m_FreeCount;
        5133  }
        5134 }
        5135 
        5136 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
        5137 
        5138 VkResult VmaBlockVector::Allocate(
        5139  VmaPool hCurrentPool,
        5140  uint32_t currentFrameIndex,
        5141  const VkMemoryRequirements& vkMemReq,
        5142  const VmaAllocationCreateInfo& createInfo,
        5143  VmaSuballocationType suballocType,
        5144  VmaAllocation* pAllocation)
        5145 {
        5146  // Validate flags.
        5147  if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) !=
        5148  (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED))
        5149  {
        5150  VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT.");
        5151  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        5152  }
        5153 
        5154  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5155 
        5156  // 1. Search existing allocations. Try to allocate without making other allocations lost.
        5157  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
        5158  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
        5159  {
        5160  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
        5161  VMA_ASSERT(pCurrBlock);
        5162  VmaAllocationRequest currRequest = {};
        5163  if(pCurrBlock->CreateAllocationRequest(
        5164  currentFrameIndex,
        5165  m_FrameInUseCount,
        5166  m_BufferImageGranularity,
        5167  vkMemReq.size,
        5168  vkMemReq.alignment,
        5169  suballocType,
        5170  false, // canMakeOtherLost
        5171  &currRequest))
        5172  {
        5173  // Allocate from pCurrBlock.
        5174  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
        5175 
        5176  // We no longer have an empty Allocation.
        5177  if(pCurrBlock->IsEmpty())
        5178  {
        5179  m_HasEmptyBlock = false;
        5180  }
        5181 
        5182  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
        5183  pCurrBlock->Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
        5184  (*pAllocation)->InitBlockAllocation(
        5185  hCurrentPool,
        5186  pCurrBlock,
        5187  currRequest.offset,
        5188  vkMemReq.alignment,
        5189  vkMemReq.size,
        5190  suballocType,
        5191  createInfo.pUserData,
        5192  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
        5193  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
        5194  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
        5195  return VK_SUCCESS;
        5196  }
        5197  }
        5198 
        5199  const bool canCreateNewBlock =
        5200  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
        5201  (m_Blocks.size() < m_MaxBlockCount);
        5202 
        5203  // 2. Try to create new block.
        5204  if(canCreateNewBlock)
        5205  {
        5206  // 2.1. Start with full preferredBlockSize.
        5207  VkDeviceSize blockSize = m_PreferredBlockSize;
        5208  size_t newBlockIndex = 0;
        5209  VkResult res = CreateBlock(blockSize, &newBlockIndex);
        5210  // Allocating blocks of other sizes is allowed only in default pools.
        5211  // In custom pools block size is fixed.
        5212  if(res < 0 && m_IsCustomPool == false)
        5213  {
        5214  // 2.2. Try half the size.
        5215  blockSize /= 2;
        5216  if(blockSize >= vkMemReq.size)
        5217  {
        5218  res = CreateBlock(blockSize, &newBlockIndex);
        5219  if(res < 0)
        5220  {
        5221  // 2.3. Try quarter the size.
        5222  blockSize /= 2;
        5223  if(blockSize >= vkMemReq.size)
        5224  {
        5225  res = CreateBlock(blockSize, &newBlockIndex);
        5226  }
        5227  }
        5228  }
        5229  }
        5230  if(res == VK_SUCCESS)
        5231  {
        5232  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
        5233  VMA_ASSERT(pBlock->m_Size >= vkMemReq.size);
        5234 
        5235  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
        5236  VmaAllocationRequest allocRequest = {};
        5237  allocRequest.item = pBlock->m_Suballocations.begin();
        5238  allocRequest.offset = 0;
        5239  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
        5240  pBlock->Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
        5241  (*pAllocation)->InitBlockAllocation(
        5242  hCurrentPool,
        5243  pBlock,
        5244  allocRequest.offset,
        5245  vkMemReq.alignment,
        5246  vkMemReq.size,
        5247  suballocType,
        5248  createInfo.pUserData,
        5249  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
        5250  VMA_HEAVY_ASSERT(pBlock->Validate());
        5251  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
        5252 
        5253  return VK_SUCCESS;
        5254  }
        5255  }
        5256 
        5257  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
        5258 
        5259  // 3. Try to allocate from existing blocks with making other allocations lost.
        5260  if(canMakeOtherLost)
        5261  {
        5262  uint32_t tryIndex = 0;
        5263  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
        5264  {
        5265  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
        5266  VmaAllocationRequest bestRequest = {};
        5267  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
        5268 
        5269  // 1. Search existing allocations.
        5270  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
        5271  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
        5272  {
        5273  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
        5274  VMA_ASSERT(pCurrBlock);
        5275  VmaAllocationRequest currRequest = {};
        5276  if(pCurrBlock->CreateAllocationRequest(
        5277  currentFrameIndex,
        5278  m_FrameInUseCount,
        5279  m_BufferImageGranularity,
        5280  vkMemReq.size,
        5281  vkMemReq.alignment,
        5282  suballocType,
        5283  canMakeOtherLost,
        5284  &currRequest))
        5285  {
        5286  const VkDeviceSize currRequestCost = currRequest.CalcCost();
        5287  if(pBestRequestBlock == VMA_NULL ||
        5288  currRequestCost < bestRequestCost)
        5289  {
        5290  pBestRequestBlock = pCurrBlock;
        5291  bestRequest = currRequest;
        5292  bestRequestCost = currRequestCost;
        5293 
        5294  if(bestRequestCost == 0)
        5295  {
        5296  break;
        5297  }
        5298  }
        5299  }
        5300  }
        5301 
        5302  if(pBestRequestBlock != VMA_NULL)
        5303  {
        5304  if(pBestRequestBlock->MakeRequestedAllocationsLost(
        5305  currentFrameIndex,
        5306  m_FrameInUseCount,
        5307  &bestRequest))
        5308  {
        5309  // We no longer have an empty Allocation.
        5310  if(pBestRequestBlock->IsEmpty())
        5311  {
        5312  m_HasEmptyBlock = false;
        5313  }
        5314  // Allocate from this pBlock.
        5315  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex);
        5316  pBestRequestBlock->Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
        5317  (*pAllocation)->InitBlockAllocation(
        5318  hCurrentPool,
        5319  pBestRequestBlock,
        5320  bestRequest.offset,
        5321  vkMemReq.alignment,
        5322  vkMemReq.size,
        5323  suballocType,
        5324  createInfo.pUserData,
        5325  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
        5326  VMA_HEAVY_ASSERT(pBlock->Validate());
        5327  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
        5328  return VK_SUCCESS;
        5329  }
        5330  // else: Some allocations must have been touched while we are here. Next try.
        5331  }
        5332  else
        5333  {
        5334  // Could not find place in any of the blocks - break outer loop.
        5335  break;
        5336  }
        5337  }
        5338  /* Maximum number of tries exceeded - a very unlike event when many other
        5339  threads are simultaneously touching allocations making it impossible to make
        5340  lost at the same time as we try to allocate. */
        5341  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
        5342  {
        5343  return VK_ERROR_TOO_MANY_OBJECTS;
        5344  }
        5345  }
        5346 
        5347  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        5348 }
        5349 
        5350 void VmaBlockVector::Free(
        5351  VmaAllocation hAllocation)
        5352 {
        5353  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
        5354 
        5355  // Scope for lock.
        5356  {
        5357  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5358 
        5359  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
        5360 
        5361  pBlock->Free(hAllocation);
        5362  VMA_HEAVY_ASSERT(pBlock->Validate());
        5363 
        5364  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
        5365 
        5366  // pBlock became empty after this deallocation.
        5367  if(pBlock->IsEmpty())
        5368  {
        5369  // Already has empty Allocation. We don't want to have two, so delete this one.
        5370  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
        5371  {
        5372  pBlockToDelete = pBlock;
        5373  Remove(pBlock);
        5374  }
        5375  // We now have first empty Allocation.
        5376  else
        5377  {
        5378  m_HasEmptyBlock = true;
        5379  }
        5380  }
        5381  // Must be called after srcBlockIndex is used, because later it may become invalid!
        5382  IncrementallySortBlocks();
        5383  }
        5384 
        5385  // Destruction of a free Allocation. Deferred until this point, outside of mutex
        5386  // lock, for performance reason.
        5387  if(pBlockToDelete != VMA_NULL)
        5388  {
        5389  VMA_DEBUG_LOG(" Deleted empty allocation");
        5390  pBlockToDelete->Destroy(m_hAllocator);
        5391  vma_delete(m_hAllocator, pBlockToDelete);
        5392  }
        5393 }
        5394 
        5395 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
        5396 {
        5397  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
        5398  {
        5399  if(m_Blocks[blockIndex] == pBlock)
        5400  {
        5401  VmaVectorRemove(m_Blocks, blockIndex);
        5402  return;
        5403  }
        5404  }
        5405  VMA_ASSERT(0);
        5406 }
        5407 
        5408 void VmaBlockVector::IncrementallySortBlocks()
        5409 {
        5410  // Bubble sort only until first swap.
        5411  for(size_t i = 1; i < m_Blocks.size(); ++i)
        5412  {
        5413  if(m_Blocks[i - 1]->m_SumFreeSize > m_Blocks[i]->m_SumFreeSize)
        5414  {
        5415  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
        5416  return;
        5417  }
        5418  }
        5419 }
        5420 
        5421 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
        5422 {
        5423  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
        5424  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
        5425  allocInfo.allocationSize = blockSize;
        5426  const VkDevice hDevice = m_hAllocator->m_hDevice;
        5427  VkDeviceMemory mem = VK_NULL_HANDLE;
        5428  VkResult res = vkAllocateMemory(hDevice, &allocInfo, m_hAllocator->GetAllocationCallbacks(), &mem);
        5429  if(res < 0)
        5430  {
        5431  return res;
        5432  }
        5433 
        5434  // New VkDeviceMemory successfully created.
        5435 
        5436  // Map memory if needed.
        5437  void* pMappedData = VMA_NULL;
        5438  const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED);
        5439  if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0)
        5440  {
        5441  res = vkMapMemory(hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData);
        5442  if(res < 0)
        5443  {
        5444  VMA_DEBUG_LOG(" vkMapMemory FAILED");
        5445  vkFreeMemory(hDevice, mem, m_hAllocator->GetAllocationCallbacks());
        5446  return res;
        5447  }
        5448  }
        5449 
        5450  // Callback.
        5451  if(m_hAllocator->m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
        5452  {
        5453  (*m_hAllocator->m_DeviceMemoryCallbacks.pfnAllocate)(m_hAllocator, m_MemoryTypeIndex, mem, allocInfo.allocationSize);
        5454  }
        5455 
        5456  // Create new Allocation for it.
        5457  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
        5458  pBlock->Init(
        5459  m_MemoryTypeIndex,
        5460  (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType,
        5461  mem,
        5462  allocInfo.allocationSize,
        5463  persistentMap,
        5464  pMappedData);
        5465 
        5466  m_Blocks.push_back(pBlock);
        5467  if(pNewBlockIndex != VMA_NULL)
        5468  {
        5469  *pNewBlockIndex = m_Blocks.size() - 1;
        5470  }
        5471 
        5472  return VK_SUCCESS;
        5473 }
        5474 
        5475 #if VMA_STATS_STRING_ENABLED
        5476 
        5477 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
        5478 {
        5479  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5480  json.BeginArray();
        5481  for(size_t i = 0; i < m_Blocks.size(); ++i)
        5482  {
        5483  m_Blocks[i]->PrintDetailedMap(json);
        5484  }
        5485  json.EndArray();
        5486 }
        5487 
        5488 #endif // #if VMA_STATS_STRING_ENABLED
        5489 
        5490 void VmaBlockVector::UnmapPersistentlyMappedMemory()
        5491 {
        5492  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5493 
        5494  for(size_t i = m_Blocks.size(); i--; )
        5495  {
        5496  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
        5497  if(pBlock->m_pMappedData != VMA_NULL)
        5498  {
        5499  VMA_ASSERT(pBlock->m_PersistentMap != false);
        5500  vkUnmapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory);
        5501  pBlock->m_pMappedData = VMA_NULL;
        5502  }
        5503  }
        5504 }
        5505 
        5506 VkResult VmaBlockVector::MapPersistentlyMappedMemory()
        5507 {
        5508  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5509 
        5510  VkResult finalResult = VK_SUCCESS;
        5511  for(size_t i = 0, count = m_Blocks.size(); i < count; ++i)
        5512  {
        5513  VmaDeviceMemoryBlock* pBlock = m_Blocks[i];
        5514  if(pBlock->m_PersistentMap)
        5515  {
        5516  VMA_ASSERT(pBlock->m_pMappedData == nullptr);
        5517  VkResult localResult = vkMapMemory(m_hAllocator->m_hDevice, pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &pBlock->m_pMappedData);
        5518  if(localResult != VK_SUCCESS)
        5519  {
        5520  finalResult = localResult;
        5521  }
        5522  }
        5523  }
        5524  return finalResult;
        5525 }
        5526 
        5527 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
        5528  VkDevice hDevice,
        5529  const VkAllocationCallbacks* pAllocationCallbacks,
        5530  uint32_t currentFrameIndex)
        5531 {
        5532  if(m_pDefragmentator == VMA_NULL)
        5533  {
        5534  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
        5535  hDevice,
        5536  pAllocationCallbacks,
        5537  this,
        5538  currentFrameIndex);
        5539  }
        5540 
        5541  return m_pDefragmentator;
        5542 }
        5543 
        5544 VkResult VmaBlockVector::Defragment(
        5545  VmaDefragmentationStats* pDefragmentationStats,
        5546  VkDeviceSize& maxBytesToMove,
        5547  uint32_t& maxAllocationsToMove)
        5548 {
        5549  if(m_pDefragmentator == VMA_NULL)
        5550  {
        5551  return VK_SUCCESS;
        5552  }
        5553 
        5554  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5555 
        5556  // Defragment.
        5557  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
        5558 
        5559  // Accumulate statistics.
        5560  if(pDefragmentationStats != VMA_NULL)
        5561  {
        5562  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
        5563  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
        5564  pDefragmentationStats->bytesMoved += bytesMoved;
        5565  pDefragmentationStats->allocationsMoved += allocationsMoved;
        5566  VMA_ASSERT(bytesMoved <= maxBytesToMove);
        5567  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
        5568  maxBytesToMove -= bytesMoved;
        5569  maxAllocationsToMove -= allocationsMoved;
        5570  }
        5571 
        5572  // Free empty blocks.
        5573  m_HasEmptyBlock = false;
        5574  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
        5575  {
        5576  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
        5577  if(pBlock->IsEmpty())
        5578  {
        5579  if(m_Blocks.size() > m_MinBlockCount)
        5580  {
        5581  if(pDefragmentationStats != VMA_NULL)
        5582  {
        5583  ++pDefragmentationStats->deviceMemoryBlocksFreed;
        5584  pDefragmentationStats->bytesFreed += pBlock->m_Size;
        5585  }
        5586 
        5587  VmaVectorRemove(m_Blocks, blockIndex);
        5588  pBlock->Destroy(m_hAllocator);
        5589  vma_delete(m_hAllocator, pBlock);
        5590  }
        5591  else
        5592  {
        5593  m_HasEmptyBlock = true;
        5594  }
        5595  }
        5596  }
        5597 
        5598  return result;
        5599 }
        5600 
        5601 void VmaBlockVector::DestroyDefragmentator()
        5602 {
        5603  if(m_pDefragmentator != VMA_NULL)
        5604  {
        5605  vma_delete(m_hAllocator, m_pDefragmentator);
        5606  m_pDefragmentator = VMA_NULL;
        5607  }
        5608 }
        5609 
        5610 void VmaBlockVector::MakePoolAllocationsLost(
        5611  uint32_t currentFrameIndex,
        5612  size_t* pLostAllocationCount)
        5613 {
        5614  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5615 
        5616  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
        5617  {
        5618  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
        5619  VMA_ASSERT(pBlock);
        5620  pBlock->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
        5621  }
        5622 }
        5623 
        5624 void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex)
        5625 {
        5626  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
        5627 
        5628  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
        5629  {
        5630  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
        5631  VMA_ASSERT(pBlock);
        5632  VMA_HEAVY_ASSERT(pBlock->Validate());
        5633  VmaStatInfo allocationStatInfo;
        5634  CalcAllocationStatInfo(allocationStatInfo, *pBlock);
        5635  VmaAddStatInfo(pStats->total, allocationStatInfo);
        5636  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
        5637  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
        5638  }
        5639 }
        5640 
        5642 // VmaDefragmentator members definition
        5643 
        5644 VmaDefragmentator::VmaDefragmentator(
        5645  VkDevice hDevice,
        5646  const VkAllocationCallbacks* pAllocationCallbacks,
        5647  VmaBlockVector* pBlockVector,
        5648  uint32_t currentFrameIndex) :
        5649  m_hDevice(hDevice),
        5650  m_pAllocationCallbacks(pAllocationCallbacks),
        5651  m_pBlockVector(pBlockVector),
        5652  m_CurrentFrameIndex(currentFrameIndex),
        5653  m_BytesMoved(0),
        5654  m_AllocationsMoved(0),
        5655  m_Allocations(VmaStlAllocator<AllocationInfo>(pAllocationCallbacks)),
        5656  m_Blocks(VmaStlAllocator<BlockInfo*>(pAllocationCallbacks))
        5657 {
        5658 }
        5659 
        5660 VmaDefragmentator::~VmaDefragmentator()
        5661 {
        5662  for(size_t i = m_Blocks.size(); i--; )
        5663  {
        5664  vma_delete(m_pAllocationCallbacks, m_Blocks[i]);
        5665  }
        5666 }
        5667 
        5668 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
        5669 {
        5670  AllocationInfo allocInfo;
        5671  allocInfo.m_hAllocation = hAlloc;
        5672  allocInfo.m_pChanged = pChanged;
        5673  m_Allocations.push_back(allocInfo);
        5674 }
        5675 
        5676 VkResult VmaDefragmentator::DefragmentRound(
        5677  VkDeviceSize maxBytesToMove,
        5678  uint32_t maxAllocationsToMove)
        5679 {
        5680  if(m_Blocks.empty())
        5681  {
        5682  return VK_SUCCESS;
        5683  }
        5684 
        5685  size_t srcBlockIndex = m_Blocks.size() - 1;
        5686  size_t srcAllocIndex = SIZE_MAX;
        5687  for(;;)
        5688  {
        5689  // 1. Find next allocation to move.
        5690  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
        5691  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
        5692  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
        5693  {
        5694  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
        5695  {
        5696  // Finished: no more allocations to process.
        5697  if(srcBlockIndex == 0)
        5698  {
        5699  return VK_SUCCESS;
        5700  }
        5701  else
        5702  {
        5703  --srcBlockIndex;
        5704  srcAllocIndex = SIZE_MAX;
        5705  }
        5706  }
        5707  else
        5708  {
        5709  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
        5710  }
        5711  }
        5712 
        5713  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
        5714  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
        5715 
        5716  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
        5717  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
        5718  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
        5719  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
        5720 
        5721  // 2. Try to find new place for this allocation in preceding or current block.
        5722  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
        5723  {
        5724  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
        5725  VmaAllocationRequest dstAllocRequest;
        5726  if(pDstBlockInfo->m_pBlock->CreateAllocationRequest(
        5727  m_CurrentFrameIndex,
        5728  m_pBlockVector->GetFrameInUseCount(),
        5729  m_pBlockVector->GetBufferImageGranularity(),
        5730  size,
        5731  alignment,
        5732  suballocType,
        5733  false, // canMakeOtherLost
        5734  &dstAllocRequest) &&
        5735  MoveMakesSense(
        5736  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
        5737  {
        5738  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
        5739 
        5740  // Reached limit on number of allocations or bytes to move.
        5741  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
        5742  (m_BytesMoved + size > maxBytesToMove))
        5743  {
        5744  return VK_INCOMPLETE;
        5745  }
        5746 
        5747  void* pDstMappedData = VMA_NULL;
        5748  VkResult res = pDstBlockInfo->EnsureMapping(m_hDevice, &pDstMappedData);
        5749  if(res != VK_SUCCESS)
        5750  {
        5751  return res;
        5752  }
        5753 
        5754  void* pSrcMappedData = VMA_NULL;
        5755  res = pSrcBlockInfo->EnsureMapping(m_hDevice, &pSrcMappedData);
        5756  if(res != VK_SUCCESS)
        5757  {
        5758  return res;
        5759  }
        5760 
        5761  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
        5762  memcpy(
        5763  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
        5764  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
        5765  size);
        5766 
        5767  pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
        5768  pSrcBlockInfo->m_pBlock->Free(allocInfo.m_hAllocation);
        5769 
        5770  allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
        5771 
        5772  if(allocInfo.m_pChanged != VMA_NULL)
        5773  {
        5774  *allocInfo.m_pChanged = VK_TRUE;
        5775  }
        5776 
        5777  ++m_AllocationsMoved;
        5778  m_BytesMoved += size;
        5779 
        5780  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
        5781 
        5782  break;
        5783  }
        5784  }
        5785 
        5786  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
        5787 
        5788  if(srcAllocIndex > 0)
        5789  {
        5790  --srcAllocIndex;
        5791  }
        5792  else
        5793  {
        5794  if(srcBlockIndex > 0)
        5795  {
        5796  --srcBlockIndex;
        5797  srcAllocIndex = SIZE_MAX;
        5798  }
        5799  else
        5800  {
        5801  return VK_SUCCESS;
        5802  }
        5803  }
        5804  }
        5805 }
        5806 
        5807 VkResult VmaDefragmentator::Defragment(
        5808  VkDeviceSize maxBytesToMove,
        5809  uint32_t maxAllocationsToMove)
        5810 {
        5811  if(m_Allocations.empty())
        5812  {
        5813  return VK_SUCCESS;
        5814  }
        5815 
        5816  // Create block info for each block.
        5817  const size_t blockCount = m_pBlockVector->m_Blocks.size();
        5818  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        5819  {
        5820  BlockInfo* pBlockInfo = vma_new(m_pAllocationCallbacks, BlockInfo)(m_pAllocationCallbacks);
        5821  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
        5822  m_Blocks.push_back(pBlockInfo);
        5823  }
        5824 
        5825  // Sort them by m_pBlock pointer value.
        5826  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
        5827 
        5828  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
        5829  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
        5830  {
        5831  AllocationInfo& allocInfo = m_Allocations[blockIndex];
        5832  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
        5833  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
        5834  {
        5835  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
        5836  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
        5837  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
        5838  {
        5839  (*it)->m_Allocations.push_back(allocInfo);
        5840  }
        5841  else
        5842  {
        5843  VMA_ASSERT(0);
        5844  }
        5845  }
        5846  }
        5847  m_Allocations.clear();
        5848 
        5849  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        5850  {
        5851  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
        5852  pBlockInfo->CalcHasNonMovableAllocations();
        5853  pBlockInfo->SortAllocationsBySizeDescecnding();
        5854  }
        5855 
        5856  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
        5857  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
        5858 
        5859  // Execute defragmentation rounds (the main part).
        5860  VkResult result = VK_SUCCESS;
        5861  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
        5862  {
        5863  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
        5864  }
        5865 
        5866  // Unmap blocks that were mapped for defragmentation.
        5867  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
        5868  {
        5869  m_Blocks[blockIndex]->Unmap(m_hDevice);
        5870  }
        5871 
        5872  return result;
        5873 }
        5874 
        5875 bool VmaDefragmentator::MoveMakesSense(
        5876  size_t dstBlockIndex, VkDeviceSize dstOffset,
        5877  size_t srcBlockIndex, VkDeviceSize srcOffset)
        5878 {
        5879  if(dstBlockIndex < srcBlockIndex)
        5880  {
        5881  return true;
        5882  }
        5883  if(dstBlockIndex > srcBlockIndex)
        5884  {
        5885  return false;
        5886  }
        5887  if(dstOffset < srcOffset)
        5888  {
        5889  return true;
        5890  }
        5891  return false;
        5892 }
        5893 
        5895 // VmaAllocator_T
        5896 
        5897 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
        5898  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
        5899  m_PhysicalDevice(pCreateInfo->physicalDevice),
        5900  m_hDevice(pCreateInfo->device),
        5901  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
        5902  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
        5903  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
        5904  m_UnmapPersistentlyMappedMemoryCounter(0),
        5905  m_PreferredLargeHeapBlockSize(0),
        5906  m_PreferredSmallHeapBlockSize(0),
        5907  m_CurrentFrameIndex(0),
        5908  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
        5909 {
        5910  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
        5911 
        5912  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
        5913  memset(&m_MemProps, 0, sizeof(m_MemProps));
        5914  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
        5915 
        5916  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
        5917  memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations));
        5918 
        5919  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
        5920  {
        5921  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
        5922  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
        5923  }
        5924 
        5925  vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties);
        5926  vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps);
        5927 
        5928  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
        5929  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
        5930  m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ?
        5931  pCreateInfo->preferredSmallHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE);
        5932 
        5933  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        5934  {
        5935  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
        5936 
        5937  for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex)
        5938  {
        5939  m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)(
        5940  this,
        5941  memTypeIndex,
        5942  static_cast<VMA_BLOCK_VECTOR_TYPE>(blockVectorTypeIndex),
        5943  preferredBlockSize,
        5944  0,
        5945  SIZE_MAX,
        5946  GetBufferImageGranularity(),
        5947  pCreateInfo->frameInUseCount,
        5948  false); // isCustomPool
        5949  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
        5950  // becase minBlockCount is 0.
        5951  m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
        5952  }
        5953  }
        5954 }
        5955 
        5956 VmaAllocator_T::~VmaAllocator_T()
        5957 {
        5958  VMA_ASSERT(m_Pools.empty());
        5959 
        5960  for(size_t i = GetMemoryTypeCount(); i--; )
        5961  {
        5962  for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; )
        5963  {
        5964  vma_delete(this, m_pOwnAllocations[i][j]);
        5965  vma_delete(this, m_pBlockVectors[i][j]);
        5966  }
        5967  }
        5968 }
        5969 
        5970 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
        5971 {
        5972  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size;
        5973  return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ?
        5974  m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize;
        5975 }
        5976 
        5977 VkResult VmaAllocator_T::AllocateMemoryOfType(
        5978  const VkMemoryRequirements& vkMemReq,
        5979  const VmaAllocationCreateInfo& createInfo,
        5980  uint32_t memTypeIndex,
        5981  VmaSuballocationType suballocType,
        5982  VmaAllocation* pAllocation)
        5983 {
        5984  VMA_ASSERT(pAllocation != VMA_NULL);
        5985  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
        5986 
        5987  uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags);
        5988  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        5989  VMA_ASSERT(blockVector);
        5990 
        5991  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
        5992  // Heuristics: Allocate own memory if requested size if greater than half of preferred block size.
        5993  const bool ownMemory =
        5994  (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 ||
        5995  VMA_DEBUG_ALWAYS_OWN_MEMORY ||
        5996  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
        5997  vkMemReq.size > preferredBlockSize / 2);
        5998 
        5999  if(ownMemory)
        6000  {
        6001  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
        6002  {
        6003  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        6004  }
        6005  else
        6006  {
        6007  return AllocateOwnMemory(
        6008  vkMemReq.size,
        6009  suballocType,
        6010  memTypeIndex,
        6011  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
        6012  createInfo.pUserData,
        6013  pAllocation);
        6014  }
        6015  }
        6016  else
        6017  {
        6018  VkResult res = blockVector->Allocate(
        6019  VK_NULL_HANDLE, // hCurrentPool
        6020  m_CurrentFrameIndex.load(),
        6021  vkMemReq,
        6022  createInfo,
        6023  suballocType,
        6024  pAllocation);
        6025  if(res == VK_SUCCESS)
        6026  {
        6027  return res;
        6028  }
        6029 
        6030  // 5. Try own memory.
        6031  res = AllocateOwnMemory(
        6032  vkMemReq.size,
        6033  suballocType,
        6034  memTypeIndex,
        6035  (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0,
        6036  createInfo.pUserData,
        6037  pAllocation);
        6038  if(res == VK_SUCCESS)
        6039  {
        6040  // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here.
        6041  VMA_DEBUG_LOG(" Allocated as OwnMemory");
        6042  return VK_SUCCESS;
        6043  }
        6044  else
        6045  {
        6046  // Everything failed: Return error code.
        6047  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
        6048  return res;
        6049  }
        6050  }
        6051 }
        6052 
        6053 VkResult VmaAllocator_T::AllocateOwnMemory(
        6054  VkDeviceSize size,
        6055  VmaSuballocationType suballocType,
        6056  uint32_t memTypeIndex,
        6057  bool map,
        6058  void* pUserData,
        6059  VmaAllocation* pAllocation)
        6060 {
        6061  VMA_ASSERT(pAllocation);
        6062 
        6063  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
        6064  allocInfo.memoryTypeIndex = memTypeIndex;
        6065  allocInfo.allocationSize = size;
        6066 
        6067  // Allocate VkDeviceMemory.
        6068  VkDeviceMemory hMemory = VK_NULL_HANDLE;
        6069  VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &hMemory);
        6070  if(res < 0)
        6071  {
        6072  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
        6073  return res;
        6074  }
        6075 
        6076  void* pMappedData = nullptr;
        6077  if(map)
        6078  {
        6079  if(m_UnmapPersistentlyMappedMemoryCounter == 0)
        6080  {
        6081  res = vkMapMemory(m_hDevice, hMemory, 0, VK_WHOLE_SIZE, 0, &pMappedData);
        6082  if(res < 0)
        6083  {
        6084  VMA_DEBUG_LOG(" vkMapMemory FAILED");
        6085  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
        6086  return res;
        6087  }
        6088  }
        6089  }
        6090 
        6091  // Callback.
        6092  if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
        6093  {
        6094  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, hMemory, size);
        6095  }
        6096 
        6097  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load());
        6098  (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData);
        6099 
        6100  // Register it in m_pOwnAllocations.
        6101  {
        6102  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        6103  AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED];
        6104  VMA_ASSERT(pOwnAllocations);
        6105  VmaVectorInsertSorted<VmaPointerLess>(*pOwnAllocations, *pAllocation);
        6106  }
        6107 
        6108  VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex);
        6109 
        6110  return VK_SUCCESS;
        6111 }
        6112 
        6113 VkResult VmaAllocator_T::AllocateMemory(
        6114  const VkMemoryRequirements& vkMemReq,
        6115  const VmaAllocationCreateInfo& createInfo,
        6116  VmaSuballocationType suballocType,
        6117  VmaAllocation* pAllocation)
        6118 {
        6119  if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 &&
        6120  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
        6121  {
        6122  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
        6123  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        6124  }
        6125  if((createInfo.pool != VK_NULL_HANDLE) &&
        6126  ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0))
        6127  {
        6128  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid.");
        6129  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        6130  }
        6131 
        6132  if(createInfo.pool != VK_NULL_HANDLE)
        6133  {
        6134  return createInfo.pool->m_BlockVector.Allocate(
        6135  createInfo.pool,
        6136  m_CurrentFrameIndex.load(),
        6137  vkMemReq,
        6138  createInfo,
        6139  suballocType,
        6140  pAllocation);
        6141  }
        6142  else
        6143  {
        6144  // Bit mask of memory Vulkan types acceptable for this allocation.
        6145  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
        6146  uint32_t memTypeIndex = UINT32_MAX;
        6147  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
        6148  if(res == VK_SUCCESS)
        6149  {
        6150  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
        6151  // Succeeded on first try.
        6152  if(res == VK_SUCCESS)
        6153  {
        6154  return res;
        6155  }
        6156  // Allocation from this memory type failed. Try other compatible memory types.
        6157  else
        6158  {
        6159  for(;;)
        6160  {
        6161  // Remove old memTypeIndex from list of possibilities.
        6162  memoryTypeBits &= ~(1u << memTypeIndex);
        6163  // Find alternative memTypeIndex.
        6164  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
        6165  if(res == VK_SUCCESS)
        6166  {
        6167  res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation);
        6168  // Allocation from this alternative memory type succeeded.
        6169  if(res == VK_SUCCESS)
        6170  {
        6171  return res;
        6172  }
        6173  // else: Allocation from this memory type failed. Try next one - next loop iteration.
        6174  }
        6175  // No other matching memory type index could be found.
        6176  else
        6177  {
        6178  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
        6179  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
        6180  }
        6181  }
        6182  }
        6183  }
        6184  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
        6185  else
        6186  return res;
        6187  }
        6188 }
        6189 
        6190 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
        6191 {
        6192  VMA_ASSERT(allocation);
        6193 
        6194  if(allocation->CanBecomeLost() == false ||
        6195  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
        6196  {
        6197  switch(allocation->GetType())
        6198  {
        6199  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
        6200  {
        6201  VmaBlockVector* pBlockVector = VMA_NULL;
        6202  VmaPool hPool = allocation->GetPool();
        6203  if(hPool != VK_NULL_HANDLE)
        6204  {
        6205  pBlockVector = &hPool->m_BlockVector;
        6206  }
        6207  else
        6208  {
        6209  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
        6210  const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType();
        6211  pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        6212  }
        6213  pBlockVector->Free(allocation);
        6214  }
        6215  break;
        6216  case VmaAllocation_T::ALLOCATION_TYPE_OWN:
        6217  FreeOwnMemory(allocation);
        6218  break;
        6219  default:
        6220  VMA_ASSERT(0);
        6221  }
        6222  }
        6223 
        6224  vma_delete(this, allocation);
        6225 }
        6226 
        6227 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
        6228 {
        6229  InitStatInfo(pStats->total);
        6230  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
        6231  InitStatInfo(pStats->memoryType[i]);
        6232  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
        6233  InitStatInfo(pStats->memoryHeap[i]);
        6234 
        6235  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        6236  {
        6237  const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex;
        6238  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        6239  {
        6240  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType];
        6241  VMA_ASSERT(pBlockVector);
        6242  pBlockVector->AddStats(pStats, memTypeIndex, heapIndex);
        6243  }
        6244  }
        6245 
        6246  VmaPostprocessCalcStatInfo(pStats->total);
        6247  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
        6248  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
        6249  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
        6250  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
        6251 }
        6252 
        6253 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
        6254 
        6255 void VmaAllocator_T::UnmapPersistentlyMappedMemory()
        6256 {
        6257  if(m_UnmapPersistentlyMappedMemoryCounter++ == 0)
        6258  {
        6259  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
        6260  {
        6261  for(size_t memTypeIndex = m_MemProps.memoryTypeCount; memTypeIndex--; )
        6262  {
        6263  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        6264  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
        6265  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        6266  {
        6267  // Process OwnAllocations.
        6268  {
        6269  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        6270  AllocationVectorType* pOwnAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        6271  for(size_t ownAllocIndex = pOwnAllocationsVector->size(); ownAllocIndex--; )
        6272  {
        6273  VmaAllocation hAlloc = (*pOwnAllocationsVector)[ownAllocIndex];
        6274  hAlloc->OwnAllocUnmapPersistentlyMappedMemory(m_hDevice);
        6275  }
        6276  }
        6277 
        6278  // Process normal Allocations.
        6279  {
        6280  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        6281  pBlockVector->UnmapPersistentlyMappedMemory();
        6282  }
        6283  }
        6284  }
        6285 
        6286  // Process custom pools.
        6287  {
        6288  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
        6289  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
        6290  {
        6291  m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory();
        6292  }
        6293  }
        6294  }
        6295  }
        6296 }
        6297 
        6298 VkResult VmaAllocator_T::MapPersistentlyMappedMemory()
        6299 {
        6300  VMA_ASSERT(m_UnmapPersistentlyMappedMemoryCounter > 0);
        6301  if(--m_UnmapPersistentlyMappedMemoryCounter == 0)
        6302  {
        6303  VkResult finalResult = VK_SUCCESS;
        6304  if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD)
        6305  {
        6306  // Process custom pools.
        6307  {
        6308  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
        6309  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
        6310  {
        6311  m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory();
        6312  }
        6313  }
        6314 
        6315  for(size_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex)
        6316  {
        6317  const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        6318  if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 &&
        6319  (memFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        6320  {
        6321  // Process OwnAllocations.
        6322  {
        6323  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        6324  AllocationVectorType* pAllocationsVector = m_pOwnAllocations[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        6325  for(size_t ownAllocIndex = 0, ownAllocCount = pAllocationsVector->size(); ownAllocIndex < ownAllocCount; ++ownAllocIndex)
        6326  {
        6327  VmaAllocation hAlloc = (*pAllocationsVector)[ownAllocIndex];
        6328  hAlloc->OwnAllocMapPersistentlyMappedMemory(m_hDevice);
        6329  }
        6330  }
        6331 
        6332  // Process normal Allocations.
        6333  {
        6334  VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED];
        6335  VkResult localResult = pBlockVector->MapPersistentlyMappedMemory();
        6336  if(localResult != VK_SUCCESS)
        6337  {
        6338  finalResult = localResult;
        6339  }
        6340  }
        6341  }
        6342  }
        6343  }
        6344  return finalResult;
        6345  }
        6346  else
        6347  return VK_SUCCESS;
        6348 }
        6349 
        6350 VkResult VmaAllocator_T::Defragment(
        6351  VmaAllocation* pAllocations,
        6352  size_t allocationCount,
        6353  VkBool32* pAllocationsChanged,
        6354  const VmaDefragmentationInfo* pDefragmentationInfo,
        6355  VmaDefragmentationStats* pDefragmentationStats)
        6356 {
        6357  if(pAllocationsChanged != VMA_NULL)
        6358  {
        6359  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
        6360  }
        6361  if(pDefragmentationStats != VMA_NULL)
        6362  {
        6363  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
        6364  }
        6365 
        6366  if(m_UnmapPersistentlyMappedMemoryCounter > 0)
        6367  {
        6368  VMA_DEBUG_LOG("ERROR: Cannot defragment when inside vmaUnmapPersistentlyMappedMemory.");
        6369  return VK_ERROR_MEMORY_MAP_FAILED;
        6370  }
        6371 
        6372  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
        6373 
        6374  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
        6375 
        6376  const size_t poolCount = m_Pools.size();
        6377 
        6378  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
        6379  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
        6380  {
        6381  VmaAllocation hAlloc = pAllocations[allocIndex];
        6382  VMA_ASSERT(hAlloc);
        6383  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
        6384  // OwnAlloc cannot be defragmented.
        6385  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
        6386  // Only HOST_VISIBLE memory types can be defragmented.
        6387  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
        6388  // Lost allocation cannot be defragmented.
        6389  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
        6390  {
        6391  VmaBlockVector* pAllocBlockVector = nullptr;
        6392 
        6393  const VmaPool hAllocPool = hAlloc->GetPool();
        6394  // This allocation belongs to custom pool.
        6395  if(hAllocPool != VK_NULL_HANDLE)
        6396  {
        6397  pAllocBlockVector = &hAllocPool->GetBlockVector();
        6398  }
        6399  // This allocation belongs to general pool.
        6400  else
        6401  {
        6402  pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()];
        6403  }
        6404 
        6405  VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(
        6406  m_hDevice,
        6407  GetAllocationCallbacks(),
        6408  currentFrameIndex);
        6409 
        6410  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
        6411  &pAllocationsChanged[allocIndex] : VMA_NULL;
        6412  pDefragmentator->AddAllocation(hAlloc, pChanged);
        6413  }
        6414  }
        6415 
        6416  VkResult result = VK_SUCCESS;
        6417 
        6418  // ======== Main processing.
        6419 
        6420  VkDeviceSize maxBytesToMove = SIZE_MAX;
        6421  uint32_t maxAllocationsToMove = UINT32_MAX;
        6422  if(pDefragmentationInfo != VMA_NULL)
        6423  {
        6424  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
        6425  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
        6426  }
        6427 
        6428  // Process standard memory.
        6429  for(uint32_t memTypeIndex = 0;
        6430  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
        6431  ++memTypeIndex)
        6432  {
        6433  // Only HOST_VISIBLE memory types can be defragmented.
        6434  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        6435  {
        6436  for(uint32_t blockVectorType = 0;
        6437  (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS);
        6438  ++blockVectorType)
        6439  {
        6440  result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment(
        6441  pDefragmentationStats,
        6442  maxBytesToMove,
        6443  maxAllocationsToMove);
        6444  }
        6445  }
        6446  }
        6447 
        6448  // Process custom pools.
        6449  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
        6450  {
        6451  result = m_Pools[poolIndex]->GetBlockVector().Defragment(
        6452  pDefragmentationStats,
        6453  maxBytesToMove,
        6454  maxAllocationsToMove);
        6455  }
        6456 
        6457  // ======== Destroy defragmentators.
        6458 
        6459  // Process custom pools.
        6460  for(size_t poolIndex = poolCount; poolIndex--; )
        6461  {
        6462  m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
        6463  }
        6464 
        6465  // Process standard memory.
        6466  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
        6467  {
        6468  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        6469  {
        6470  for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; )
        6471  {
        6472  m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator();
        6473  }
        6474  }
        6475  }
        6476 
        6477  return result;
        6478 }
        6479 
        6480 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
        6481 {
        6482  if(hAllocation->CanBecomeLost())
        6483  {
        6484  /*
        6485  Warning: This is a carefully designed algorithm.
        6486  Do not modify unless you really know what you're doing :)
        6487  */
        6488  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
        6489  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
        6490  for(;;)
        6491  {
        6492  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
        6493  {
        6494  pAllocationInfo->memoryType = UINT32_MAX;
        6495  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
        6496  pAllocationInfo->offset = 0;
        6497  pAllocationInfo->size = hAllocation->GetSize();
        6498  pAllocationInfo->pMappedData = VMA_NULL;
        6499  pAllocationInfo->pUserData = hAllocation->GetUserData();
        6500  return;
        6501  }
        6502  else if(localLastUseFrameIndex == localCurrFrameIndex)
        6503  {
        6504  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
        6505  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
        6506  pAllocationInfo->offset = hAllocation->GetOffset();
        6507  pAllocationInfo->size = hAllocation->GetSize();
        6508  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
        6509  pAllocationInfo->pUserData = hAllocation->GetUserData();
        6510  return;
        6511  }
        6512  else // Last use time earlier than current time.
        6513  {
        6514  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
        6515  {
        6516  localLastUseFrameIndex = localCurrFrameIndex;
        6517  }
        6518  }
        6519  }
        6520  }
        6521  // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic.
        6522  else
        6523  {
        6524  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
        6525  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
        6526  pAllocationInfo->offset = hAllocation->GetOffset();
        6527  pAllocationInfo->size = hAllocation->GetSize();
        6528  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
        6529  pAllocationInfo->pUserData = hAllocation->GetUserData();
        6530  }
        6531 }
        6532 
        6533 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
        6534 {
        6535  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
        6536 
        6537  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
        6538 
        6539  if(newCreateInfo.maxBlockCount == 0)
        6540  {
        6541  newCreateInfo.maxBlockCount = SIZE_MAX;
        6542  }
        6543  if(newCreateInfo.blockSize == 0)
        6544  {
        6545  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
        6546  }
        6547 
        6548  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
        6549 
        6550  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
        6551  if(res != VK_SUCCESS)
        6552  {
        6553  vma_delete(this, *pPool);
        6554  *pPool = VMA_NULL;
        6555  return res;
        6556  }
        6557 
        6558  // Add to m_Pools.
        6559  {
        6560  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
        6561  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
        6562  }
        6563 
        6564  return VK_SUCCESS;
        6565 }
        6566 
        6567 void VmaAllocator_T::DestroyPool(VmaPool pool)
        6568 {
        6569  // Remove from m_Pools.
        6570  {
        6571  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
        6572  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
        6573  VMA_ASSERT(success && "Pool not found in Allocator.");
        6574  }
        6575 
        6576  vma_delete(this, pool);
        6577 }
        6578 
        6579 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
        6580 {
        6581  pool->m_BlockVector.GetPoolStats(pPoolStats);
        6582 }
        6583 
        6584 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
        6585 {
        6586  m_CurrentFrameIndex.store(frameIndex);
        6587 }
        6588 
        6589 void VmaAllocator_T::MakePoolAllocationsLost(
        6590  VmaPool hPool,
        6591  size_t* pLostAllocationCount)
        6592 {
        6593  hPool->m_BlockVector.MakePoolAllocationsLost(
        6594  m_CurrentFrameIndex.load(),
        6595  pLostAllocationCount);
        6596 }
        6597 
        6598 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
        6599 {
        6600  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST);
        6601  (*pAllocation)->InitLost();
        6602 }
        6603 
        6604 void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation)
        6605 {
        6606  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_OWN);
        6607 
        6608  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
        6609  {
        6610  VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        6611  AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()];
        6612  VMA_ASSERT(pOwnAllocations);
        6613  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pOwnAllocations, allocation);
        6614  VMA_ASSERT(success);
        6615  }
        6616 
        6617  VkDeviceMemory hMemory = allocation->GetMemory();
        6618 
        6619  // Callback.
        6620  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
        6621  {
        6622  (*m_DeviceMemoryCallbacks.pfnFree)(this, memTypeIndex, hMemory, allocation->GetSize());
        6623  }
        6624 
        6625  if(allocation->GetMappedData() != VMA_NULL)
        6626  {
        6627  vkUnmapMemory(m_hDevice, hMemory);
        6628  }
        6629 
        6630  vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks());
        6631 
        6632  VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex);
        6633 }
        6634 
        6635 #if VMA_STATS_STRING_ENABLED
        6636 
        6637 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
        6638 {
        6639  bool ownAllocationsStarted = false;
        6640  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        6641  {
        6642  VmaMutexLock ownAllocationsLock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex);
        6643  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        6644  {
        6645  AllocationVectorType* const pOwnAllocVector = m_pOwnAllocations[memTypeIndex][blockVectorType];
        6646  VMA_ASSERT(pOwnAllocVector);
        6647  if(pOwnAllocVector->empty() == false)
        6648  {
        6649  if(ownAllocationsStarted == false)
        6650  {
        6651  ownAllocationsStarted = true;
        6652  json.WriteString("OwnAllocations");
        6653  json.BeginObject();
        6654  }
        6655 
        6656  json.BeginString("Type ");
        6657  json.ContinueString(memTypeIndex);
        6658  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
        6659  {
        6660  json.ContinueString(" Mapped");
        6661  }
        6662  json.EndString();
        6663 
        6664  json.BeginArray();
        6665 
        6666  for(size_t i = 0; i < pOwnAllocVector->size(); ++i)
        6667  {
        6668  const VmaAllocation hAlloc = (*pOwnAllocVector)[i];
        6669  json.BeginObject(true);
        6670 
        6671  json.WriteString("Size");
        6672  json.WriteNumber(hAlloc->GetSize());
        6673 
        6674  json.WriteString("Type");
        6675  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
        6676 
        6677  json.EndObject();
        6678  }
        6679 
        6680  json.EndArray();
        6681  }
        6682  }
        6683  }
        6684  if(ownAllocationsStarted)
        6685  {
        6686  json.EndObject();
        6687  }
        6688 
        6689  {
        6690  bool allocationsStarted = false;
        6691  for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
        6692  {
        6693  for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType)
        6694  {
        6695  if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false)
        6696  {
        6697  if(allocationsStarted == false)
        6698  {
        6699  allocationsStarted = true;
        6700  json.WriteString("Allocations");
        6701  json.BeginObject();
        6702  }
        6703 
        6704  json.BeginString("Type ");
        6705  json.ContinueString(memTypeIndex);
        6706  if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)
        6707  {
        6708  json.ContinueString(" Mapped");
        6709  }
        6710  json.EndString();
        6711 
        6712  m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json);
        6713  }
        6714  }
        6715  }
        6716  if(allocationsStarted)
        6717  {
        6718  json.EndObject();
        6719  }
        6720  }
        6721 
        6722  {
        6723  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
        6724  const size_t poolCount = m_Pools.size();
        6725  if(poolCount > 0)
        6726  {
        6727  json.WriteString("Pools");
        6728  json.BeginArray();
        6729  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
        6730  {
        6731  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
        6732  }
        6733  json.EndArray();
        6734  }
        6735  }
        6736 }
        6737 
        6738 #endif // #if VMA_STATS_STRING_ENABLED
        6739 
        6740 static VkResult AllocateMemoryForImage(
        6741  VmaAllocator allocator,
        6742  VkImage image,
        6743  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        6744  VmaSuballocationType suballocType,
        6745  VmaAllocation* pAllocation)
        6746 {
        6747  VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
        6748 
        6749  VkMemoryRequirements vkMemReq = {};
        6750  vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq);
        6751 
        6752  return allocator->AllocateMemory(
        6753  vkMemReq,
        6754  *pAllocationCreateInfo,
        6755  suballocType,
        6756  pAllocation);
        6757 }
        6758 
        6760 // Public interface
        6761 
        6762 VkResult vmaCreateAllocator(
        6763  const VmaAllocatorCreateInfo* pCreateInfo,
        6764  VmaAllocator* pAllocator)
        6765 {
        6766  VMA_ASSERT(pCreateInfo && pAllocator);
        6767  VMA_DEBUG_LOG("vmaCreateAllocator");
        6768  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
        6769  return VK_SUCCESS;
        6770 }
        6771 
        6772 void vmaDestroyAllocator(
        6773  VmaAllocator allocator)
        6774 {
        6775  if(allocator != VK_NULL_HANDLE)
        6776  {
        6777  VMA_DEBUG_LOG("vmaDestroyAllocator");
        6778  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
        6779  vma_delete(&allocationCallbacks, allocator);
        6780  }
        6781 }
        6782 
        6783 void vmaGetPhysicalDeviceProperties(
        6784  VmaAllocator allocator,
        6785  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
        6786 {
        6787  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
        6788  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
        6789 }
        6790 
        6791 void vmaGetMemoryProperties(
        6792  VmaAllocator allocator,
        6793  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
        6794 {
        6795  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
        6796  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
        6797 }
        6798 
        6799 void vmaGetMemoryTypeProperties(
        6800  VmaAllocator allocator,
        6801  uint32_t memoryTypeIndex,
        6802  VkMemoryPropertyFlags* pFlags)
        6803 {
        6804  VMA_ASSERT(allocator && pFlags);
        6805  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
        6806  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
        6807 }
        6808 
        6809 void vmaSetCurrentFrameIndex(
        6810  VmaAllocator allocator,
        6811  uint32_t frameIndex)
        6812 {
        6813  VMA_ASSERT(allocator);
        6814  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
        6815 
        6816  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        6817 
        6818  allocator->SetCurrentFrameIndex(frameIndex);
        6819 }
        6820 
        6821 void vmaCalculateStats(
        6822  VmaAllocator allocator,
        6823  VmaStats* pStats)
        6824 {
        6825  VMA_ASSERT(allocator && pStats);
        6826  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        6827  allocator->CalculateStats(pStats);
        6828 }
        6829 
        6830 #if VMA_STATS_STRING_ENABLED
        6831 
        6832 void vmaBuildStatsString(
        6833  VmaAllocator allocator,
        6834  char** ppStatsString,
        6835  VkBool32 detailedMap)
        6836 {
        6837  VMA_ASSERT(allocator && ppStatsString);
        6838  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        6839 
        6840  VmaStringBuilder sb(allocator);
        6841  {
        6842  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
        6843  json.BeginObject();
        6844 
        6845  VmaStats stats;
        6846  allocator->CalculateStats(&stats);
        6847 
        6848  json.WriteString("Total");
        6849  VmaPrintStatInfo(json, stats.total);
        6850 
        6851  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
        6852  {
        6853  json.BeginString("Heap ");
        6854  json.ContinueString(heapIndex);
        6855  json.EndString();
        6856  json.BeginObject();
        6857 
        6858  json.WriteString("Size");
        6859  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
        6860 
        6861  json.WriteString("Flags");
        6862  json.BeginArray(true);
        6863  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
        6864  {
        6865  json.WriteString("DEVICE_LOCAL");
        6866  }
        6867  json.EndArray();
        6868 
        6869  if(stats.memoryHeap[heapIndex].BlockCount > 0)
        6870  {
        6871  json.WriteString("Stats");
        6872  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
        6873  }
        6874 
        6875  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
        6876  {
        6877  if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex)
        6878  {
        6879  json.BeginString("Type ");
        6880  json.ContinueString(typeIndex);
        6881  json.EndString();
        6882 
        6883  json.BeginObject();
        6884 
        6885  json.WriteString("Flags");
        6886  json.BeginArray(true);
        6887  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
        6888  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
        6889  {
        6890  json.WriteString("DEVICE_LOCAL");
        6891  }
        6892  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
        6893  {
        6894  json.WriteString("HOST_VISIBLE");
        6895  }
        6896  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
        6897  {
        6898  json.WriteString("HOST_COHERENT");
        6899  }
        6900  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
        6901  {
        6902  json.WriteString("HOST_CACHED");
        6903  }
        6904  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
        6905  {
        6906  json.WriteString("LAZILY_ALLOCATED");
        6907  }
        6908  json.EndArray();
        6909 
        6910  if(stats.memoryType[typeIndex].BlockCount > 0)
        6911  {
        6912  json.WriteString("Stats");
        6913  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
        6914  }
        6915 
        6916  json.EndObject();
        6917  }
        6918  }
        6919 
        6920  json.EndObject();
        6921  }
        6922  if(detailedMap == VK_TRUE)
        6923  {
        6924  allocator->PrintDetailedMap(json);
        6925  }
        6926 
        6927  json.EndObject();
        6928  }
        6929 
        6930  const size_t len = sb.GetLength();
        6931  char* const pChars = vma_new_array(allocator, char, len + 1);
        6932  if(len > 0)
        6933  {
        6934  memcpy(pChars, sb.GetData(), len);
        6935  }
        6936  pChars[len] = '\0';
        6937  *ppStatsString = pChars;
        6938 }
        6939 
        6940 void vmaFreeStatsString(
        6941  VmaAllocator allocator,
        6942  char* pStatsString)
        6943 {
        6944  if(pStatsString != VMA_NULL)
        6945  {
        6946  VMA_ASSERT(allocator);
        6947  size_t len = strlen(pStatsString);
        6948  vma_delete_array(allocator, pStatsString, len + 1);
        6949  }
        6950 }
        6951 
        6952 #endif // #if VMA_STATS_STRING_ENABLED
        6953 
        6956 VkResult vmaFindMemoryTypeIndex(
        6957  VmaAllocator allocator,
        6958  uint32_t memoryTypeBits,
        6959  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        6960  uint32_t* pMemoryTypeIndex)
        6961 {
        6962  VMA_ASSERT(allocator != VK_NULL_HANDLE);
        6963  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
        6964  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
        6965 
        6966  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
        6967  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
        6968  if(preferredFlags == 0)
        6969  {
        6970  preferredFlags = requiredFlags;
        6971  }
        6972  // preferredFlags, if not 0, must be a superset of requiredFlags.
        6973  VMA_ASSERT((requiredFlags & ~preferredFlags) == 0);
        6974 
        6975  // Convert usage to requiredFlags and preferredFlags.
        6976  switch(pAllocationCreateInfo->usage)
        6977  {
        6978  case VMA_MEMORY_USAGE_UNKNOWN:
        6979  break;
        6980  case VMA_MEMORY_USAGE_GPU_ONLY:
        6981  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
        6982  break;
        6983  case VMA_MEMORY_USAGE_CPU_ONLY:
        6984  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
        6985  break;
        6986  case VMA_MEMORY_USAGE_CPU_TO_GPU:
        6987  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        6988  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
        6989  break;
        6990  case VMA_MEMORY_USAGE_GPU_TO_CPU:
        6991  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        6992  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
        6993  break;
        6994  default:
        6995  break;
        6996  }
        6997 
        6998  if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0)
        6999  {
        7000  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
        7001  }
        7002 
        7003  *pMemoryTypeIndex = UINT32_MAX;
        7004  uint32_t minCost = UINT32_MAX;
        7005  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
        7006  memTypeIndex < allocator->GetMemoryTypeCount();
        7007  ++memTypeIndex, memTypeBit <<= 1)
        7008  {
        7009  // This memory type is acceptable according to memoryTypeBits bitmask.
        7010  if((memTypeBit & memoryTypeBits) != 0)
        7011  {
        7012  const VkMemoryPropertyFlags currFlags =
        7013  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
        7014  // This memory type contains requiredFlags.
        7015  if((requiredFlags & ~currFlags) == 0)
        7016  {
        7017  // Calculate cost as number of bits from preferredFlags not present in this memory type.
        7018  uint32_t currCost = CountBitsSet(preferredFlags & ~currFlags);
        7019  // Remember memory type with lowest cost.
        7020  if(currCost < minCost)
        7021  {
        7022  *pMemoryTypeIndex = memTypeIndex;
        7023  if(currCost == 0)
        7024  {
        7025  return VK_SUCCESS;
        7026  }
        7027  minCost = currCost;
        7028  }
        7029  }
        7030  }
        7031  }
        7032  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
        7033 }
        7034 
        7035 VkResult vmaCreatePool(
        7036  VmaAllocator allocator,
        7037  const VmaPoolCreateInfo* pCreateInfo,
        7038  VmaPool* pPool)
        7039 {
        7040  VMA_ASSERT(allocator && pCreateInfo && pPool);
        7041 
        7042  VMA_DEBUG_LOG("vmaCreatePool");
        7043 
        7044  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7045 
        7046  return allocator->CreatePool(pCreateInfo, pPool);
        7047 }
        7048 
        7049 void vmaDestroyPool(
        7050  VmaAllocator allocator,
        7051  VmaPool pool)
        7052 {
        7053  VMA_ASSERT(allocator && pool);
        7054 
        7055  VMA_DEBUG_LOG("vmaDestroyPool");
        7056 
        7057  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7058 
        7059  allocator->DestroyPool(pool);
        7060 }
        7061 
        7062 void vmaGetPoolStats(
        7063  VmaAllocator allocator,
        7064  VmaPool pool,
        7065  VmaPoolStats* pPoolStats)
        7066 {
        7067  VMA_ASSERT(allocator && pool && pPoolStats);
        7068 
        7069  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7070 
        7071  allocator->GetPoolStats(pool, pPoolStats);
        7072 }
        7073 
        7074 void vmaMakePoolAllocationsLost(
        7075  VmaAllocator allocator,
        7076  VmaPool pool,
        7077  size_t* pLostAllocationCount)
        7078 {
        7079  VMA_ASSERT(allocator && pool);
        7080 
        7081  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7082 
        7083  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
        7084 }
        7085 
        7086 VkResult vmaAllocateMemory(
        7087  VmaAllocator allocator,
        7088  const VkMemoryRequirements* pVkMemoryRequirements,
        7089  const VmaAllocationCreateInfo* pCreateInfo,
        7090  VmaAllocation* pAllocation,
        7091  VmaAllocationInfo* pAllocationInfo)
        7092 {
        7093  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
        7094 
        7095  VMA_DEBUG_LOG("vmaAllocateMemory");
        7096 
        7097  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7098 
        7099  VkResult result = allocator->AllocateMemory(
        7100  *pVkMemoryRequirements,
        7101  *pCreateInfo,
        7102  VMA_SUBALLOCATION_TYPE_UNKNOWN,
        7103  pAllocation);
        7104 
        7105  if(pAllocationInfo && result == VK_SUCCESS)
        7106  {
        7107  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        7108  }
        7109 
        7110  return result;
        7111 }
        7112 
        7113 VkResult vmaAllocateMemoryForBuffer(
        7114  VmaAllocator allocator,
        7115  VkBuffer buffer,
        7116  const VmaAllocationCreateInfo* pCreateInfo,
        7117  VmaAllocation* pAllocation,
        7118  VmaAllocationInfo* pAllocationInfo)
        7119 {
        7120  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
        7121 
        7122  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
        7123 
        7124  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7125 
        7126  VkMemoryRequirements vkMemReq = {};
        7127  vkGetBufferMemoryRequirements(allocator->m_hDevice, buffer, &vkMemReq);
        7128 
        7129  VkResult result = allocator->AllocateMemory(
        7130  vkMemReq,
        7131  *pCreateInfo,
        7132  VMA_SUBALLOCATION_TYPE_BUFFER,
        7133  pAllocation);
        7134 
        7135  if(pAllocationInfo && result == VK_SUCCESS)
        7136  {
        7137  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        7138  }
        7139 
        7140  return result;
        7141 }
        7142 
        7143 VkResult vmaAllocateMemoryForImage(
        7144  VmaAllocator allocator,
        7145  VkImage image,
        7146  const VmaAllocationCreateInfo* pCreateInfo,
        7147  VmaAllocation* pAllocation,
        7148  VmaAllocationInfo* pAllocationInfo)
        7149 {
        7150  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
        7151 
        7152  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
        7153 
        7154  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7155 
        7156  VkResult result = AllocateMemoryForImage(
        7157  allocator,
        7158  image,
        7159  pCreateInfo,
        7160  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
        7161  pAllocation);
        7162 
        7163  if(pAllocationInfo && result == VK_SUCCESS)
        7164  {
        7165  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        7166  }
        7167 
        7168  return result;
        7169 }
        7170 
        7171 void vmaFreeMemory(
        7172  VmaAllocator allocator,
        7173  VmaAllocation allocation)
        7174 {
        7175  VMA_ASSERT(allocator && allocation);
        7176 
        7177  VMA_DEBUG_LOG("vmaFreeMemory");
        7178 
        7179  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7180 
        7181  allocator->FreeMemory(allocation);
        7182 }
        7183 
        7184 void vmaGetAllocationInfo(
        7185  VmaAllocator allocator,
        7186  VmaAllocation allocation,
        7187  VmaAllocationInfo* pAllocationInfo)
        7188 {
        7189  VMA_ASSERT(allocator && allocation && pAllocationInfo);
        7190 
        7191  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7192 
        7193  allocator->GetAllocationInfo(allocation, pAllocationInfo);
        7194 }
        7195 
        7196 void vmaSetAllocationUserData(
        7197  VmaAllocator allocator,
        7198  VmaAllocation allocation,
        7199  void* pUserData)
        7200 {
        7201  VMA_ASSERT(allocator && allocation);
        7202 
        7203  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7204 
        7205  allocation->SetUserData(pUserData);
        7206 }
        7207 
        7208 void vmaCreateLostAllocation(
        7209  VmaAllocator allocator,
        7210  VmaAllocation* pAllocation)
        7211 {
        7212  VMA_ASSERT(allocator && pAllocation);
        7213 
        7214  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
        7215 
        7216  allocator->CreateLostAllocation(pAllocation);
        7217 }
        7218 
        7219 VkResult vmaMapMemory(
        7220  VmaAllocator allocator,
        7221  VmaAllocation allocation,
        7222  void** ppData)
        7223 {
        7224  VMA_ASSERT(allocator && allocation && ppData);
        7225 
        7226  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7227 
        7228  return vkMapMemory(allocator->m_hDevice, allocation->GetMemory(),
        7229  allocation->GetOffset(), allocation->GetSize(), 0, ppData);
        7230 }
        7231 
        7232 void vmaUnmapMemory(
        7233  VmaAllocator allocator,
        7234  VmaAllocation allocation)
        7235 {
        7236  VMA_ASSERT(allocator && allocation);
        7237 
        7238  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7239 
        7240  vkUnmapMemory(allocator->m_hDevice, allocation->GetMemory());
        7241 }
        7242 
        7243 void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
        7244 {
        7245  VMA_ASSERT(allocator);
        7246 
        7247  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7248 
        7249  allocator->UnmapPersistentlyMappedMemory();
        7250 }
        7251 
        7252 VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
        7253 {
        7254  VMA_ASSERT(allocator);
        7255 
        7256  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7257 
        7258  return allocator->MapPersistentlyMappedMemory();
        7259 }
        7260 
        7261 VkResult vmaDefragment(
        7262  VmaAllocator allocator,
        7263  VmaAllocation* pAllocations,
        7264  size_t allocationCount,
        7265  VkBool32* pAllocationsChanged,
        7266  const VmaDefragmentationInfo *pDefragmentationInfo,
        7267  VmaDefragmentationStats* pDefragmentationStats)
        7268 {
        7269  VMA_ASSERT(allocator && pAllocations);
        7270 
        7271  VMA_DEBUG_LOG("vmaDefragment");
        7272 
        7273  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7274 
        7275  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
        7276 }
        7277 
        7278 VkResult vmaCreateBuffer(
        7279  VmaAllocator allocator,
        7280  const VkBufferCreateInfo* pBufferCreateInfo,
        7281  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        7282  VkBuffer* pBuffer,
        7283  VmaAllocation* pAllocation,
        7284  VmaAllocationInfo* pAllocationInfo)
        7285 {
        7286  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
        7287 
        7288  VMA_DEBUG_LOG("vmaCreateBuffer");
        7289 
        7290  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7291 
        7292  *pBuffer = VK_NULL_HANDLE;
        7293  *pAllocation = VK_NULL_HANDLE;
        7294 
        7295  // 1. Create VkBuffer.
        7296  VkResult res = vkCreateBuffer(allocator->m_hDevice, pBufferCreateInfo, allocator->GetAllocationCallbacks(), pBuffer);
        7297  if(res >= 0)
        7298  {
        7299  // 2. vkGetBufferMemoryRequirements.
        7300  VkMemoryRequirements vkMemReq = {};
        7301  vkGetBufferMemoryRequirements(allocator->m_hDevice, *pBuffer, &vkMemReq);
        7302 
        7303  // 3. Allocate memory using allocator.
        7304  res = allocator->AllocateMemory(
        7305  vkMemReq,
        7306  *pAllocationCreateInfo,
        7307  VMA_SUBALLOCATION_TYPE_BUFFER,
        7308  pAllocation);
        7309  if(res >= 0)
        7310  {
        7311  // 3. Bind buffer with memory.
        7312  res = vkBindBufferMemory(allocator->m_hDevice, *pBuffer, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
        7313  if(res >= 0)
        7314  {
        7315  // All steps succeeded.
        7316  if(pAllocationInfo != VMA_NULL)
        7317  {
        7318  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        7319  }
        7320  return VK_SUCCESS;
        7321  }
        7322  allocator->FreeMemory(*pAllocation);
        7323  *pAllocation = VK_NULL_HANDLE;
        7324  return res;
        7325  }
        7326  vkDestroyBuffer(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
        7327  *pBuffer = VK_NULL_HANDLE;
        7328  return res;
        7329  }
        7330  return res;
        7331 }
        7332 
        7333 void vmaDestroyBuffer(
        7334  VmaAllocator allocator,
        7335  VkBuffer buffer,
        7336  VmaAllocation allocation)
        7337 {
        7338  if(buffer != VK_NULL_HANDLE)
        7339  {
        7340  VMA_ASSERT(allocator);
        7341 
        7342  VMA_DEBUG_LOG("vmaDestroyBuffer");
        7343 
        7344  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7345 
        7346  vkDestroyBuffer(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
        7347 
        7348  allocator->FreeMemory(allocation);
        7349  }
        7350 }
        7351 
        7352 VkResult vmaCreateImage(
        7353  VmaAllocator allocator,
        7354  const VkImageCreateInfo* pImageCreateInfo,
        7355  const VmaAllocationCreateInfo* pAllocationCreateInfo,
        7356  VkImage* pImage,
        7357  VmaAllocation* pAllocation,
        7358  VmaAllocationInfo* pAllocationInfo)
        7359 {
        7360  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
        7361 
        7362  VMA_DEBUG_LOG("vmaCreateImage");
        7363 
        7364  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7365 
        7366  *pImage = VK_NULL_HANDLE;
        7367  *pAllocation = VK_NULL_HANDLE;
        7368 
        7369  // 1. Create VkImage.
        7370  VkResult res = vkCreateImage(allocator->m_hDevice, pImageCreateInfo, allocator->GetAllocationCallbacks(), pImage);
        7371  if(res >= 0)
        7372  {
        7373  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
        7374  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
        7375  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
        7376 
        7377  // 2. Allocate memory using allocator.
        7378  res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
        7379  if(res >= 0)
        7380  {
        7381  // 3. Bind image with memory.
        7382  res = vkBindImageMemory(allocator->m_hDevice, *pImage, (*pAllocation)->GetMemory(), (*pAllocation)->GetOffset());
        7383  if(res >= 0)
        7384  {
        7385  // All steps succeeded.
        7386  if(pAllocationInfo != VMA_NULL)
        7387  {
        7388  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
        7389  }
        7390  return VK_SUCCESS;
        7391  }
        7392  allocator->FreeMemory(*pAllocation);
        7393  *pAllocation = VK_NULL_HANDLE;
        7394  return res;
        7395  }
        7396  vkDestroyImage(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
        7397  *pImage = VK_NULL_HANDLE;
        7398  return res;
        7399  }
        7400  return res;
        7401 }
        7402 
        7403 void vmaDestroyImage(
        7404  VmaAllocator allocator,
        7405  VkImage image,
        7406  VmaAllocation allocation)
        7407 {
        7408  if(image != VK_NULL_HANDLE)
        7409  {
        7410  VMA_ASSERT(allocator);
        7411 
        7412  VMA_DEBUG_LOG("vmaDestroyImage");
        7413 
        7414  VMA_DEBUG_GLOBAL_MUTEX_LOCK
        7415 
        7416  vkDestroyImage(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
        7417 
        7418  allocator->FreeMemory(allocation);
        7419  }
        7420 }
        7421 
        7422 #endif // #ifdef VMA_IMPLEMENTATION
        VmaAllocatorCreateInfo::physicalDevice
        VkPhysicalDevice physicalDevice
        Vulkan physical device.
        Definition: vk_mem_alloc.h:457
        +
        VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM
        Definition: vk_mem_alloc.h:756
        +
        vmaGetPoolStats
        void vmaGetPoolStats(VmaAllocator allocator, VmaPool pool, VmaPoolStats *pPoolStats)
        Retrieves statistics of existing VmaPool object.
        +
        VmaStatInfo::BlockCount
        uint32_t BlockCount
        Number of VkDeviceMemory Vulkan memory blocks allocated.
        Definition: vk_mem_alloc.h:540
        +
        VMA_MEMORY_USAGE_GPU_TO_CPU
        Memory will be used for frequent writing on device and readback on host (download).
        Definition: vk_mem_alloc.h:607
        +
        vmaFindMemoryTypeIndex
        VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaAllocationCreateInfo *pAllocationCreateInfo, uint32_t *pMemoryTypeIndex)
        +
        VmaAllocationInfo::deviceMemory
        VkDeviceMemory deviceMemory
        Handle to Vulkan memory object.
        Definition: vk_mem_alloc.h:877
        +
        VmaDefragmentationInfo::maxAllocationsToMove
        uint32_t maxAllocationsToMove
        Maximum number of allocations that can be moved to different place.
        Definition: vk_mem_alloc.h:1027
        +
        vmaCreateImage
        VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pImageCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        Function similar to vmaCreateBuffer().
        vmaGetAllocationInfo
        void vmaGetAllocationInfo(VmaAllocator allocator, VmaAllocation allocation, VmaAllocationInfo *pAllocationInfo)
        Returns current information about specified allocation.
        -
        vmaUnmapPersistentlyMappedMemory
        void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
        Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.
        +
        vmaUnmapPersistentlyMappedMemory
        void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator)
        Unmaps persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
        vmaDestroyImage
        void vmaDestroyImage(VmaAllocator allocator, VkImage image, VmaAllocation allocation)
        +
        VmaPoolStats::size
        VkDeviceSize size
        Total amount of VkDeviceMemory allocated from Vulkan for this pool, in bytes.
        Definition: vk_mem_alloc.h:808
        VmaDefragmentationInfo
        struct VmaDefragmentationInfo VmaDefragmentationInfo
        Optional configuration parameters to be passed to function vmaDefragment().
        -
        PFN_vmaFreeDeviceMemoryFunction
        void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
        Callback function called before vkFreeMemory.
        Definition: vk_mem_alloc.h:177
        -
        VmaAllocatorCreateInfo::pAllocationCallbacks
        const VkAllocationCallbacks * pAllocationCallbacks
        Custom allocation callbacks.
        Definition: vk_mem_alloc.h:226
        -
        vmaAllocateMemoryForBuffer
        VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        -
        VmaAllocatorCreateInfo
        Description of a Allocator to be created.
        Definition: vk_mem_alloc.h:208
        -
        VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
        VkDeviceSize preferredSmallHeapBlockSize
        Size of a single memory block to allocate for resources from a small heap <= 512 MB.
        Definition: vk_mem_alloc.h:223
        -
        VmaMemoryRequirements::flags
        VmaMemoryRequirementFlags flags
        Definition: vk_mem_alloc.h:369
        -
        VmaAllocatorFlags
        VkFlags VmaAllocatorFlags
        Definition: vk_mem_alloc.h:205
        -
        VmaDefragmentationStats
        Statistics returned by function vmaDefragment().
        Definition: vk_mem_alloc.h:571
        -
        VmaStats::total
        VmaStatInfo total
        Definition: vk_mem_alloc.h:284
        -
        VmaDefragmentationStats::deviceMemoryBlocksFreed
        uint32_t deviceMemoryBlocksFreed
        Number of empty VkDeviceMemory objects that have been released to the system.
        Definition: vk_mem_alloc.h:579
        -
        VmaDefragmentationInfo::maxBytesToMove
        VkDeviceSize maxBytesToMove
        Maximum total numbers of bytes that can be copied while moving allocations to different places...
        Definition: vk_mem_alloc.h:562
        -
        VmaAllocatorCreateInfo::device
        VkDevice device
        Vulkan device.
        Definition: vk_mem_alloc.h:217
        -
        VmaAllocationInfo::size
        VkDeviceSize size
        Size of this allocation, in bytes.
        Definition: vk_mem_alloc.h:441
        +
        VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
        Definition: vk_mem_alloc.h:656
        +
        VmaAllocationCreateInfo::preferredFlags
        VkMemoryPropertyFlags preferredFlags
        Flags that preferably should be set in a Memory Type chosen for an allocation.
        Definition: vk_mem_alloc.h:689
        +
        PFN_vmaFreeDeviceMemoryFunction
        void(VKAPI_PTR * PFN_vmaFreeDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
        Callback function called before vkFreeMemory.
        Definition: vk_mem_alloc.h:420
        +
        vmaMakePoolAllocationsLost
        void vmaMakePoolAllocationsLost(VmaAllocator allocator, VmaPool pool, size_t *pLostAllocationCount)
        Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInf...
        +
        VmaAllocatorCreateInfo::pAllocationCallbacks
        const VkAllocationCallbacks * pAllocationCallbacks
        Custom CPU memory allocation callbacks.
        Definition: vk_mem_alloc.h:469
        +
        VmaPoolCreateFlags
        VkFlags VmaPoolCreateFlags
        Definition: vk_mem_alloc.h:758
        +
        VmaAllocatorCreateInfo
        Description of a Allocator to be created.
        Definition: vk_mem_alloc.h:451
        +
        VmaAllocatorCreateInfo::preferredSmallHeapBlockSize
        VkDeviceSize preferredSmallHeapBlockSize
        Size of a single memory block to allocate for resources from a small heap <= 512 MB.
        Definition: vk_mem_alloc.h:466
        +
        VmaAllocatorFlags
        VkFlags VmaAllocatorFlags
        Definition: vk_mem_alloc.h:448
        +
        VmaDefragmentationStats
        Statistics returned by function vmaDefragment().
        Definition: vk_mem_alloc.h:1031
        +
        VmaAllocatorCreateInfo::frameInUseCount
        uint32_t frameInUseCount
        Maximum number of additional frames that are in use at the same time as current frame.
        Definition: vk_mem_alloc.h:486
        +
        VmaStats::total
        VmaStatInfo total
        Definition: vk_mem_alloc.h:558
        +
        VmaDefragmentationStats::deviceMemoryBlocksFreed
        uint32_t deviceMemoryBlocksFreed
        Number of empty VkDeviceMemory objects that have been released to the system.
        Definition: vk_mem_alloc.h:1039
        +
        VmaAllocationCreateInfo::flags
        VmaAllocationCreateFlags flags
        Use VmaAllocationCreateFlagBits enum.
        Definition: vk_mem_alloc.h:672
        +
        VmaDefragmentationInfo::maxBytesToMove
        VkDeviceSize maxBytesToMove
        Maximum total numbers of bytes that can be copied while moving allocations to different places...
        Definition: vk_mem_alloc.h:1022
        +
        vmaAllocateMemoryForBuffer
        VkResult vmaAllocateMemoryForBuffer(VmaAllocator allocator, VkBuffer buffer, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        +
        VmaAllocatorCreateInfo::device
        VkDevice device
        Vulkan device.
        Definition: vk_mem_alloc.h:460
        +
        VmaPoolCreateInfo
        Describes parameter of created VmaPool.
        Definition: vk_mem_alloc.h:762
        +
        VmaPoolStats
        struct VmaPoolStats VmaPoolStats
        Describes parameter of existing VmaPool.
        +
        VmaAllocationInfo::size
        VkDeviceSize size
        Size of this allocation, in bytes.
        Definition: vk_mem_alloc.h:887
        vmaFreeMemory
        void vmaFreeMemory(VmaAllocator allocator, VmaAllocation allocation)
        Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
        -
        VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT
        Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
        Definition: vk_mem_alloc.h:354
        +
        vmaCreateBuffer
        VkResult vmaCreateBuffer(VmaAllocator allocator, const VkBufferCreateInfo *pBufferCreateInfo, const VmaAllocationCreateInfo *pAllocationCreateInfo, VkBuffer *pBuffer, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        +
        vmaAllocateMemory
        VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        General purpose memory allocation.
        +
        VmaAllocationCreateInfo::pUserData
        void * pUserData
        Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
        Definition: vk_mem_alloc.h:691
        +
        VmaPoolCreateInfo::minBlockCount
        size_t minBlockCount
        Minimum number of blocks to be always allocated in this pool, even if they stay empty.
        Definition: vk_mem_alloc.h:778
        +
        VmaPoolStats::allocationCount
        size_t allocationCount
        Number of VmaAllocation objects created from this pool that were not destroyed or lost...
        Definition: vk_mem_alloc.h:814
        +
        VmaPoolCreateInfo::memoryTypeIndex
        uint32_t memoryTypeIndex
        Vulkan memory type index to allocate this pool from.
        Definition: vk_mem_alloc.h:765
        vmaBuildStatsString
        void vmaBuildStatsString(VmaAllocator allocator, char **ppStatsString, VkBool32 detailedMap)
        Builds and returns statistics as string in JSON format.
        -
        VmaDefragmentationInfo
        Optional configuration parameters to be passed to function vmaDefragment().
        Definition: vk_mem_alloc.h:557
        -
        VmaDefragmentationStats::bytesFreed
        VkDeviceSize bytesFreed
        Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
        Definition: vk_mem_alloc.h:575
        -
        VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM
        Definition: vk_mem_alloc.h:363
        -
        VmaMemoryRequirements::pUserData
        void * pUserData
        Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo...
        Definition: vk_mem_alloc.h:385
        -
        VmaStats
        General statistics from current state of Allocator.
        Definition: vk_mem_alloc.h:280
        +
        VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM
        Definition: vk_mem_alloc.h:665
        +
        VmaDefragmentationInfo
        Optional configuration parameters to be passed to function vmaDefragment().
        Definition: vk_mem_alloc.h:1017
        +
        vmaCreatePool
        VkResult vmaCreatePool(VmaAllocator allocator, const VmaPoolCreateInfo *pCreateInfo, VmaPool *pPool)
        Allocates Vulkan device memory and creates VmaPool object.
        +
        VmaStatInfo::AllocationSizeMax
        VkDeviceSize AllocationSizeMax
        Definition: vk_mem_alloc.h:549
        +
        VMA_POOL_CREATE_PERSISTENT_MAP_BIT
        Definition: vk_mem_alloc.h:736
        +
        VmaDefragmentationStats::bytesFreed
        VkDeviceSize bytesFreed
        Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects...
        Definition: vk_mem_alloc.h:1035
        +
        vmaSetCurrentFrameIndex
        void vmaSetCurrentFrameIndex(VmaAllocator allocator, uint32_t frameIndex)
        Sets index of the current frame.
        +
        VmaStats
        General statistics from current state of Allocator.
        Definition: vk_mem_alloc.h:554
        vmaCreateAllocator
        VkResult vmaCreateAllocator(const VmaAllocatorCreateInfo *pCreateInfo, VmaAllocator *pAllocator)
        Creates Allocator object.
        -
        VmaMemoryRequirements::preferredFlags
        VkMemoryPropertyFlags preferredFlags
        Flags that preferably should be set in a Memory Type chosen for an allocation.
        Definition: vk_mem_alloc.h:383
        -
        VmaDefragmentationStats::allocationsMoved
        uint32_t allocationsMoved
        Number of allocations that have been moved to different places.
        Definition: vk_mem_alloc.h:577
        -
        VmaMemoryUsage
        VmaMemoryUsage
        Definition: vk_mem_alloc.h:317
        +
        vmaAllocateMemoryForImage
        VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaAllocationCreateInfo *pCreateInfo, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        Function similar to vmaAllocateMemoryForBuffer().
        +
        VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT
        Set this flag to use a memory that will be persistently mapped and retrieve pointer to it...
        Definition: vk_mem_alloc.h:645
        +
        VmaDefragmentationStats::allocationsMoved
        uint32_t allocationsMoved
        Number of allocations that have been moved to different places.
        Definition: vk_mem_alloc.h:1037
        +
        VmaMemoryUsage
        VmaMemoryUsage
        Definition: vk_mem_alloc.h:593
        vmaDestroyAllocator
        void vmaDestroyAllocator(VmaAllocator allocator)
        Destroys allocator object.
        -
        VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
        Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
        Definition: vk_mem_alloc.h:201
        +
        VmaAllocationCreateInfo::requiredFlags
        VkMemoryPropertyFlags requiredFlags
        Flags that must be set in a Memory Type chosen for an allocation.
        Definition: vk_mem_alloc.h:683
        +
        VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT
        Allocator and all objects created from it will not be synchronized internally, so you must guarantee ...
        Definition: vk_mem_alloc.h:444
        vmaCalculateStats
        void vmaCalculateStats(VmaAllocator allocator, VmaStats *pStats)
        Retrieves statistics from current state of the Allocator.
        -
        VmaAllocatorFlagBits
        VmaAllocatorFlagBits
        Flags for created VmaAllocator.
        Definition: vk_mem_alloc.h:196
        +
        VmaAllocatorFlagBits
        VmaAllocatorFlagBits
        Flags for created VmaAllocator.
        Definition: vk_mem_alloc.h:439
        vmaSetAllocationUserData
        void vmaSetAllocationUserData(VmaAllocator allocator, VmaAllocation allocation, void *pUserData)
        Sets pUserData in given allocation to new value.
        -
        VmaStatInfo
        Definition: vk_mem_alloc.h:268
        -
        VmaDeviceMemoryCallbacks
        Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
        Definition: vk_mem_alloc.h:188
        -
        VmaMemoryRequirements::requiredFlags
        VkMemoryPropertyFlags requiredFlags
        Flags that must be set in a Memory Type chosen for an allocation.
        Definition: vk_mem_alloc.h:378
        -
        VmaMemoryRequirements
        Definition: vk_mem_alloc.h:367
        -
        VmaDeviceMemoryCallbacks::pfnFree
        PFN_vmaFreeDeviceMemoryFunction pfnFree
        Optional, can be null.
        Definition: vk_mem_alloc.h:192
        -
        vmaMapPersistentlyMappedMemory
        VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
        Maps back persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL.
        -
        VmaMemoryRequirementFlags
        VkFlags VmaMemoryRequirementFlags
        Definition: vk_mem_alloc.h:365
        -
        VmaStatInfo::UnusedRangeSizeMax
        VkDeviceSize UnusedRangeSizeMax
        Definition: vk_mem_alloc.h:276
        -
        VmaStatInfo::SuballocationSizeMax
        VkDeviceSize SuballocationSizeMax
        Definition: vk_mem_alloc.h:275
        +
        VmaStatInfo
        Definition: vk_mem_alloc.h:537
        +
        VmaPoolCreateInfo::blockSize
        VkDeviceSize blockSize
        Size of a single VkDeviceMemory block to be allocated as part of this pool, in bytes.
        Definition: vk_mem_alloc.h:773
        +
        VmaDeviceMemoryCallbacks
        Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
        Definition: vk_mem_alloc.h:431
        +
        VmaDeviceMemoryCallbacks::pfnFree
        PFN_vmaFreeDeviceMemoryFunction pfnFree
        Optional, can be null.
        Definition: vk_mem_alloc.h:435
        +
        vmaMapPersistentlyMappedMemory
        VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator)
        Maps back persistently mapped memory of types that are HOST_COHERENT and DEVICE_LOCAL.
        +
        VmaPoolCreateInfo::flags
        VmaPoolCreateFlags flags
        Use combination of VmaPoolCreateFlagBits.
        Definition: vk_mem_alloc.h:768
        +
        VmaStatInfo::UnusedRangeSizeMax
        VkDeviceSize UnusedRangeSizeMax
        Definition: vk_mem_alloc.h:550
        VmaAllocatorCreateInfo
        struct VmaAllocatorCreateInfo VmaAllocatorCreateInfo
        Description of a Allocator to be created.
        -
        PFN_vmaAllocateDeviceMemoryFunction
        void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
        Callback function called after successful vkAllocateMemory.
        Definition: vk_mem_alloc.h:171
        -
        vmaAllocateMemory
        VkResult vmaAllocateMemory(VmaAllocator allocator, const VkMemoryRequirements *pVkMemoryRequirements, const VmaMemoryRequirements *pVmaMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        General purpose memory allocation.
        -
        VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
        const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
        Informative callbacks for vkAllocateMemory, vkFreeMemory.
        Definition: vk_mem_alloc.h:229
        -
        VmaStats::memoryType
        VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
        Definition: vk_mem_alloc.h:282
        -
        VmaStatInfo::AllocationCount
        uint32_t AllocationCount
        Definition: vk_mem_alloc.h:270
        +
        PFN_vmaAllocateDeviceMemoryFunction
        void(VKAPI_PTR * PFN_vmaAllocateDeviceMemoryFunction)(VmaAllocator allocator, uint32_t memoryType, VkDeviceMemory memory, VkDeviceSize size)
        Callback function called after successful vkAllocateMemory.
        Definition: vk_mem_alloc.h:414
        +
        VmaAllocationCreateInfo::usage
        VmaMemoryUsage usage
        Intended usage of memory.
        Definition: vk_mem_alloc.h:678
        +
        VmaAllocationCreateInfo
        Definition: vk_mem_alloc.h:669
        +
        VmaPoolCreateInfo::maxBlockCount
        size_t maxBlockCount
        Maximum number of blocks that can be allocated in this pool.
        Definition: vk_mem_alloc.h:786
        +
        VmaStatInfo::AllocationSizeMin
        VkDeviceSize AllocationSizeMin
        Definition: vk_mem_alloc.h:549
        +
        VmaAllocatorCreateInfo::pDeviceMemoryCallbacks
        const VmaDeviceMemoryCallbacks * pDeviceMemoryCallbacks
        Informative callbacks for vkAllocateMemory, vkFreeMemory.
        Definition: vk_mem_alloc.h:472
        +
        VmaPoolStats::unusedRangeCount
        size_t unusedRangeCount
        Number of continuous memory ranges in the pool not used by any VmaAllocation.
        Definition: vk_mem_alloc.h:817
        +
        VmaAllocationCreateInfo::pool
        VmaPool pool
        Pool that this allocation should be created in.
        Definition: vk_mem_alloc.h:696
        +
        VmaStats::memoryType
        VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES]
        Definition: vk_mem_alloc.h:556
        +
        VmaStatInfo::AllocationSizeAvg
        VkDeviceSize AllocationSizeAvg
        Definition: vk_mem_alloc.h:549
        +
        VmaAllocationCreateInfo
        struct VmaAllocationCreateInfo VmaAllocationCreateInfo
        +
        VmaStatInfo::AllocationCount
        uint32_t AllocationCount
        Number of VmaAllocation allocation objects allocated.
        Definition: vk_mem_alloc.h:542
        vmaMapMemory
        VkResult vmaMapMemory(VmaAllocator allocator, VmaAllocation allocation, void **ppData)
        -
        VmaDeviceMemoryCallbacks::pfnAllocate
        PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
        Optional, can be null.
        Definition: vk_mem_alloc.h:190
        -
        VmaAllocatorCreateInfo::flags
        VmaAllocatorFlags flags
        Flags for created allocator. Use VmaAllocatorFlagBits enum.
        Definition: vk_mem_alloc.h:211
        +
        VmaDeviceMemoryCallbacks::pfnAllocate
        PFN_vmaAllocateDeviceMemoryFunction pfnAllocate
        Optional, can be null.
        Definition: vk_mem_alloc.h:433
        +
        VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
        Definition: vk_mem_alloc.h:663
        +
        VmaPoolCreateInfo::frameInUseCount
        uint32_t frameInUseCount
        Maximum number of additional frames that are in use at the same time as current frame.
        Definition: vk_mem_alloc.h:800
        +
        VmaAllocatorCreateInfo::flags
        VmaAllocatorFlags flags
        Flags for created allocator. Use VmaAllocatorFlagBits enum.
        Definition: vk_mem_alloc.h:454
        vmaGetPhysicalDeviceProperties
        void vmaGetPhysicalDeviceProperties(VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
        -
        VmaStatInfo::UsedBytes
        VkDeviceSize UsedBytes
        Definition: vk_mem_alloc.h:273
        -
        VmaAllocationInfo::pUserData
        void * pUserData
        Custom general-purpose pointer that was passed as VmaMemoryRequirements::pUserData or set using vmaSe...
        Definition: vk_mem_alloc.h:452
        -
        VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
        VkDeviceSize preferredLargeHeapBlockSize
        Size of a single memory block to allocate for resources.
        Definition: vk_mem_alloc.h:220
        -
        vmaAllocateMemoryForImage
        VkResult vmaAllocateMemoryForImage(VmaAllocator allocator, VkImage image, const VmaMemoryRequirements *pMemoryRequirements, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        Function similar to vmaAllocateMemoryForBuffer().
        -
        VmaStatInfo::UnusedRangeCount
        uint32_t UnusedRangeCount
        Definition: vk_mem_alloc.h:272
        -
        VMA_MEMORY_USAGE_CPU_ONLY
        Memory will be mapped on host. Could be used for transfer to device.
        Definition: vk_mem_alloc.h:325
        +
        VmaStatInfo::UsedBytes
        VkDeviceSize UsedBytes
        Total number of bytes occupied by all allocations.
        Definition: vk_mem_alloc.h:546
        +
        VmaAllocationInfo::pUserData
        void * pUserData
        Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vma...
        Definition: vk_mem_alloc.h:898
        +
        VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT
        Set this flag if the allocation should have its own memory block.
        Definition: vk_mem_alloc.h:624
        +
        VmaAllocatorCreateInfo::preferredLargeHeapBlockSize
        VkDeviceSize preferredLargeHeapBlockSize
        Size of a single memory block to allocate for resources.
        Definition: vk_mem_alloc.h:463
        +
        VmaStatInfo::UnusedRangeCount
        uint32_t UnusedRangeCount
        Number of free ranges of memory between allocations.
        Definition: vk_mem_alloc.h:544
        +
        VmaPoolStats
        Describes parameter of existing VmaPool.
        Definition: vk_mem_alloc.h:805
        +
        VMA_MEMORY_USAGE_CPU_ONLY
        Memory will be mapped on host. Could be used for transfer to/from device.
        Definition: vk_mem_alloc.h:601
        vmaGetMemoryProperties
        void vmaGetMemoryProperties(VmaAllocator allocator, const VkPhysicalDeviceMemoryProperties **ppPhysicalDeviceMemoryProperties)
        -
        VmaStatInfo::SuballocationCount
        uint32_t SuballocationCount
        Definition: vk_mem_alloc.h:271
        -
        VmaStatInfo::UnusedRangeSizeAvg
        VkDeviceSize UnusedRangeSizeAvg
        Definition: vk_mem_alloc.h:276
        -
        VmaAllocationInfo::offset
        VkDeviceSize offset
        Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
        Definition: vk_mem_alloc.h:436
        -
        VmaDefragmentationStats::bytesMoved
        VkDeviceSize bytesMoved
        Total number of bytes that have been copied while moving allocations to different places...
        Definition: vk_mem_alloc.h:573
        +
        VmaStatInfo::UnusedRangeSizeAvg
        VkDeviceSize UnusedRangeSizeAvg
        Definition: vk_mem_alloc.h:550
        +
        VmaAllocationInfo::offset
        VkDeviceSize offset
        Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
        Definition: vk_mem_alloc.h:882
        +
        VmaDefragmentationStats::bytesMoved
        VkDeviceSize bytesMoved
        Total number of bytes that have been copied while moving allocations to different places...
        Definition: vk_mem_alloc.h:1033
        vmaDefragment
        VkResult vmaDefragment(VmaAllocator allocator, VmaAllocation *pAllocations, size_t allocationCount, VkBool32 *pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, VmaDefragmentationStats *pDefragmentationStats)
        Compacts memory by moving allocations.
        VmaDeviceMemoryCallbacks
        struct VmaDeviceMemoryCallbacks VmaDeviceMemoryCallbacks
        Set of callbacks that the library will call for vkAllocateMemory and vkFreeMemory.
        -
        VmaStatInfo::SuballocationSizeMin
        VkDeviceSize SuballocationSizeMin
        Definition: vk_mem_alloc.h:275
        -
        vmaCreateImage
        VkResult vmaCreateImage(VmaAllocator allocator, const VkImageCreateInfo *pCreateInfo, const VmaMemoryRequirements *pMemoryRequirements, VkImage *pImage, VmaAllocation *pAllocation, VmaAllocationInfo *pAllocationInfo)
        Function similar to vmaCreateBuffer().
        -
        VmaAllocationInfo::pMappedData
        void * pMappedData
        Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
        Definition: vk_mem_alloc.h:447
        -
        VmaStatInfo::SuballocationSizeAvg
        VkDeviceSize SuballocationSizeAvg
        Definition: vk_mem_alloc.h:275
        +
        VmaAllocationCreateFlags
        VkFlags VmaAllocationCreateFlags
        Definition: vk_mem_alloc.h:667
        +
        VmaPoolCreateFlagBits
        VmaPoolCreateFlagBits
        Flags to be passed as VmaPoolCreateInfo::flags.
        Definition: vk_mem_alloc.h:727
        +
        VmaAllocationInfo::pMappedData
        void * pMappedData
        Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistent...
        Definition: vk_mem_alloc.h:893
        vmaFreeStatsString
        void vmaFreeStatsString(VmaAllocator allocator, char *pStatsString)
        -
        VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT
        Set to use a memory that will be persistently mapped and retrieve pointer to it.
        Definition: vk_mem_alloc.h:361
        -
        VMA_MEMORY_USAGE_UNKNOWN
        No intended memory usage specified.
        Definition: vk_mem_alloc.h:320
        -
        VMA_MEMORY_USAGE_MAX_ENUM
        Definition: vk_mem_alloc.h:332
        -
        VmaAllocationInfo
        Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
        Definition: vk_mem_alloc.h:419
        -
        VMA_MEMORY_USAGE_CPU_TO_GPU
        Memory will be used for frequent (dynamic) updates from host and reads on device. ...
        Definition: vk_mem_alloc.h:328
        -
        VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM
        Definition: vk_mem_alloc.h:203
        +
        VMA_MEMORY_USAGE_UNKNOWN
        No intended memory usage specified.
        Definition: vk_mem_alloc.h:596
        +
        vmaCreateLostAllocation
        void vmaCreateLostAllocation(VmaAllocator allocator, VmaAllocation *pAllocation)
        Creates new allocation that is in lost state from the beginning.
        +
        VMA_MEMORY_USAGE_MAX_ENUM
        Definition: vk_mem_alloc.h:608
        +
        VmaAllocationInfo
        Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
        Definition: vk_mem_alloc.h:863
        +
        VMA_MEMORY_USAGE_CPU_TO_GPU
        Memory will be used for frequent (dynamic) updates from host and reads on device (upload).
        Definition: vk_mem_alloc.h:604
        +
        VmaAllocationCreateFlagBits
        VmaAllocationCreateFlagBits
        Flags to be passed as VmaAllocationCreateInfo::flags.
        Definition: vk_mem_alloc.h:612
        +
        VMA_ALLOCATOR_FLAG_BITS_MAX_ENUM
        Definition: vk_mem_alloc.h:446
        VmaAllocationInfo
        struct VmaAllocationInfo VmaAllocationInfo
        Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
        vmaGetMemoryTypeProperties
        void vmaGetMemoryTypeProperties(VmaAllocator allocator, uint32_t memoryTypeIndex, VkMemoryPropertyFlags *pFlags)
        Given Memory Type Index, returns Property Flags of this memory type.
        -
        VMA_MEMORY_USAGE_GPU_ONLY
        Memory will be used on device only, no need to be mapped on host.
        Definition: vk_mem_alloc.h:322
        +
        VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT
        Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such b...
        Definition: vk_mem_alloc.h:635
        +
        VMA_MEMORY_USAGE_GPU_ONLY
        Memory will be used on device only, so faster access from the device is preferred. No need to be mappable on host.
        Definition: vk_mem_alloc.h:598
        VmaStatInfo
        struct VmaStatInfo VmaStatInfo
        -
        VmaStatInfo::UnusedBytes
        VkDeviceSize UnusedBytes
        Definition: vk_mem_alloc.h:274
        +
        VmaStatInfo::UnusedBytes
        VkDeviceSize UnusedBytes
        Total number of bytes occupied by unused ranges.
        Definition: vk_mem_alloc.h:548
        vmaUnmapMemory
        void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
        -
        VmaStats::memoryHeap
        VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
        Definition: vk_mem_alloc.h:283
        -
        vmaFindMemoryTypeIndex
        VkResult vmaFindMemoryTypeIndex(VmaAllocator allocator, uint32_t memoryTypeBits, const VmaMemoryRequirements *pMemoryRequirements, uint32_t *pMemoryTypeIndex)
        +
        VmaStats::memoryHeap
        VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS]
        Definition: vk_mem_alloc.h:557
        VmaDefragmentationStats
        struct VmaDefragmentationStats VmaDefragmentationStats
        Statistics returned by function vmaDefragment().
        +
        vmaDestroyPool
        void vmaDestroyPool(VmaAllocator allocator, VmaPool pool)
        Destroys VmaPool object and frees Vulkan device memory.
        +
        VmaPoolStats::unusedSize
        VkDeviceSize unusedSize
        Total number of bytes in the pool not used by any VmaAllocation.
        Definition: vk_mem_alloc.h:811
        +
        VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT
        Use this flag if you always allocate only buffers and linear images or only optimal images out of thi...
        Definition: vk_mem_alloc.h:754
        vmaDestroyBuffer
        void vmaDestroyBuffer(VmaAllocator allocator, VkBuffer buffer, VmaAllocation allocation)
        -
        VmaStatInfo::UnusedRangeSizeMin
        VkDeviceSize UnusedRangeSizeMin
        Definition: vk_mem_alloc.h:276
        -
        VmaAllocationInfo::memoryType
        uint32_t memoryType
        Memory type index that this allocation was allocated from.
        Definition: vk_mem_alloc.h:424
        +
        VmaStatInfo::UnusedRangeSizeMin
        VkDeviceSize UnusedRangeSizeMin
        Definition: vk_mem_alloc.h:550
        +
        VmaAllocationInfo::memoryType
        uint32_t memoryType
        Memory type index that this allocation was allocated from.
        Definition: vk_mem_alloc.h:868
        +
        VmaPoolCreateInfo
        struct VmaPoolCreateInfo VmaPoolCreateInfo
        Describes parameter of created VmaPool.
        diff --git a/src/VulkanSample.cpp b/src/VulkanSample.cpp index f06f49f..b399385 100644 --- a/src/VulkanSample.cpp +++ b/src/VulkanSample.cpp @@ -32,6 +32,7 @@ #pragma warning(push, 4) #pragma warning(disable: 4127) // warning C4127: conditional expression is constant #pragma warning(disable: 4100) // warning C4100: '...': unreferenced formal parameter +#pragma warning(disable: 4189) // warning C4189: '...': local variable is initialized but not referenced #define VMA_IMPLEMENTATION #include "vk_mem_alloc.h" #pragma warning(pop) @@ -298,23 +299,23 @@ static void CreateMesh() vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; vbInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - VmaMemoryRequirements vbMemReq = {}; - vbMemReq.usage = VMA_MEMORY_USAGE_CPU_ONLY; - vbMemReq.flags = VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT; + VmaAllocationCreateInfo vbAllocCreateInfo = {}; + vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + vbAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT; VkBuffer stagingVertexBuffer = VK_NULL_HANDLE; VmaAllocation stagingVertexBufferAlloc = VK_NULL_HANDLE; VmaAllocationInfo stagingVertexBufferAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbMemReq, &stagingVertexBuffer, &stagingVertexBufferAlloc, &stagingVertexBufferAllocInfo) ); + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &stagingVertexBuffer, &stagingVertexBufferAlloc, &stagingVertexBufferAllocInfo) ); memcpy(stagingVertexBufferAllocInfo.pMappedData, vertices, vertexBufferSize); // No need to flush stagingVertexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. vbInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; - vbMemReq.usage = VMA_MEMORY_USAGE_GPU_ONLY; - vbMemReq.flags = 0; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbMemReq, &g_hVertexBuffer, &g_hVertexBufferAlloc, nullptr) ); + vbAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + vbAllocCreateInfo.flags = 0; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &vbInfo, &vbAllocCreateInfo, &g_hVertexBuffer, &g_hVertexBufferAlloc, nullptr) ); // Create index buffer @@ -323,23 +324,23 @@ static void CreateMesh() ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; ibInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - VmaMemoryRequirements ibMemReq = {}; - ibMemReq.usage = VMA_MEMORY_USAGE_CPU_ONLY; - ibMemReq.flags = VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT; + VmaAllocationCreateInfo ibAllocCreateInfo = {}; + ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + ibAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT; VkBuffer stagingIndexBuffer = VK_NULL_HANDLE; VmaAllocation stagingIndexBufferAlloc = VK_NULL_HANDLE; VmaAllocationInfo stagingIndexBufferAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibMemReq, &stagingIndexBuffer, &stagingIndexBufferAlloc, &stagingIndexBufferAllocInfo) ); + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &stagingIndexBuffer, &stagingIndexBufferAlloc, &stagingIndexBufferAllocInfo) ); memcpy(stagingIndexBufferAllocInfo.pMappedData, indices, indexBufferSize); // No need to flush stagingIndexBuffer memory because CPU_ONLY memory is always HOST_COHERENT. ibInfo.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; - ibMemReq.usage = VMA_MEMORY_USAGE_GPU_ONLY; - ibMemReq.flags = 0; - ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibMemReq, &g_hIndexBuffer, &g_hIndexBufferAlloc, nullptr) ); + ibAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + ibAllocCreateInfo.flags = 0; + ERR_GUARD_VULKAN( vmaCreateBuffer(g_hAllocator, &ibInfo, &ibAllocCreateInfo, &g_hIndexBuffer, &g_hIndexBufferAlloc, nullptr) ); // Copy buffers @@ -423,14 +424,14 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) stagingImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; stagingImageInfo.flags = 0; - VmaMemoryRequirements stagingImageMemReq = {}; - stagingImageMemReq.usage = VMA_MEMORY_USAGE_CPU_ONLY; - stagingImageMemReq.flags = VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT; + VmaAllocationCreateInfo stagingImageAllocCreateInfo = {}; + stagingImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + stagingImageAllocCreateInfo.flags = VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT; VkImage stagingImage = VK_NULL_HANDLE; VmaAllocation stagingImageAlloc = VK_NULL_HANDLE; VmaAllocationInfo stagingImageAllocInfo = {}; - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &stagingImageInfo, &stagingImageMemReq, &stagingImage, &stagingImageAlloc, &stagingImageAllocInfo) ); + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &stagingImageInfo, &stagingImageAllocCreateInfo, &stagingImage, &stagingImageAlloc, &stagingImageAllocInfo) ); VkImageSubresource imageSubresource = {}; imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; @@ -476,10 +477,10 @@ static void CreateTexture(uint32_t sizeX, uint32_t sizeY) imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; imageInfo.flags = 0; - VmaMemoryRequirements imageMemReq = {}; - imageMemReq.usage = VMA_MEMORY_USAGE_GPU_ONLY; + VmaAllocationCreateInfo imageAllocCreateInfo = {}; + imageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &imageInfo, &imageMemReq, &g_hTextureImage, &g_hTextureImageAlloc, nullptr) ); + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &imageInfo, &imageAllocCreateInfo, &g_hTextureImage, &g_hTextureImageAlloc, nullptr) ); // Transition image layouts, copy image. @@ -775,10 +776,10 @@ static void CreateSwapchain() depthImageInfo.samples = VK_SAMPLE_COUNT_1_BIT; depthImageInfo.flags = 0; - VmaMemoryRequirements depthImageMemReq = {}; - depthImageMemReq.usage = VMA_MEMORY_USAGE_GPU_ONLY; + VmaAllocationCreateInfo depthImageAllocCreateInfo = {}; + depthImageAllocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; - ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &depthImageInfo, &depthImageMemReq, &g_hDepthImage, &g_hDepthImageAlloc, nullptr) ); + ERR_GUARD_VULKAN( vmaCreateImage(g_hAllocator, &depthImageInfo, &depthImageAllocCreateInfo, &g_hDepthImage, &g_hDepthImageAlloc, nullptr) ); VkImageViewCreateInfo depthImageViewInfo = { VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO }; depthImageViewInfo.image = g_hDepthImage; diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index 3acfbf8..c6dab4b 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -25,13 +25,15 @@ /** \mainpage Vulkan Memory Allocator -Version 2.0.0-alpha.2 (2017-07-11) +\tableofcontents + +Version 2.0.0-alpha.3 (2017-09-12) Members grouped: see Modules. All members: see vk_mem_alloc.h. -\section problem Problem Statement +\section problem Problem statement Memory allocation and resource (buffer and image) creation in Vulkan is difficult (comparing to older graphics API-s, like D3D11 or OpenGL) for several @@ -39,8 +41,8 @@ reasons: - It requires a lot of boilerplate code, just like everything else in Vulkan, because it is a low-level and high-performance API. -- There is additional level of indirection: VkDeviceMemory is allocated - separately from creating VkBuffer/VkImage and they must be bound together. The +- There is additional level of indirection: `VkDeviceMemory` is allocated + separately from creating `VkBuffer`/`VkImage` and they must be bound together. The binding cannot be changed later - resource must be recreated. - Driver must be queried for supported memory heaps and memory types. Different IHV-s provide different types of it. @@ -58,7 +60,7 @@ be divided into several layers, low level to high level: - Required or preferred traits of the memory are expressed using higher-level description comparing to Vulkan flags. -# Functions that allocate memory blocks, reserve and return parts of them - (VkDeviceMemory + offset + size) to the user. + (`VkDeviceMemory` + offset + size) to the user. - Library keeps track of allocated memory blocks, used and unused ranges inside them, finds best matching unused ranges for new allocations, takes all the rules of alignment into consideration. @@ -73,10 +75,12 @@ be divided into several layers, low level to high level: C++. - Interface documented using Doxygen-style comments. - Platform-independent, but developed and tested on Windows using Visual Studio. -- Error handling implemented by returning VkResult error codes - same way as in +- Error handling implemented by returning `VkResult` error codes - same way as in Vulkan. -\section quick_start Quick Start +\section user_guide User guide + +\subsection quick_start Quick start In your project code: @@ -90,8 +94,8 @@ In your project code: At program startup: --# Initialize Vulkan to have VkPhysicalDevice and VkDevice object. --# Fill VmaAllocatorCreateInfo structure and create VmaAllocator object by +-# Initialize Vulkan to have `VkPhysicalDevice` and `VkDevice` object. +-# Fill VmaAllocatorCreateInfo structure and create `VmaAllocator` object by calling vmaCreateAllocator(). @@ -104,9 +108,9 @@ At program startup: When you want to create a buffer or image: --# Fill VkBufferCreateInfo / VkImageCreateInfo structure. --# Fill VmaMemoryRequirements structure. --# Call vmaCreateBuffer() / vmaCreateImage() to get VkBuffer/VkImage with memory +-# Fill `VkBufferCreateInfo` / `VkImageCreateInfo` structure. +-# Fill VmaAllocationCreateInfo structure. +-# Call vmaCreateBuffer() / vmaCreateImage() to get `VkBuffer`/`VkImage` with memory already allocated and bound to it. @@ -114,28 +118,267 @@ When you want to create a buffer or image: bufferInfo.size = myBufferSize; bufferInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; - VmaMemoryRequirements memReq = {}; + VmaAllocationCreateInfo memReq = {}; memReq.usage = VMA_MEMORY_USAGE_GPU_ONLY; VkBuffer buffer; VmaAllocation allocation; vmaCreateBuffer(allocator, &bufferInfo, &memReq, &buffer, &allocation, nullptr); -Don't forget to destroy your objects: +Don't forget to destroy your objects when no longer needed: vmaDestroyBuffer(allocator, buffer, allocation); vmaDestroyAllocator(allocator); +\subsection persistently_mapped_memory Persistently mapped memory + +If you need to map memory on host, it may happen that two allocations are +assigned to the same `VkDeviceMemory` block, so if you map them both at the same +time, it will cause error because mapping single memory block multiple times is +illegal in Vulkan. + +It is safer, more convenient and more efficient to use special feature designed +for that: persistently mapped memory. Allocations made with +`VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT` flag set in +VmaAllocationCreateInfo::flags are returned from device memory +blocks that stay mapped all the time, so you can just access CPU pointer to it. +VmaAllocationInfo::pMappedData pointer is already offseted to the beginning of +particular allocation. Example: + + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 1024; + bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT; + + VkBuffer buf; + VmaAllocation alloc; + VmaAllocationInfo allocInfo; + vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); + + .// Buffer is immediately mapped. You can access its memory. + memcpy(allocInfo.pMappedData, myData, 1024); + +Memory in Vulkan doesn't need to be unmapped before using it e.g. for transfers, +but if you are not sure whether it's `HOST_COHERENT` (here is surely is because +it's created with `VMA_MEMORY_USAGE_CPU_ONLY`), you should check it. If it's +not, you should call `vkInvalidateMappedMemoryRanges()` before reading and +`vkFlushMappedMemoryRanges()` after writing to mapped memory on CPU. Example: + + + VkMemoryPropertyFlags memFlags; + vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags); + if((memFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0) + { + VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE }; + memRange.memory = allocInfo.deviceMemory; + memRange.offset = allocInfo.offset; + memRange.size = allocInfo.size; + vkFlushMappedMemoryRanges(device, 1, &memRange); + } + +For performance reasons it is also recommended to unmap Vulkan memory for the +time of call to `vkQueueSubmit()` or `vkQueuePresent()`. You can do it for all +persistently mapped memory using just one function call. For details, see +function vmaUnmapPersistentlyMappedMemory(), vmaMapPersistentlyMappedMemory(). + +\subsection custom_memory_pools Custom memory pools + +The library automatically creates and manages default memory pool for each +memory type available on the device. A pool contains a number of +`VkDeviceMemory` blocks. You can create custom pool and allocate memory out of +it. It can be useful if you want to: + +- Keep certain kind of allocations separate from others. +- Enforce particular size of Vulkan memory blocks. +- Limit maximum amount of Vulkan memory allocated for that pool. + +To use custom memory pools: + +-# Fill VmaPoolCreateInfo structure. +-# Call vmaCreatePool() to obtain `VmaPool` handle. +-# When making an allocation, set VmaAllocationCreateInfo::pool to this handle. + You don't need to specify any other parameters of this structure, like usage. + +Example: + + + .// Create a pool that could have at most 2 blocks, 128 MB each. + VmaPoolCreateInfo poolCreateInfo = {}; + poolCreateInfo.memoryTypeIndex = ... + poolCreateInfo.blockSize = 128ull * 1024 * 1024; + poolCreateInfo.maxBlockCount = 2; + + VmaPool pool; + vmaCreatePool(allocator, &poolCreateInfo, &pool); + + .// Allocate a buffer out of it. + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 1024; + bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.pool = pool; + + VkBuffer buf; + VmaAllocation alloc; + VmaAllocationInfo allocInfo; + vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); + +You have to free all allocations made from this pool before destroying it. + + + vmaDestroyBuffer(allocator, buf, alloc); + vmaDestroyPool(allocator, pool); + +\subsection defragmentation Defragmentation + +Interleaved allocations and deallocations of many objects of varying size can +cause fragmentation, which can lead to a situation where the library is unable +to find a continuous range of free memory for a new allocation despite there is +enough free space, just scattered across many small free ranges between existing +allocations. + +To mitigate this problem, you can use vmaDefragment(). Given set of allocations, +this function can move them to compact used memory, ensure more continuous free +space and possibly also free some `VkDeviceMemory`. It can work only on +allocations made from memory type that is `HOST_VISIBLE`. Allocations are +modified to point to the new `VkDeviceMemory` and offset. Data in this memory is +also `memmove`-ed to the new place. However, if you have images or buffers bound +to these allocations (and you certainly do), you need to destroy, recreate, and +bind them to the new place in memory. + +For further details and example code, see documentation of function +vmaDefragment(). + +\subsection lost_allocations Lost allocations + +If your game oversubscribes video memory, if may work OK in previous-generation +graphics APIs (DirectX 9, 10, 11, OpenGL) because resources are automatically +paged to system RAM. In Vulkan you can't do it because when you run out of +memory, an allocation just fails. If you have more data (e.g. textures) that can +fit into VRAM and you don't need it all at once, you may want to upload them to +GPU on demand and "push out" ones that are not used for a long time to make room +for the new ones, effectively using VRAM (or a cartain memory pool) as a form of +cache. Vulkan Memory Allocator can help you with that by supporting a concept of +"lost allocations". + +To create an allocation that can become lost, include `VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT` +flag in VmaAllocationCreateInfo::flags. Before using a buffer or image bound to +such allocation in every new frame, you need to query it if it's not lost. To +check it: call vmaGetAllocationInfo() and see if VmaAllocationInfo::deviceMemory +is not `VK_NULL_HANDLE`. If the allocation is lost, you should not use it or +buffer/image bound to it. You mustn't forget to destroy this allocation and this +buffer/image. + +To create an allocation that can make some other allocations lost to make room +for it, use `VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` flag. You will +usually use both flags `VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` and +`VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT` at the same time. + +Warning! Current implementation uses quite naive, brute force algorithm, +which can make allocation calls that use `VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` +flag quite slow. A new, more optimal algorithm and data structure to speed this +up is planned for the future. + +When interleaving creation of new allocations with usage of existing ones, +how do you make sure that an allocation won't become lost while it's used in the +current frame? + +It is ensured because vmaGetAllocationInfo() not only returns allocation +parameters and checks whether it's not lost, but when it's not, it also +atomically marks it as used in the current frame, which makes it impossible to +become lost in that frame. It uses lockless algorithm, so it works fast and +doesn't involve locking any internal mutex. + +What if my allocation may still be in use by the GPU when it's rendering a +previous frame while I already submit new frame on the CPU? + +You can make sure that allocations "touched" by vmaGetAllocationInfo() will not +become lost for a number of additional frames back from the current one by +specifying this number as VmaAllocatorCreateInfo::frameInUseCount (for default +memory pool) and VmaPoolCreateInfo::frameInUseCount (for custom pool). + +How do you inform the library when new frame starts? + +You need to call function vmaSetCurrentFrameIndex(). + +Example code: + + + struct MyBuffer + { + VkBuffer m_Buf = nullptr; + VmaAllocation m_Alloc = nullptr; + + .// Called when the buffer is really needed in the current frame. + void EnsureBuffer(); + }; + + void MyBuffer::EnsureBuffer() + { + .// Buffer has been created. + if(m_Buf != VK_NULL_HANDLE) + { + .// Check if its allocation is not lost + mark it as used in current frame. + VmaAllocationInfo allocInfo; + vmaGetAllocationInfo(allocator, m_Alloc, &allocInfo); + if(allocInfo.deviceMemory != VK_NULL_HANDLE) + { + .// It's all OK - safe to use m_Buf. + return; + } + } + + .// Buffer not yet exists or lost - destroy and recreate it. + + vmaDestroyBuffer(allocator, m_Buf, m_Alloc); + + VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufCreateInfo.size = 1024; + bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + VmaAllocationCreateInfo allocCreateInfo = {}; + allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT | + VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT; + + vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr); + } + +When using lost allocations, you may see some Vulkan validation layer warnings +about overlapping regions of memory bound to different kinds of buffers and +images. This is still valid as long as you implement proper handling of lost +allocations (like in the example above) and don't use them. + +The library uses following algorithm for allocation, in order: + +-# Try to find free range of memory in existing blocks. +-# If failed, try to create a new block of `VkDeviceMemory`, with preferred block size. +-# If failed, try to create such block with size/2 and size/4. +-# If failed and `VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` flag was + specified, try to find space in existing blocks, possilby making some other + allocations lost. +-# If failed, try to allocate separate `VkDeviceMemory` for this allocation, + just like when you use `VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT`. +-# If failed, choose other memory type that meets the requirements specified in + VmaAllocationCreateInfo and go to point 1. +-# If failed, return `VK_ERROR_OUT_OF_DEVICE_MEMORY`. + + \section configuration Configuration Please check "CONFIGURATION SECTION" in the code to find macros that you can define -before each #include of this file or change directly in this file to provide +before each `#include` of this file or change directly in this file to provide your own implementation of basic facilities like assert, min and max functions, mutex etc. C++ STL is used by default, but changing these allows you to get rid of any STL usage if you want, as many game developers tend to do. -\section custom_memory_allocator Custom memory allocator +\subsection custom_memory_allocator Custom memory allocator You can use custom memory allocator by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These functions will be passed to @@ -143,17 +386,17 @@ Vulkan, as well as used by the library itself to make any CPU-side allocations. \section thread_safety Thread safety -- The library has no global state, so separate VmaAllocator objects can be used +- The library has no global state, so separate `VmaAllocator` objects can be used independently. -- By default, all calls to functions that take VmaAllocator as first parameter +- By default, all calls to functions that take `VmaAllocator` as first parameter are safe to call from multiple threads simultaneously because they are synchronized internally when needed. -- When the allocator is created with VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT - flag, calls to functions that take such VmaAllocator object must be +- When the allocator is created with `VMA_ALLOCATOR_EXTERNALLY_SYNCHRONIZED_BIT` + flag, calls to functions that take such `VmaAllocator` object must be synchronized externally. -- Access to a VmaAllocation object must be externally synchronized. For example, +- Access to a `VmaAllocation` object must be externally synchronized. For example, you must not call vmaGetAllocationInfo() and vmaDefragment() from different - threads at the same time if you pass the same VmaAllocation object to these + threads at the same time if you pass the same `VmaAllocation` object to these functions. */ @@ -210,10 +453,10 @@ typedef struct VmaAllocatorCreateInfo /// Flags for created allocator. Use VmaAllocatorFlagBits enum. VmaAllocatorFlags flags; /// Vulkan physical device. - /** It must be valid throughout whole lifetime of created Allocator. */ + /** It must be valid throughout whole lifetime of created allocator. */ VkPhysicalDevice physicalDevice; /// Vulkan device. - /** It must be valid throughout whole lifetime of created Allocator. */ + /** It must be valid throughout whole lifetime of created allocator. */ VkDevice device; /// Size of a single memory block to allocate for resources. /** Set to 0 to use default, which is currently 256 MB. */ @@ -221,12 +464,26 @@ typedef struct VmaAllocatorCreateInfo /// Size of a single memory block to allocate for resources from a small heap <= 512 MB. /** Set to 0 to use default, which is currently 64 MB. */ VkDeviceSize preferredSmallHeapBlockSize; - /// Custom allocation callbacks. + /// Custom CPU memory allocation callbacks. /** Optional, can be null. When specified, will also be used for all CPU-side memory allocations. */ const VkAllocationCallbacks* pAllocationCallbacks; /// Informative callbacks for vkAllocateMemory, vkFreeMemory. /** Optional, can be null. */ const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks; + /** \brief Maximum number of additional frames that are in use at the same time as current frame. + + This value is used only when you make allocations with + VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become + lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount. + + For example, if you double-buffer your command buffers, so resources used for + rendering in previous frame may still be in use by the GPU at the moment you + allocate resources needed for the current frame, set this value to 1. + + If you want to allow any allocations other than used in the current frame to + become lost, set this value to 0. + */ + uint32_t frameInUseCount; } VmaAllocatorCreateInfo; /// Creates Allocator object. @@ -265,14 +522,31 @@ void vmaGetMemoryTypeProperties( uint32_t memoryTypeIndex, VkMemoryPropertyFlags* pFlags); +/** \brief Sets index of the current frame. + +This function must be used if you make allocations with +`VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT` and +`VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` flags to inform the allocator +when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot +become lost in the current frame. +*/ +void vmaSetCurrentFrameIndex( + VmaAllocator allocator, + uint32_t frameIndex); + typedef struct VmaStatInfo { + /// Number of `VkDeviceMemory` Vulkan memory blocks allocated. + uint32_t BlockCount; + /// Number of `VmaAllocation` allocation objects allocated. uint32_t AllocationCount; - uint32_t SuballocationCount; + /// Number of free ranges of memory between allocations. uint32_t UnusedRangeCount; + /// Total number of bytes occupied by all allocations. VkDeviceSize UsedBytes; + /// Total number of bytes occupied by unused ranges. VkDeviceSize UnusedBytes; - VkDeviceSize SuballocationSizeMin, SuballocationSizeAvg, SuballocationSizeMax; + VkDeviceSize AllocationSizeMin, AllocationSizeAvg, AllocationSizeMax; VkDeviceSize UnusedRangeSizeMin, UnusedRangeSizeAvg, UnusedRangeSizeMax; } VmaStatInfo; @@ -314,84 +588,121 @@ void vmaFreeStatsString( @{ */ +VK_DEFINE_HANDLE(VmaPool) + typedef enum VmaMemoryUsage { /// No intended memory usage specified. VMA_MEMORY_USAGE_UNKNOWN = 0, - /// Memory will be used on device only, no need to be mapped on host. + /// Memory will be used on device only, so faster access from the device is preferred. No need to be mappable on host. VMA_MEMORY_USAGE_GPU_ONLY = 1, - /// Memory will be mapped on host. Could be used for transfer to device. - /** Guarantees to be HOST_VISIBLE and HOST_COHERENT. */ + /// Memory will be mapped on host. Could be used for transfer to/from device. + /** Guarantees to be `HOST_VISIBLE` and `HOST_COHERENT`. */ VMA_MEMORY_USAGE_CPU_ONLY = 2, - /// Memory will be used for frequent (dynamic) updates from host and reads on device. - /** Guarantees to be HOST_VISIBLE. */ + /// Memory will be used for frequent (dynamic) updates from host and reads on device (upload). + /** Guarantees to be `HOST_VISIBLE`. */ VMA_MEMORY_USAGE_CPU_TO_GPU = 3, - /// Memory will be used for writing on device and readback on host. - /** Guarantees to be HOST_VISIBLE. */ + /// Memory will be used for frequent writing on device and readback on host (download). + /** Guarantees to be `HOST_VISIBLE`. */ VMA_MEMORY_USAGE_GPU_TO_CPU = 4, VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF } VmaMemoryUsage; -/// Flags to be passed as VmaMemoryRequirements::flags. -typedef enum VmaMemoryRequirementFlagBits { +/// Flags to be passed as VmaAllocationCreateInfo::flags. +typedef enum VmaAllocationCreateFlagBits { /** \brief Set this flag if the allocation should have its own memory block. Use it for special, big resources, like fullscreen images used as attachments. This flag must also be used for host visible resources that you want to map simultaneously because otherwise they might end up as regions of the same - VkDeviceMemory, while mapping same VkDeviceMemory multiple times is illegal. - */ - VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT = 0x00000001, + `VkDeviceMemory`, while mapping same `VkDeviceMemory` multiple times + simultaneously is illegal. - /** \brief Set this flag to only try to allocate from existing VkDeviceMemory blocks and never create new such block. + You should not use this flag if VmaAllocationCreateInfo::pool is not null. + */ + VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT = 0x00000001, + + /** \brief Set this flag to only try to allocate from existing `VkDeviceMemory` blocks and never create new such block. If new allocation cannot be placed in any of the existing blocks, allocation - fails with VK_ERROR_OUT_OF_DEVICE_MEMORY error. + fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY` error. - It makes no sense to set VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT and - VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT at the same time. */ - VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT = 0x00000002, - /** \brief Set to use a memory that will be persistently mapped and retrieve pointer to it. + You should not use `VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT` and + `VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT` at the same time. It makes no sense. + + If VmaAllocationCreateInfo::pool is not null, this flag is implied and ignored. */ + VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002, + /** \brief Set this flag to use a memory that will be persistently mapped and retrieve pointer to it. Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. You cannot - map the memory on your own as multiple maps of a single VkDeviceMemory are + map the memory on your own as multiple mappings of a single `VkDeviceMemory` are illegal. + + If VmaAllocationCreateInfo::pool is not null, usage of this flag must match + usage of flag `VMA_POOL_CREATE_PERSISTENT_MAP_BIT` used during pool creation. */ - VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT = 0x00000004, + VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT = 0x00000004, + /** Allocation created with this flag can become lost as a result of another + allocation with `VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT` flag, so you + must check it before use. - VMA_MEMORY_REQUIREMENT_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF -} VmaMemoryRequirementFlagBits; -typedef VkFlags VmaMemoryRequirementFlags; + To check if allocation is not lost, call vmaGetAllocationInfo() and check if + VmaAllocationInfo::deviceMemory is not `VK_NULL_HANDLE`. -typedef struct VmaMemoryRequirements + For details about supporting lost allocations, see Lost Allocations + chapter of User Guide on Main Page. + */ + VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008, + /** While creating allocation using this flag, other allocations that were + created with flag `VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT` can become lost. + + For details about supporting lost allocations, see Lost Allocations + chapter of User Guide on Main Page. + */ + VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010, + + VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +} VmaAllocationCreateFlagBits; +typedef VkFlags VmaAllocationCreateFlags; + +typedef struct VmaAllocationCreateInfo { - VmaMemoryRequirementFlags flags; + /// Use VmaAllocationCreateFlagBits enum. + VmaAllocationCreateFlags flags; /** \brief Intended usage of memory. - Leave VMA_MEMORY_USAGE_UNKNOWN if you specify requiredFlags. You can also use both. + Leave `VMA_MEMORY_USAGE_UNKNOWN` if you specify `requiredFlags`. You can also use both. \n + If `pool` is not null, this member is ignored. */ VmaMemoryUsage usage; /** \brief Flags that must be set in a Memory Type chosen for an allocation. - Leave 0 if you specify requirement via usage. */ + Leave 0 if you specify requirement via usage. \n + If `pool` is not null, this member is ignored.*/ VkMemoryPropertyFlags requiredFlags; /** \brief Flags that preferably should be set in a Memory Type chosen for an allocation. - Set to 0 if no additional flags are prefered and only requiredFlags should be used. - If not 0, it must be a superset or equal to requiredFlags. */ + Set to 0 if no additional flags are prefered and only `requiredFlags` should be used. \n + If not 0, it must be a superset or equal to `requiredFlags`. \n + If `pool` is not null, this member is ignored. */ VkMemoryPropertyFlags preferredFlags; /** \brief Custom general-purpose pointer that will be stored in VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData(). */ void* pUserData; -} VmaMemoryRequirements; + /** \brief Pool that this allocation should be created in. + + Leave `VK_NULL_HANDLE` to allocate from general memory. + */ + VmaPool pool; +} VmaAllocationCreateInfo; /** This algorithm tries to find a memory type that: - Is allowed by memoryTypeBits. -- Contains all the flags from pMemoryRequirements->requiredFlags. +- Contains all the flags from pAllocationCreateInfo->requiredFlags. - Matches intended usage. -- Has as many flags from pMemoryRequirements->preferredFlags as possible. +- Has as many flags from pAllocationCreateInfo->preferredFlags as possible. \return Returns VK_ERROR_FEATURE_NOT_PRESENT if not found. Receiving such result from this function or any other allocating function probably means that your @@ -402,7 +713,7 @@ resource, like image layout (OPTIMAL versus LINEAR) or mip level count. VkResult vmaFindMemoryTypeIndex( VmaAllocator allocator, uint32_t memoryTypeBits, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pAllocationCreateInfo, uint32_t* pMemoryTypeIndex); /** @} */ @@ -412,9 +723,142 @@ VkResult vmaFindMemoryTypeIndex( @{ */ +/// Flags to be passed as VmaPoolCreateInfo::flags. +typedef enum VmaPoolCreateFlagBits { + /** Set this flag to use a memory that will be persistently mapped. + + Each allocation made from this pool will have VmaAllocationInfo::pMappedData + available. + + Usage of this flag must match usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT + flag for every allocation made from this pool. + */ + VMA_POOL_CREATE_PERSISTENT_MAP_BIT = 0x00000001, + /** \brief Use this flag if you always allocate only buffers and linear images or only optimal images out of this pool and so Buffer-Image Granularity can be ignored. + + This is na optional optimization flag. + + If you always allocate using vmaCreateBuffer(), vmaCreateImage(), + vmaAllocateMemoryForBuffer(), then you don't need to use it because allocator + knows exact type of your allocations so it can handle Buffer-Image Granularity + in the optimal way. + + If you also allocate using vmaAllocateMemoryForImage() or vmaAllocateMemory(), + exact type of such allocations is not known, so allocator must be conservative + in handling Buffer-Image Granularity, which can lead to suboptimal allocation + (wasted memory). In that case, if you can make sure you always allocate only + buffers and linear images or only optimal images out of this pool, use this flag + to make allocator disregard Buffer-Image Granularity and so make allocations + more optimal. + */ + VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002, + + VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF +} VmaPoolCreateFlagBits; +typedef VkFlags VmaPoolCreateFlags; + +/** \brief Describes parameter of created `VmaPool`. +*/ +typedef struct VmaPoolCreateInfo { + /** \brief Vulkan memory type index to allocate this pool from. + */ + uint32_t memoryTypeIndex; + /** \brief Use combination of `VmaPoolCreateFlagBits`. + */ + VmaPoolCreateFlags flags; + /** \brief Size of a single `VkDeviceMemory` block to be allocated as part of this pool, in bytes. + + Optional. Leave 0 to use default. + */ + VkDeviceSize blockSize; + /** \brief Minimum number of blocks to be always allocated in this pool, even if they stay empty. + + Set to 0 to have no preallocated blocks and let the pool be completely empty. + */ + size_t minBlockCount; + /** \brief Maximum number of blocks that can be allocated in this pool. + + Optional. Set to 0 to use `SIZE_MAX`, which means no limit. + + Set to same value as minBlockCount to have fixed amount of memory allocated + throuout whole lifetime of this pool. + */ + size_t maxBlockCount; + /** \brief Maximum number of additional frames that are in use at the same time as current frame. + + This value is used only when you make allocations with + `VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT` flag. Such allocation cannot become + lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount. + + For example, if you double-buffer your command buffers, so resources used for + rendering in previous frame may still be in use by the GPU at the moment you + allocate resources needed for the current frame, set this value to 1. + + If you want to allow any allocations other than used in the current frame to + become lost, set this value to 0. + */ + uint32_t frameInUseCount; +} VmaPoolCreateInfo; + +/** \brief Describes parameter of existing VmaPool. +*/ +typedef struct VmaPoolStats { + /** \brief Total amount of `VkDeviceMemory` allocated from Vulkan for this pool, in bytes. + */ + VkDeviceSize size; + /** \brief Total number of bytes in the pool not used by any `VmaAllocation`. + */ + VkDeviceSize unusedSize; + /** \brief Number of VmaAllocation objects created from this pool that were not destroyed or lost. + */ + size_t allocationCount; + /** \brief Number of continuous memory ranges in the pool not used by any `VmaAllocation`. + */ + size_t unusedRangeCount; +} VmaPoolStats; + +/** \brief Allocates Vulkan device memory and creates `VmaPool` object. + +@param allocator Allocator object. +@param pCreateInfo Parameters of pool to create. +@param[out] pPool Handle to created pool. +*/ +VkResult vmaCreatePool( + VmaAllocator allocator, + const VmaPoolCreateInfo* pCreateInfo, + VmaPool* pPool); + +/** \brief Destroys VmaPool object and frees Vulkan device memory. +*/ +void vmaDestroyPool( + VmaAllocator allocator, + VmaPool pool); + +/** \brief Retrieves statistics of existing VmaPool object. + +@param allocator Allocator object. +@param pool Pool object. +@param[out] pPoolStats Statistics of specified pool. +*/ +void vmaGetPoolStats( + VmaAllocator allocator, + VmaPool pool, + VmaPoolStats* pPoolStats); + +/** \brief Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now. + +@param allocator Allocator object. +@param pool Pool. +@param[out] pLostAllocationCount Number of allocations marked as lost. Optional - pass null if you don't need this information. +*/ +void vmaMakePoolAllocationsLost( + VmaAllocator allocator, + VmaPool pool, + size_t* pLostAllocationCount); + VK_DEFINE_HANDLE(VmaAllocation) -/** \brief Parameters of VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo(). +/** \brief Parameters of `VmaAllocation` objects, that can be retrieved using function vmaGetAllocationInfo(). */ typedef struct VmaAllocationInfo { /** \brief Memory type index that this allocation was allocated from. @@ -426,17 +870,19 @@ typedef struct VmaAllocationInfo { Same memory object can be shared by multiple allocations. - It can change after call to vmaDefragment() if this allocation is passed to the function. + It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost. + + If the allocation is lost, it is equal to `VK_NULL_HANDLE`. */ VkDeviceMemory deviceMemory; /** \brief Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation. - It can change after call to vmaDefragment() if this allocation is passed to the function. + It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost. */ VkDeviceSize offset; /** \brief Size of this allocation, in bytes. - It never changes. + It never changes, unless allocation is lost. */ VkDeviceSize size; /** \brief Pointer to the beginning of this allocation as mapped data. Null if this alloaction is not persistently mapped. @@ -445,7 +891,7 @@ typedef struct VmaAllocationInfo { It can also change after call to vmaDefragment() if this allocation is passed to the function. */ void* pMappedData; - /** \brief Custom general-purpose pointer that was passed as VmaMemoryRequirements::pUserData or set using vmaSetAllocationUserData(). + /** \brief Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData(). It can change after call to vmaSetAllocationUserData() for this allocation. */ @@ -465,7 +911,7 @@ vmaCreateBuffer(), vmaCreateImage() instead whenever possible. VkResult vmaAllocateMemory( VmaAllocator allocator, const VkMemoryRequirements* pVkMemoryRequirements, - const VmaMemoryRequirements* pVmaMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo); @@ -478,7 +924,7 @@ You should free the memory using vmaFreeMemory(). VkResult vmaAllocateMemoryForBuffer( VmaAllocator allocator, VkBuffer buffer, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo); @@ -486,7 +932,7 @@ VkResult vmaAllocateMemoryForBuffer( VkResult vmaAllocateMemoryForImage( VmaAllocator allocator, VkImage image, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo); @@ -507,13 +953,27 @@ void vmaSetAllocationUserData( VmaAllocation allocation, void* pUserData); +/** \brief Creates new allocation that is in lost state from the beginning. + +It can be useful if you need a dummy, non-null allocation. + +You still need to destroy created object using vmaFreeMemory(). + +Returned allocation is not tied to any specific memory pool or memory type and +not bound to any image or buffer. It has size = 0. It cannot be turned into +a real, non-empty allocation. +*/ +void vmaCreateLostAllocation( + VmaAllocator allocator, + VmaAllocation* pAllocation); + /** Feel free to use vkMapMemory on these memory blocks on you own if you want, but just for convenience and to make sure correct offset and size is always specified, usage of vmaMapMemory() / vmaUnmapMemory() is recommended. -Do not use it on memory allocated with VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT -as multiple maps to same VkDeviceMemory is illegal. +Do not use it on memory allocated with `VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT` +as multiple maps to same `VkDeviceMemory` is illegal. */ VkResult vmaMapMemory( VmaAllocator allocator, @@ -524,10 +984,10 @@ void vmaUnmapMemory( VmaAllocator allocator, VmaAllocation allocation); -/** \brief Unmaps persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. +/** \brief Unmaps persistently mapped memory of types that are `HOST_COHERENT` and `DEVICE_LOCAL`. This is optional performance optimization. On Windows you should call it before -every call to vkQueueSubmit and vkQueuePresent. After which you can remap the +every call to `vkQueueSubmit` and `vkQueuePresent`. After which you can remap the allocations again using vmaMapPersistentlyMappedMemory(). This is because of the internal behavior of WDDM. Example: @@ -544,7 +1004,7 @@ vmaUnmapPersistentlyMappedMemory(). */ void vmaUnmapPersistentlyMappedMemory(VmaAllocator allocator); -/** \brief Maps back persistently mapped memory of types that is HOST_COHERENT and DEVICE_LOCAL. +/** \brief Maps back persistently mapped memory of types that are `HOST_COHERENT` and `DEVICE_LOCAL`. See vmaUnmapPersistentlyMappedMemory(). @@ -557,12 +1017,12 @@ VkResult vmaMapPersistentlyMappedMemory(VmaAllocator allocator); typedef struct VmaDefragmentationInfo { /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places. - Default is VK_WHOLE_SIZE, which means no limit. + Default is `VK_WHOLE_SIZE`, which means no limit. */ VkDeviceSize maxBytesToMove; /** \brief Maximum number of allocations that can be moved to different place. - Default is UINT32_MAX, which means no limit. + Default is `UINT32_MAX`, which means no limit. */ uint32_t maxAllocationsToMove; } VmaDefragmentationInfo; @@ -571,11 +1031,11 @@ typedef struct VmaDefragmentationInfo { typedef struct VmaDefragmentationStats { /// Total number of bytes that have been copied while moving allocations to different places. VkDeviceSize bytesMoved; - /// Total number of bytes that have been released to the system by freeing empty VkDeviceMemory objects. + /// Total number of bytes that have been released to the system by freeing empty `VkDeviceMemory` objects. VkDeviceSize bytesFreed; /// Number of allocations that have been moved to different places. uint32_t allocationsMoved; - /// Number of empty VkDeviceMemory objects that have been released to the system. + /// Number of empty `VkDeviceMemory` objects that have been released to the system. uint32_t deviceMemoryBlocksFreed; } VmaDefragmentationStats; @@ -589,22 +1049,22 @@ typedef struct VmaDefragmentationStats { @return VK_SUCCESS if completed, VK_INCOMPLETE if succeeded but didn't make all possible optimizations because limits specified in pDefragmentationInfo have been reached, negative error code in case of error. This function works by moving allocations to different places (different -VkDeviceMemory objects and/or different offsets) in order to optimize memory +`VkDeviceMemory` objects and/or different offsets) in order to optimize memory usage. Only allocations that are in pAllocations array can be moved. All other allocations are considered nonmovable in this call. Basic rules: - Only allocations made in memory types that have - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT flag can be compacted. You may pass other + `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag can be compacted. You may pass other allocations but it makes no sense - these will never be moved. -- You may pass allocations made with VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT but +- You may pass allocations made with `VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT` but it makes no sense - they will never be moved. -- Both allocations made with or without VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT +- Both allocations made with or without `VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT` flag can be compacted. If not persistently mapped, memory will be mapped temporarily inside this function if needed, so it shouldn't be mapped by you for the time of this call. -- You must not pass same VmaAllocation object multiple times in pAllocations array. +- You must not pass same `VmaAllocation` object multiple times in pAllocations array. -The function also frees empty VkDeviceMemory blocks. +The function also frees empty `VkDeviceMemory` blocks. After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or VmaAllocationInfo::offset changes. You must query them again using @@ -613,8 +1073,8 @@ vmaGetAllocationInfo() if you need them. If an allocation has been moved, data in memory is copied to new place automatically, but if it was bound to a buffer or an image, you must destroy that object yourself, create new one and bind it to the new memory pointed by -the allocation. You must use vkDestroyBuffer(), vkDestroyImage(), -vkCreateBuffer(), vkCreateImage() for that purpose and NOT vmaDestroyBuffer(), +the allocation. You must use `vkDestroyBuffer()`, `vkDestroyImage()`, +`vkCreateBuffer()`, `vkCreateImage()` for that purpose and NOT vmaDestroyBuffer(), vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example: @@ -684,8 +1144,8 @@ separately, using vkDestroyBuffer() and vmaFreeMemory(). */ VkResult vmaCreateBuffer( VmaAllocator allocator, - const VkBufferCreateInfo* pCreateInfo, - const VmaMemoryRequirements* pMemoryRequirements, + const VkBufferCreateInfo* pBufferCreateInfo, + const VmaAllocationCreateInfo* pAllocationCreateInfo, VkBuffer* pBuffer, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo); @@ -698,8 +1158,8 @@ void vmaDestroyBuffer( /// Function similar to vmaCreateBuffer(). VkResult vmaCreateImage( VmaAllocator allocator, - const VkImageCreateInfo* pCreateInfo, - const VmaMemoryRequirements* pMemoryRequirements, + const VkImageCreateInfo* pImageCreateInfo, + const VmaAllocationCreateInfo* pAllocationCreateInfo, VkImage* pImage, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo); @@ -713,6 +1173,11 @@ void vmaDestroyImage( #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H +// For Visual Studio IntelliSense. +#ifdef __INTELLISENSE__ +#define VMA_IMPLEMENTATION +#endif + #ifdef VMA_IMPLEMENTATION #undef VMA_IMPLEMENTATION @@ -761,6 +1226,7 @@ remove them if not needed. #include // for assert #include // for min, max #include // for std::mutex +#include // for std::atomic_int64_t #if !defined(_WIN32) #include // for aligned_alloc() @@ -846,6 +1312,10 @@ remove them if not needed. { snprintf(outStr, strLen, "%llu", static_cast(num)); } + static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr) + { + snprintf(outStr, strLen, "%p", ptr); + } #endif #ifndef VMA_MUTEX @@ -862,6 +1332,18 @@ remove them if not needed. #define VMA_MUTEX VmaMutex #endif +/* +If providing your own implementation, you need to implement a subset of std::atomic: + +- Constructor(uint32_t desired) +- uint32_t load() const +- void store(uint32_t desired) +- bool compare_exchange_weak(uint32_t& expected, uint32_t desired) +*/ +#ifndef VMA_ATOMIC_UINT32 + #define VMA_ATOMIC_UINT32 std::atomic_uint32_t +#endif + #ifndef VMA_BEST_FIT /** Main parameter for function assessing how good is a free suballocation for a new @@ -933,6 +1415,8 @@ remove them if not needed. #define VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE (64 * 1024 * 1024) #endif +static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX; + /******************************************************************************* END OF CONFIGURATION */ @@ -973,13 +1457,13 @@ Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp) { Iterator centerValue = end; --centerValue; Iterator insertIndex = beg; - for(Iterator i = beg; i < centerValue; ++i) + for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex) { - if(cmp(*i, *centerValue)) + if(cmp(*memTypeIndex, *centerValue)) { - if(insertIndex != i) + if(insertIndex != memTypeIndex) { - VMA_SWAP(*i, *insertIndex); + VMA_SWAP(*memTypeIndex, *insertIndex); } ++insertIndex; } @@ -1264,6 +1748,8 @@ template class VmaVector { public: + typedef T value_type; + VmaVector(const AllocatorT& allocator) : m_Allocator(allocator), m_pArray(VMA_NULL), @@ -1274,7 +1760,7 @@ public: VmaVector(size_t count, const AllocatorT& allocator) : m_Allocator(allocator), - m_pArray(count ? (T*)VmaAllocateArray(allocator->m_pCallbacks, count) : VMA_NULL), + m_pArray(count ? (T*)VmaAllocateArray(allocator.m_pCallbacks, count) : VMA_NULL), m_Count(count), m_Capacity(count) { @@ -1282,7 +1768,7 @@ public: VmaVector(const VmaVector& src) : m_Allocator(src.m_Allocator), - m_pArray(src.m_Count ? (T*)VmaAllocateArray(src->m_pCallbacks, src.m_Count) : VMA_NULL), + m_pArray(src.m_Count ? (T*)VmaAllocateArray(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL), m_Count(src.m_Count), m_Capacity(src.m_Count) { @@ -1475,6 +1961,55 @@ static void VmaVectorRemove(VmaVector& vec, size_t index) #endif // #if VMA_USE_STL_VECTOR +template +size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value) +{ + const size_t indexToInsert = VmaBinaryFindFirstNotLess( + vector.data(), + vector.data() + vector.size(), + value, + CmpLess()) - vector.data(); + VmaVectorInsert(vector, indexToInsert, value); + return indexToInsert; +} + +template +bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value) +{ + CmpLess comparator; + VectorT::iterator it = VmaBinaryFindFirstNotLess( + vector.data(), + vector.data() + vector.size(), + value, + comparator); + if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it)) + { + size_t indexToRemove = it - vector.begin(); + VmaVectorRemove(vector, indexToRemove); + return true; + } + return false; +} + +template +size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value) +{ + CmpLess comparator; + VectorT::iterator it = VmaBinaryFindFirstNotLess( + vector.data(), + vector.data() + vector.size(), + value, + comparator); + if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it)) + { + return it - vector.begin(); + } + else + { + return vector.size(); + } +} + //////////////////////////////////////////////////////////////////////////////// // class VmaPoolAllocator @@ -2074,6 +2609,9 @@ private: //////////////////////////////////////////////////////////////////////////////// // class VmaMap +// Unused in this version. +#if 0 + #if VMA_USE_STL_UNORDERED_MAP #define VmaPair std::pair @@ -2168,9 +2706,11 @@ void VmaMap::erase(iterator it) #endif // #if VMA_USE_STL_UNORDERED_MAP +#endif // #if 0 + //////////////////////////////////////////////////////////////////////////////// -class VmaBlock; +class VmaDeviceMemoryBlock; enum VMA_BLOCK_VECTOR_TYPE { @@ -2179,9 +2719,9 @@ enum VMA_BLOCK_VECTOR_TYPE VMA_BLOCK_VECTOR_TYPE_COUNT }; -static VMA_BLOCK_VECTOR_TYPE VmaMemoryRequirementFlagsToBlockVectorType(VmaMemoryRequirementFlags flags) +static VMA_BLOCK_VECTOR_TYPE VmaAllocationCreateFlagsToBlockVectorType(VmaAllocationCreateFlags flags) { - return (flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0 ? + return (flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0 ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED; } @@ -2196,18 +2736,25 @@ public: ALLOCATION_TYPE_OWN, }; - VmaAllocation_T() + VmaAllocation_T(uint32_t currentFrameIndex) : + m_Alignment(1), + m_Size(0), + m_pUserData(VMA_NULL), + m_Type(ALLOCATION_TYPE_NONE), + m_SuballocationType(VMA_SUBALLOCATION_TYPE_UNKNOWN), + m_LastUseFrameIndex(currentFrameIndex) { - memset(this, 0, sizeof(VmaAllocation_T)); } void InitBlockAllocation( - VmaBlock* block, + VmaPool hPool, + VmaDeviceMemoryBlock* block, VkDeviceSize offset, VkDeviceSize alignment, VkDeviceSize size, VmaSuballocationType suballocationType, - void* pUserData) + void* pUserData, + bool canBecomeLost) { VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE); VMA_ASSERT(block != VMA_NULL); @@ -2216,12 +2763,25 @@ public: m_Size = size; m_pUserData = pUserData; m_SuballocationType = suballocationType; + m_BlockAllocation.m_hPool = hPool; m_BlockAllocation.m_Block = block; m_BlockAllocation.m_Offset = offset; + m_BlockAllocation.m_CanBecomeLost = canBecomeLost; + } + + void InitLost() + { + VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE); + VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST); + m_Type = ALLOCATION_TYPE_BLOCK; + m_BlockAllocation.m_hPool = VK_NULL_HANDLE; + m_BlockAllocation.m_Block = VMA_NULL; + m_BlockAllocation.m_Offset = 0; + m_BlockAllocation.m_CanBecomeLost = true; } void ChangeBlockAllocation( - VmaBlock* block, + VmaDeviceMemoryBlock* block, VkDeviceSize offset) { VMA_ASSERT(block != VMA_NULL); @@ -2259,19 +2819,18 @@ public: void SetUserData(void* pUserData) { m_pUserData = pUserData; } VmaSuballocationType GetSuballocationType() const { return m_SuballocationType; } - VmaBlock* GetBlock() const + VmaDeviceMemoryBlock* GetBlock() const { VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK); return m_BlockAllocation.m_Block; } - VkDeviceSize GetOffset() const - { - return (m_Type == ALLOCATION_TYPE_BLOCK) ? m_BlockAllocation.m_Offset : 0; - } + VkDeviceSize GetOffset() const; VkDeviceMemory GetMemory() const; uint32_t GetMemoryTypeIndex() const; VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const; void* GetMappedData() const; + bool CanBecomeLost() const; + VmaPool GetPool() const; VkResult OwnAllocMapPersistentlyMappedMemory(VkDevice hDevice) { @@ -2292,6 +2851,24 @@ public: m_OwnAllocation.m_pMappedData = VMA_NULL; } } + + uint32_t GetLastUseFrameIndex() const + { + return m_LastUseFrameIndex.load(); + } + bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired) + { + return m_LastUseFrameIndex.compare_exchange_weak(expected, desired); + } + /* + - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex, + makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true. + - Else, returns false. + + If hAllocation is already lost, assert - you should not call it then. + If hAllocation was not created with CAN_BECOME_LOST_BIT, assert. + */ + bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); private: VkDeviceSize m_Alignment; @@ -2299,12 +2876,15 @@ private: void* m_pUserData; ALLOCATION_TYPE m_Type; VmaSuballocationType m_SuballocationType; + VMA_ATOMIC_UINT32 m_LastUseFrameIndex; - // Allocation out of VmaBlock. + // Allocation out of VmaDeviceMemoryBlock. struct BlockAllocation { - VmaBlock* m_Block; + VmaPool m_hPool; // Null if belongs to general memory. + VmaDeviceMemoryBlock* m_Block; VkDeviceSize m_Offset; + bool m_CanBecomeLost; }; // Allocation for an object that has its own private VkDeviceMemory. @@ -2318,7 +2898,7 @@ private: union { - // Allocation out of VmaBlock. + // Allocation out of VmaDeviceMemoryBlock. BlockAllocation m_BlockAllocation; // Allocation for an object that has its own private VkDeviceMemory. OwnAllocation m_OwnAllocation; @@ -2326,28 +2906,56 @@ private: }; /* -Represents a region of VmaBlock that is either assigned and returned as +Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as allocated memory block or free. */ struct VmaSuballocation { VkDeviceSize offset; VkDeviceSize size; + VmaAllocation hAllocation; VmaSuballocationType type; }; typedef VmaList< VmaSuballocation, VmaStlAllocator > VmaSuballocationList; -// Parameters of an allocation. +// Cost of one additional allocation lost, as equivalent in bytes. +static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576; + +/* +Parameters of planned allocation inside a VmaDeviceMemoryBlock. + +If canMakeOtherLost was false: +- item points to a FREE suballocation. +- itemsToMakeLostCount is 0. + +If canMakeOtherLost was true: +- item points to first of sequence of suballocations, which are either FREE, + or point to VmaAllocations that can become lost. +- itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for + the requested allocation to succeed. +*/ struct VmaAllocationRequest { - VmaSuballocationList::iterator freeSuballocationItem; VkDeviceSize offset; + VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation. + VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation. + VmaSuballocationList::iterator item; + size_t itemsToMakeLostCount; + + VkDeviceSize CalcCost() const + { + return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST; + } }; -/* Single block of memory - VkDeviceMemory with all the data about its regions -assigned or free. */ -class VmaBlock +/* +Represents a single block of device memory (VkDeviceMemory ) with all the +data about its regions (aka suballocations, VmaAllocation), assigned and free. + +Thread-safety: This class must be externally synchronized. +*/ +class VmaDeviceMemoryBlock { public: uint32_t m_MemoryTypeIndex; @@ -2363,9 +2971,9 @@ public: // Sorted by size, ascending. VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize; - VmaBlock(VmaAllocator hAllocator); + VmaDeviceMemoryBlock(VmaAllocator hAllocator); - ~VmaBlock() + ~VmaDeviceMemoryBlock() { VMA_ASSERT(m_hMemory == VK_NULL_HANDLE); } @@ -2388,22 +2996,19 @@ public: // If succeeded, fills pAllocationRequest and returns true. // If failed, returns false. bool CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, VkDeviceSize allocSize, VkDeviceSize allocAlignment, VmaSuballocationType allocType, + bool canMakeOtherLost, VmaAllocationRequest* pAllocationRequest); - // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem. - // If yes, fills pOffset and returns true. If no, returns false. - bool CheckAllocation( - VkDeviceSize bufferImageGranularity, - VkDeviceSize allocSize, - VkDeviceSize allocAlignment, - VmaSuballocationType allocType, - VmaSuballocationList::const_iterator freeSuballocItem, - VkDeviceSize* pOffset) const; - + bool MakeRequestedAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest); + + uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount); + // Returns true if this allocation is empty - contains only single free suballocation. bool IsEmpty() const; @@ -2412,27 +3017,47 @@ public: void Alloc( const VmaAllocationRequest& request, VmaSuballocationType type, - VkDeviceSize allocSize); + VkDeviceSize allocSize, + VmaAllocation hAllocation); // Frees suballocation assigned to given memory region. void Free(const VmaAllocation allocation); #if VMA_STATS_STRING_ENABLED - void PrintDetailedMap(class VmaStringBuilder& sb) const; + void PrintDetailedMap(class VmaJsonWriter& json) const; #endif private: + // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem. + // If yes, fills pOffset and returns true. If no, returns false. + bool CheckAllocation( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, + VkDeviceSize bufferImageGranularity, + VkDeviceSize allocSize, + VkDeviceSize allocAlignment, + VmaSuballocationType allocType, + VmaSuballocationList::const_iterator suballocItem, + bool canMakeOtherLost, + VkDeviceSize* pOffset, + size_t* itemsToMakeLostCount, + VkDeviceSize* pSumFreeSize, + VkDeviceSize* pSumItemSize) const; + // Given free suballocation, it merges it with following one, which must also be free. void MergeFreeWithNext(VmaSuballocationList::iterator item); - // Releases given suballocation, making it free. Merges it with adjacent free - // suballocations if applicable. - void FreeSuballocation(VmaSuballocationList::iterator suballocItem); + // Releases given suballocation, making it free. + // Merges it with adjacent free suballocations if applicable. + // Returns iterator to new free suballocation at this place. + VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem); // Given free suballocation, it inserts it into sorted list of // m_FreeSuballocationsBySize if it's suitable. void RegisterFreeSuballocation(VmaSuballocationList::iterator item); // Given free suballocation, it removes it from sorted list of // m_FreeSuballocationsBySize if it's suitable. void UnregisterFreeSuballocation(VmaSuballocationList::iterator item); + + bool ValidateFreeSuballocationList() const; }; struct VmaPointerLess @@ -2443,37 +3068,284 @@ struct VmaPointerLess } }; -/* Sequence of VmaBlock. Represents memory blocks allocated for a specific -Vulkan memory type. */ +class VmaDefragmentator; + +/* +Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific +Vulkan memory type. + +Synchronized internally with a mutex. +*/ struct VmaBlockVector { - // Incrementally sorted by sumFreeSize, ascending. - VmaVector< VmaBlock*, VmaStlAllocator > m_Blocks; - - VmaBlockVector(VmaAllocator hAllocator); + VmaBlockVector( + VmaAllocator hAllocator, + uint32_t memoryTypeIndex, + VMA_BLOCK_VECTOR_TYPE blockVectorType, + VkDeviceSize preferredBlockSize, + size_t minBlockCount, + size_t maxBlockCount, + VkDeviceSize bufferImageGranularity, + uint32_t frameInUseCount, + bool isCustomPool); ~VmaBlockVector(); + VkResult CreateMinBlocks(); + + uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; } + VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; } + VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; } + uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; } + VMA_BLOCK_VECTOR_TYPE GetBlockVectorType() const { return m_BlockVectorType; } + + void GetPoolStats(VmaPoolStats* pStats); + bool IsEmpty() const { return m_Blocks.empty(); } - // Finds and removes given block from vector. - void Remove(VmaBlock* pBlock); + VkResult Allocate( + VmaPool hCurrentPool, + uint32_t currentFrameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + VmaSuballocationType suballocType, + VmaAllocation* pAllocation); + + void Free( + VmaAllocation hAllocation); - // Performs single step in sorting m_Blocks. They may not be fully sorted - // after this call. - void IncrementallySortBlocks(); - // Adds statistics of this BlockVector to pStats. - void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const; + void AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex); #if VMA_STATS_STRING_ENABLED - void PrintDetailedMap(class VmaStringBuilder& sb) const; + void PrintDetailedMap(class VmaJsonWriter& json); #endif void UnmapPersistentlyMappedMemory(); VkResult MapPersistentlyMappedMemory(); + void MakePoolAllocationsLost( + uint32_t currentFrameIndex, + size_t* pLostAllocationCount); + + VmaDefragmentator* EnsureDefragmentator( + VkDevice hDevice, + const VkAllocationCallbacks* pAllocationCallbacks, + uint32_t currentFrameIndex); + + VkResult Defragment( + VmaDefragmentationStats* pDefragmentationStats, + VkDeviceSize& maxBytesToMove, + uint32_t& maxAllocationsToMove); + + void DestroyDefragmentator(); + private: - VmaAllocator m_hAllocator; + friend class VmaDefragmentator; + + const VmaAllocator m_hAllocator; + const uint32_t m_MemoryTypeIndex; + const VMA_BLOCK_VECTOR_TYPE m_BlockVectorType; + const VkDeviceSize m_PreferredBlockSize; + const size_t m_MinBlockCount; + const size_t m_MaxBlockCount; + const VkDeviceSize m_BufferImageGranularity; + const uint32_t m_FrameInUseCount; + const bool m_IsCustomPool; + VMA_MUTEX m_Mutex; + // Incrementally sorted by sumFreeSize, ascending. + VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator > m_Blocks; + /* There can be at most one allocation that is completely empty - a + hysteresis to avoid pessimistic case of alternating creation and destruction + of a VkDeviceMemory. */ + bool m_HasEmptyBlock; + VmaDefragmentator* m_pDefragmentator; + + // Finds and removes given block from vector. + void Remove(VmaDeviceMemoryBlock* pBlock); + + // Performs single step in sorting m_Blocks. They may not be fully sorted + // after this call. + void IncrementallySortBlocks(); + + VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex); +}; + +struct VmaPool_T +{ +public: + VmaBlockVector m_BlockVector; + + // Takes ownership. + VmaPool_T( + VmaAllocator hAllocator, + const VmaPoolCreateInfo& createInfo); + ~VmaPool_T(); + + VmaBlockVector& GetBlockVector() { return m_BlockVector; } + +#if VMA_STATS_STRING_ENABLED + //void PrintDetailedMap(class VmaStringBuilder& sb); +#endif +}; + +class VmaDefragmentator +{ + const VkDevice m_hDevice; + const VkAllocationCallbacks* const m_pAllocationCallbacks; + VmaBlockVector* const m_pBlockVector; + uint32_t m_CurrentFrameIndex; + VMA_BLOCK_VECTOR_TYPE m_BlockVectorType; + VkDeviceSize m_BytesMoved; + uint32_t m_AllocationsMoved; + + struct AllocationInfo + { + VmaAllocation m_hAllocation; + VkBool32* m_pChanged; + + AllocationInfo() : + m_hAllocation(VK_NULL_HANDLE), + m_pChanged(VMA_NULL) + { + } + }; + + struct AllocationInfoSizeGreater + { + bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const + { + return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize(); + } + }; + + // Used between AddAllocation and Defragment. + VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; + + struct BlockInfo + { + VmaDeviceMemoryBlock* m_pBlock; + bool m_HasNonMovableAllocations; + VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; + + BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) : + m_pBlock(VMA_NULL), + m_HasNonMovableAllocations(true), + m_Allocations(pAllocationCallbacks), + m_pMappedDataForDefragmentation(VMA_NULL) + { + } + + void CalcHasNonMovableAllocations() + { + const size_t blockAllocCount = + m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount; + const size_t defragmentAllocCount = m_Allocations.size(); + m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount; + } + + void SortAllocationsBySizeDescecnding() + { + VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater()); + } + + VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData) + { + // It has already been mapped for defragmentation. + if(m_pMappedDataForDefragmentation) + { + *ppMappedData = m_pMappedDataForDefragmentation; + return VK_SUCCESS; + } + + // It is persistently mapped. + if(m_pBlock->m_PersistentMap) + { + VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL); + *ppMappedData = m_pBlock->m_pMappedData; + return VK_SUCCESS; + } + + // Map on first usage. + VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation); + *ppMappedData = m_pMappedDataForDefragmentation; + return res; + } + + void Unmap(VkDevice hDevice) + { + if(m_pMappedDataForDefragmentation != VMA_NULL) + { + vkUnmapMemory(hDevice, m_pBlock->m_hMemory); + } + } + + private: + // Not null if mapped for defragmentation only, not persistently mapped. + void* m_pMappedDataForDefragmentation; + }; + + struct BlockPointerLess + { + bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const + { + return pLhsBlockInfo->m_pBlock < pRhsBlock; + } + bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const + { + return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock; + } + }; + + // 1. Blocks with some non-movable allocations go first. + // 2. Blocks with smaller sumFreeSize go first. + struct BlockInfoCompareMoveDestination + { + bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const + { + if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations) + { + return true; + } + if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations) + { + return false; + } + if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize) + { + return true; + } + return false; + } + }; + + typedef VmaVector< BlockInfo*, VmaStlAllocator > BlockInfoVector; + BlockInfoVector m_Blocks; + + VkResult DefragmentRound( + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove); + + static bool MoveMakesSense( + size_t dstBlockIndex, VkDeviceSize dstOffset, + size_t srcBlockIndex, VkDeviceSize srcOffset); + +public: + VmaDefragmentator( + VkDevice hDevice, + const VkAllocationCallbacks* pAllocationCallbacks, + VmaBlockVector* pBlockVector, + uint32_t currentFrameIndex); + + ~VmaDefragmentator(); + + VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } + uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } + + void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged); + + VkResult Defragment( + VkDeviceSize maxBytesToMove, + uint32_t maxAllocationsToMove); }; // Main allocator object. @@ -2484,8 +3356,6 @@ struct VmaAllocator_T bool m_AllocationCallbacksSpecified; VkAllocationCallbacks m_AllocationCallbacks; VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks; - VkDeviceSize m_PreferredLargeHeapBlockSize; - VkDeviceSize m_PreferredSmallHeapBlockSize; // Non-zero when we are inside UnmapPersistentlyMappedMemory...MapPersistentlyMappedMemory. // Counter to allow nested calls to these functions. uint32_t m_UnmapPersistentlyMappedMemoryCounter; @@ -2494,11 +3364,6 @@ struct VmaAllocator_T VkPhysicalDeviceMemoryProperties m_MemProps; VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT]; - /* There can be at most one allocation that is completely empty - a - hysteresis to avoid pessimistic case of alternating creation and destruction - of a VkDeviceMemory. */ - bool m_HasEmptyBlock[VK_MAX_MEMORY_TYPES]; - VMA_MUTEX m_BlocksMutex[VK_MAX_MEMORY_TYPES]; // Each vector is sorted by memory (handle value). typedef VmaVector< VmaAllocation, VmaStlAllocator > AllocationVectorType; @@ -2513,8 +3378,6 @@ struct VmaAllocator_T return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0; } - VkDeviceSize GetPreferredBlockSize(uint32_t memTypeIndex) const; - VkDeviceSize GetBufferImageGranularity() const { return VMA_MAX( @@ -2528,7 +3391,7 @@ struct VmaAllocator_T // Main allocation function. VkResult AllocateMemory( const VkMemoryRequirements& vkMemReq, - const VmaMemoryRequirements& vmaMemReq, + const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, VmaAllocation* pAllocation); @@ -2538,7 +3401,7 @@ struct VmaAllocator_T void CalculateStats(VmaStats* pStats); #if VMA_STATS_STRING_ENABLED - void PrintDetailedMap(class VmaStringBuilder& sb); + void PrintDetailedMap(class VmaJsonWriter& json); #endif void UnmapPersistentlyMappedMemory(); @@ -2551,18 +3414,40 @@ struct VmaAllocator_T const VmaDefragmentationInfo* pDefragmentationInfo, VmaDefragmentationStats* pDefragmentationStats); - static void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo); + void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo); + + VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool); + void DestroyPool(VmaPool pool); + void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats); + + void SetCurrentFrameIndex(uint32_t frameIndex); + + void MakePoolAllocationsLost( + VmaPool hPool, + size_t* pLostAllocationCount); + + void CreateLostAllocation(VmaAllocation* pAllocation); private: + VkDeviceSize m_PreferredLargeHeapBlockSize; + VkDeviceSize m_PreferredSmallHeapBlockSize; + VkPhysicalDevice m_PhysicalDevice; + VMA_ATOMIC_UINT32 m_CurrentFrameIndex; + + VMA_MUTEX m_PoolsMutex; + // Protected by m_PoolsMutex. Sorted by pointer value. + VmaVector > m_Pools; + + VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex); VkResult AllocateMemoryOfType( const VkMemoryRequirements& vkMemReq, - const VmaMemoryRequirements& vmaMemReq, + const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, VmaSuballocationType suballocType, VmaAllocation* pAllocation); - + // Allocates and registers new VkDeviceMemory specifically for single allocation. VkResult AllocateOwnMemory( VkDeviceSize size, @@ -2639,9 +3524,7 @@ public: void AddNewLine() { Add('\n'); } void AddNumber(uint32_t num); void AddNumber(uint64_t num); - void AddBool(bool b) { Add(b ? "true" : "false"); } - void AddNull() { Add("null"); } - void AddString(const char* pStr); + void AddPointer(const void* ptr); private: VmaVector< char, VmaStlAllocator > m_Data; @@ -2672,63 +3555,345 @@ void VmaStringBuilder::AddNumber(uint64_t num) Add(buf); } -void VmaStringBuilder::AddString(const char* pStr) +void VmaStringBuilder::AddPointer(const void* ptr) { - Add('"'); + char buf[21]; + VmaPtrToStr(buf, sizeof(buf), ptr); + Add(buf); +} + +#endif // #if VMA_STATS_STRING_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +// VmaJsonWriter + +#if VMA_STATS_STRING_ENABLED + +class VmaJsonWriter +{ +public: + VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb); + ~VmaJsonWriter(); + + void BeginObject(bool singleLine = false); + void EndObject(); + + void BeginArray(bool singleLine = false); + void EndArray(); + + void WriteString(const char* pStr); + void BeginString(const char* pStr = VMA_NULL); + void ContinueString(const char* pStr); + void ContinueString(uint32_t n); + void ContinueString(uint64_t n); + void EndString(const char* pStr = VMA_NULL); + + void WriteNumber(uint32_t n); + void WriteNumber(uint64_t n); + void WriteBool(bool b); + void WriteNull(); + +private: + static const char* const INDENT; + + enum COLLECTION_TYPE + { + COLLECTION_TYPE_OBJECT, + COLLECTION_TYPE_ARRAY, + }; + struct StackItem + { + COLLECTION_TYPE type; + uint32_t valueCount; + bool singleLineMode; + }; + + VmaStringBuilder& m_SB; + VmaVector< StackItem, VmaStlAllocator > m_Stack; + bool m_InsideString; + + void BeginValue(bool isString); + void WriteIndent(bool oneLess = false); +}; + +const char* const VmaJsonWriter::INDENT = " "; + +VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) : + m_SB(sb), + m_Stack(VmaStlAllocator(pAllocationCallbacks)), + m_InsideString(false) +{ +} + +VmaJsonWriter::~VmaJsonWriter() +{ + VMA_ASSERT(!m_InsideString); + VMA_ASSERT(m_Stack.empty()); +} + +void VmaJsonWriter::BeginObject(bool singleLine) +{ + VMA_ASSERT(!m_InsideString); + + BeginValue(false); + m_SB.Add('{'); + + StackItem item; + item.type = COLLECTION_TYPE_OBJECT; + item.valueCount = 0; + item.singleLineMode = singleLine; + m_Stack.push_back(item); +} + +void VmaJsonWriter::EndObject() +{ + VMA_ASSERT(!m_InsideString); + + WriteIndent(true); + m_SB.Add('}'); + + VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT); + m_Stack.pop_back(); +} + +void VmaJsonWriter::BeginArray(bool singleLine) +{ + VMA_ASSERT(!m_InsideString); + + BeginValue(false); + m_SB.Add('['); + + StackItem item; + item.type = COLLECTION_TYPE_ARRAY; + item.valueCount = 0; + item.singleLineMode = singleLine; + m_Stack.push_back(item); +} + +void VmaJsonWriter::EndArray() +{ + VMA_ASSERT(!m_InsideString); + + WriteIndent(true); + m_SB.Add(']'); + + VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY); + m_Stack.pop_back(); +} + +void VmaJsonWriter::WriteString(const char* pStr) +{ + BeginString(pStr); + EndString(); +} + +void VmaJsonWriter::BeginString(const char* pStr) +{ + VMA_ASSERT(!m_InsideString); + + BeginValue(true); + m_SB.Add('"'); + m_InsideString = true; + if(pStr != VMA_NULL && pStr[0] != '\0') + { + ContinueString(pStr); + } +} + +void VmaJsonWriter::ContinueString(const char* pStr) +{ + VMA_ASSERT(m_InsideString); + const size_t strLen = strlen(pStr); for(size_t i = 0; i < strLen; ++i) { char ch = pStr[i]; if(ch == '\'') { - Add("\\\\"); + m_SB.Add("\\\\"); } else if(ch == '"') { - Add("\\\""); + m_SB.Add("\\\""); } else if(ch >= 32) { - Add(ch); + m_SB.Add(ch); } else switch(ch) { case '\n': - Add("\\n"); + m_SB.Add("\\n"); break; case '\r': - Add("\\r"); + m_SB.Add("\\r"); break; case '\t': - Add("\\t"); + m_SB.Add("\\t"); break; default: VMA_ASSERT(0 && "Character not currently supported."); break; } } - Add('"'); } +void VmaJsonWriter::ContinueString(uint32_t n) +{ + VMA_ASSERT(m_InsideString); + m_SB.AddNumber(n); +} + +void VmaJsonWriter::ContinueString(uint64_t n) +{ + VMA_ASSERT(m_InsideString); + m_SB.AddNumber(n); +} + +void VmaJsonWriter::EndString(const char* pStr) +{ + VMA_ASSERT(m_InsideString); + if(pStr != VMA_NULL && pStr[0] != '\0') + { + ContinueString(pStr); + } + m_SB.Add('"'); + m_InsideString = false; +} + +void VmaJsonWriter::WriteNumber(uint32_t n) +{ + VMA_ASSERT(!m_InsideString); + BeginValue(false); + m_SB.AddNumber(n); +} + +void VmaJsonWriter::WriteNumber(uint64_t n) +{ + VMA_ASSERT(!m_InsideString); + BeginValue(false); + m_SB.AddNumber(n); +} + +void VmaJsonWriter::WriteBool(bool b) +{ + VMA_ASSERT(!m_InsideString); + BeginValue(false); + m_SB.Add(b ? "true" : "false"); +} + +void VmaJsonWriter::WriteNull() +{ + VMA_ASSERT(!m_InsideString); + BeginValue(false); + m_SB.Add("null"); +} + +void VmaJsonWriter::BeginValue(bool isString) +{ + if(!m_Stack.empty()) + { + StackItem& currItem = m_Stack.back(); + if(currItem.type == COLLECTION_TYPE_OBJECT && + currItem.valueCount % 2 == 0) + { + VMA_ASSERT(isString); + } + + if(currItem.type == COLLECTION_TYPE_OBJECT && + currItem.valueCount % 2 != 0) + { + m_SB.Add(": "); + } + else if(currItem.valueCount > 0) + { + m_SB.Add(", "); + WriteIndent(); + } + else + { + WriteIndent(); + } + ++currItem.valueCount; + } +} + +void VmaJsonWriter::WriteIndent(bool oneLess) +{ + if(!m_Stack.empty() && !m_Stack.back().singleLineMode) + { + m_SB.AddNewLine(); + + size_t count = m_Stack.size(); + if(count > 0 && oneLess) + { + --count; + } + for(size_t i = 0; i < count; ++i) + { + m_SB.Add(INDENT); + } + } +} + +#endif // #if VMA_STATS_STRING_ENABLED + //////////////////////////////////////////////////////////////////////////////// +VkDeviceSize VmaAllocation_T::GetOffset() const +{ + switch(m_Type) + { + case ALLOCATION_TYPE_BLOCK: + return m_BlockAllocation.m_Offset; + case ALLOCATION_TYPE_OWN: + return 0; + default: + VMA_ASSERT(0); + return 0; + } +} + VkDeviceMemory VmaAllocation_T::GetMemory() const { - return (m_Type == ALLOCATION_TYPE_BLOCK) ? - m_BlockAllocation.m_Block->m_hMemory : m_OwnAllocation.m_hMemory; + switch(m_Type) + { + case ALLOCATION_TYPE_BLOCK: + return m_BlockAllocation.m_Block->m_hMemory; + case ALLOCATION_TYPE_OWN: + return m_OwnAllocation.m_hMemory; + default: + VMA_ASSERT(0); + return VK_NULL_HANDLE; + } } uint32_t VmaAllocation_T::GetMemoryTypeIndex() const { - return (m_Type == ALLOCATION_TYPE_BLOCK) ? - m_BlockAllocation.m_Block->m_MemoryTypeIndex : m_OwnAllocation.m_MemoryTypeIndex; + switch(m_Type) + { + case ALLOCATION_TYPE_BLOCK: + return m_BlockAllocation.m_Block->m_MemoryTypeIndex; + case ALLOCATION_TYPE_OWN: + return m_OwnAllocation.m_MemoryTypeIndex; + default: + VMA_ASSERT(0); + return UINT32_MAX; + } } VMA_BLOCK_VECTOR_TYPE VmaAllocation_T::GetBlockVectorType() const { - return (m_Type == ALLOCATION_TYPE_BLOCK) ? - m_BlockAllocation.m_Block->m_BlockVectorType : - (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED); + switch(m_Type) + { + case ALLOCATION_TYPE_BLOCK: + return m_BlockAllocation.m_Block->m_BlockVectorType; + case ALLOCATION_TYPE_OWN: + return (m_OwnAllocation.m_PersistentMap ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED); + default: + VMA_ASSERT(0); + return VMA_BLOCK_VECTOR_TYPE_COUNT; + } } void* VmaAllocation_T::GetMappedData() const @@ -2753,6 +3918,60 @@ void* VmaAllocation_T::GetMappedData() const } } +bool VmaAllocation_T::CanBecomeLost() const +{ + switch(m_Type) + { + case ALLOCATION_TYPE_BLOCK: + return m_BlockAllocation.m_CanBecomeLost; + case ALLOCATION_TYPE_OWN: + return false; + default: + VMA_ASSERT(0); + return false; + } +} + +VmaPool VmaAllocation_T::GetPool() const +{ + VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK); + return m_BlockAllocation.m_hPool; +} + +bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) +{ + VMA_ASSERT(CanBecomeLost()); + + /* + Warning: This is a carefully designed algorithm. + Do not modify unless you really know what you're doing :) + */ + uint32_t localLastUseFrameIndex = GetLastUseFrameIndex(); + for(;;) + { + if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST) + { + VMA_ASSERT(0); + return false; + } + else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex) + { + return false; + } + else // Last use time earlier than current time. + { + if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST)) + { + // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST. + // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock. + return true; + } + } + } +} + +#if VMA_STATS_STRING_ENABLED + // Correspond to values of enum VmaSuballocationType. static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = { "FREE", @@ -2763,31 +3982,46 @@ static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = { "IMAGE_OPTIMAL", }; -static void VmaPrintStatInfo(VmaStringBuilder& sb, const VmaStatInfo& stat) +static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat) { - sb.Add("{ \"Allocations\": "); - sb.AddNumber(stat.AllocationCount); - sb.Add(", \"Suballocations\": "); - sb.AddNumber(stat.SuballocationCount); - sb.Add(", \"UnusedRanges\": "); - sb.AddNumber(stat.UnusedRangeCount); - sb.Add(", \"UsedBytes\": "); - sb.AddNumber(stat.UsedBytes); - sb.Add(", \"UnusedBytes\": "); - sb.AddNumber(stat.UnusedBytes); - sb.Add(", \"SuballocationSize\": { \"Min\": "); - sb.AddNumber(stat.SuballocationSizeMin); - sb.Add(", \"Avg\": "); - sb.AddNumber(stat.SuballocationSizeAvg); - sb.Add(", \"Max\": "); - sb.AddNumber(stat.SuballocationSizeMax); - sb.Add(" }, \"UnusedRangeSize\": { \"Min\": "); - sb.AddNumber(stat.UnusedRangeSizeMin); - sb.Add(", \"Avg\": "); - sb.AddNumber(stat.UnusedRangeSizeAvg); - sb.Add(", \"Max\": "); - sb.AddNumber(stat.UnusedRangeSizeMax); - sb.Add(" } }"); + json.BeginObject(); + + json.WriteString("Blocks"); + json.WriteNumber(stat.BlockCount); + + json.WriteString("Allocations"); + json.WriteNumber(stat.AllocationCount); + + json.WriteString("UnusedRanges"); + json.WriteNumber(stat.UnusedRangeCount); + + json.WriteString("UsedBytes"); + json.WriteNumber(stat.UsedBytes); + + json.WriteString("UnusedBytes"); + json.WriteNumber(stat.UnusedBytes); + + json.WriteString("AllocationSize"); + json.BeginObject(true); + json.WriteString("Min"); + json.WriteNumber(stat.AllocationSizeMin); + json.WriteString("Avg"); + json.WriteNumber(stat.AllocationSizeAvg); + json.WriteString("Max"); + json.WriteNumber(stat.AllocationSizeMax); + json.EndObject(); + + json.WriteString("UnusedRangeSize"); + json.BeginObject(true); + json.WriteString("Min"); + json.WriteNumber(stat.UnusedRangeSizeMin); + json.WriteString("Avg"); + json.WriteNumber(stat.UnusedRangeSizeAvg); + json.WriteString("Max"); + json.WriteNumber(stat.UnusedRangeSizeMax); + json.EndObject(); + + json.EndObject(); } #endif // #if VMA_STATS_STRING_ENABLED @@ -2808,7 +4042,7 @@ struct VmaSuballocationItemSizeLess } }; -VmaBlock::VmaBlock(VmaAllocator hAllocator) : +VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) : m_MemoryTypeIndex(UINT32_MAX), m_BlockVectorType(VMA_BLOCK_VECTOR_TYPE_COUNT), m_hMemory(VK_NULL_HANDLE), @@ -2822,7 +4056,7 @@ VmaBlock::VmaBlock(VmaAllocator hAllocator) : { } -void VmaBlock::Init( +void VmaDeviceMemoryBlock::Init( uint32_t newMemoryTypeIndex, VMA_BLOCK_VECTOR_TYPE newBlockVectorType, VkDeviceMemory newMemory, @@ -2848,6 +4082,7 @@ void VmaBlock::Init( suballoc.offset = 0; suballoc.size = newSize; suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + suballoc.hAllocation = VK_NULL_HANDLE; m_Suballocations.push_back(suballoc); VmaSuballocationList::iterator suballocItem = m_Suballocations.end(); @@ -2855,9 +4090,12 @@ void VmaBlock::Init( m_FreeSuballocationsBySize.push_back(suballocItem); } -void VmaBlock::Destroy(VmaAllocator allocator) +void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator) { - VMA_ASSERT(IsEmpty()); + // This is the most important assert in the entire library. + // Hitting it means you have some memory leak - unreleased VmaAllocation objects. + VMA_ASSERT(IsEmpty() && "Some allocations were not freed before destruction of this memory block!"); + VMA_ASSERT(m_hMemory != VK_NULL_HANDLE); if(m_pMappedData != VMA_NULL) { @@ -2875,7 +4113,7 @@ void VmaBlock::Destroy(VmaAllocator allocator) m_hMemory = VK_NULL_HANDLE; } -bool VmaBlock::Validate() const +bool VmaDeviceMemoryBlock::Validate() const { if((m_hMemory == VK_NULL_HANDLE) || (m_Size == 0) || @@ -2916,6 +4154,11 @@ bool VmaBlock::Validate() const } prevFree = currFree; + if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE)) + { + return false; + } + if(currFree) { calculatedSumFreeSize += subAlloc.size; @@ -2972,11 +4215,14 @@ How many suitable free suballocations to analyze before choosing best one. */ //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8; -bool VmaBlock::CreateAllocationRequest( +bool VmaDeviceMemoryBlock::CreateAllocationRequest( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, VkDeviceSize allocSize, VkDeviceSize allocAlignment, VmaSuballocationType allocType, + bool canMakeOtherLost, VmaAllocationRequest* pAllocationRequest) { VMA_ASSERT(allocSize > 0); @@ -2984,41 +4230,12 @@ bool VmaBlock::CreateAllocationRequest( VMA_ASSERT(pAllocationRequest != VMA_NULL); VMA_HEAVY_ASSERT(Validate()); - // There is not enough total free space in this allocation to fullfill the request: Early return. - if(m_SumFreeSize < allocSize) + // There is not enough total free space in this block to fullfill the request: Early return. + if(canMakeOtherLost == false && m_SumFreeSize < allocSize) { return false; } - // Old brute-force algorithm, linearly searching suballocations. - /* - uint32_t suitableSuballocationsFound = 0; - for(VmaSuballocationList::iterator suballocItem = suballocations.Front(); - suballocItem != VMA_NULL && - suitableSuballocationsFound < MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK; - suballocItem = suballocItem->Next) - { - if(suballocItem->Value.type == VMA_SUBALLOCATION_TYPE_FREE) - { - VkDeviceSize offset = 0, cost = 0; - if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset, &cost)) - { - ++suitableSuballocationsFound; - if(cost < costLimit) - { - pAllocationRequest->freeSuballocationItem = suballocItem; - pAllocationRequest->offset = offset; - pAllocationRequest->cost = cost; - if(cost == 0) - return true; - costLimit = cost; - betterSuballocationFound = true; - } - } - } - } - */ - // New algorithm, efficiently searching freeSuballocationsBySize. const size_t freeSuballocCount = m_FreeSuballocationsBySize.size(); if(freeSuballocCount > 0) @@ -3034,12 +4251,21 @@ bool VmaBlock::CreateAllocationRequest( size_t index = it - m_FreeSuballocationsBySize.data(); for(; index < freeSuballocCount; ++index) { - VkDeviceSize offset = 0; - const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index]; - if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset)) + if(CheckAllocation( + currentFrameIndex, + frameInUseCount, + bufferImageGranularity, + allocSize, + allocAlignment, + allocType, + m_FreeSuballocationsBySize[index], + false, // canMakeOtherLost + &pAllocationRequest->offset, + &pAllocationRequest->itemsToMakeLostCount, + &pAllocationRequest->sumFreeSize, + &pAllocationRequest->sumItemSize)) { - pAllocationRequest->freeSuballocationItem = suballocItem; - pAllocationRequest->offset = offset; + pAllocationRequest->item = m_FreeSuballocationsBySize[index]; return true; } } @@ -3049,121 +4275,397 @@ bool VmaBlock::CreateAllocationRequest( // Search staring from biggest suballocations. for(size_t index = freeSuballocCount; index--; ) { - VkDeviceSize offset = 0; - const VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[index]; - if(CheckAllocation(bufferImageGranularity, allocSize, allocAlignment, allocType, suballocItem, &offset)) + if(CheckAllocation( + currentFrameIndex, + frameInUseCount, + bufferImageGranularity, + allocSize, + allocAlignment, + allocType, + m_FreeSuballocationsBySize[index], + false, // canMakeOtherLost + &pAllocationRequest->offset, + &pAllocationRequest->itemsToMakeLostCount, + &pAllocationRequest->sumFreeSize, + &pAllocationRequest->sumItemSize)) { - pAllocationRequest->freeSuballocationItem = suballocItem; - pAllocationRequest->offset = offset; + pAllocationRequest->item = m_FreeSuballocationsBySize[index]; return true; } } } } + if(canMakeOtherLost) + { + // Brute-force algorithm. TODO: Come up with something better. + + pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE; + pAllocationRequest->sumItemSize = VK_WHOLE_SIZE; + + VmaAllocationRequest tmpAllocRequest = {}; + for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin(); + suballocIt != m_Suballocations.end(); + ++suballocIt) + { + if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE || + suballocIt->hAllocation->CanBecomeLost()) + { + if(CheckAllocation( + currentFrameIndex, + frameInUseCount, + bufferImageGranularity, + allocSize, + allocAlignment, + allocType, + suballocIt, + canMakeOtherLost, + &tmpAllocRequest.offset, + &tmpAllocRequest.itemsToMakeLostCount, + &tmpAllocRequest.sumFreeSize, + &tmpAllocRequest.sumItemSize)) + { + tmpAllocRequest.item = suballocIt; + + if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost()) + { + *pAllocationRequest = tmpAllocRequest; + } + } + } + } + + if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE) + { + return true; + } + } + return false; } -bool VmaBlock::CheckAllocation( +bool VmaDeviceMemoryBlock::MakeRequestedAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount, VmaAllocationRequest* pAllocationRequest) +{ + while(pAllocationRequest->itemsToMakeLostCount > 0) + { + if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE) + { + ++pAllocationRequest->item; + } + VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end()); + VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE); + VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost()); + if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount)) + { + pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item); + --pAllocationRequest->itemsToMakeLostCount; + } + else + { + return false; + } + } + + VMA_HEAVY_ASSERT(Validate()); + VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end()); + VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE); + + return true; +} + +uint32_t VmaDeviceMemoryBlock::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) +{ + uint32_t lostAllocationCount = 0; + for(VmaSuballocationList::iterator it = m_Suballocations.begin(); + it != m_Suballocations.end(); + ++it) + { + if(it->type != VMA_SUBALLOCATION_TYPE_FREE && + it->hAllocation->CanBecomeLost() && + it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount)) + { + it = FreeSuballocation(it); + ++lostAllocationCount; + } + } + return lostAllocationCount; +} + +bool VmaDeviceMemoryBlock::CheckAllocation( + uint32_t currentFrameIndex, + uint32_t frameInUseCount, VkDeviceSize bufferImageGranularity, VkDeviceSize allocSize, VkDeviceSize allocAlignment, VmaSuballocationType allocType, - VmaSuballocationList::const_iterator freeSuballocItem, - VkDeviceSize* pOffset) const + VmaSuballocationList::const_iterator suballocItem, + bool canMakeOtherLost, + VkDeviceSize* pOffset, + size_t* itemsToMakeLostCount, + VkDeviceSize* pSumFreeSize, + VkDeviceSize* pSumItemSize) const { VMA_ASSERT(allocSize > 0); VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE); - VMA_ASSERT(freeSuballocItem != m_Suballocations.cend()); + VMA_ASSERT(suballocItem != m_Suballocations.cend()); VMA_ASSERT(pOffset != VMA_NULL); - - const VmaSuballocation& suballoc = *freeSuballocItem; - VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); - - // Size of this suballocation is too small for this request: Early return. - if(suballoc.size < allocSize) - { - return false; - } - - // Start from offset equal to beginning of this suballocation. - *pOffset = suballoc.offset; - // Apply VMA_DEBUG_MARGIN at the beginning. - if((VMA_DEBUG_MARGIN > 0) && freeSuballocItem != m_Suballocations.cbegin()) + *itemsToMakeLostCount = 0; + *pSumFreeSize = 0; + *pSumItemSize = 0; + + if(canMakeOtherLost) { - *pOffset += VMA_DEBUG_MARGIN; - } - - // Apply alignment. - const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast(VMA_DEBUG_ALIGNMENT)); - *pOffset = VmaAlignUp(*pOffset, alignment); - - // Check previous suballocations for BufferImageGranularity conflicts. - // Make bigger alignment if necessary. - if(bufferImageGranularity > 1) - { - bool bufferImageGranularityConflict = false; - VmaSuballocationList::const_iterator prevSuballocItem = freeSuballocItem; - while(prevSuballocItem != m_Suballocations.cbegin()) + if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE) { - --prevSuballocItem; - const VmaSuballocation& prevSuballoc = *prevSuballocItem; - if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity)) + *pSumFreeSize = suballocItem->size; + } + else + { + if(suballocItem->hAllocation->CanBecomeLost() && + suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex) { - if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType)) - { - bufferImageGranularityConflict = true; - break; - } + ++*itemsToMakeLostCount; + *pSumItemSize = suballocItem->size; } else - // Already on previous page. - break; - } - if(bufferImageGranularityConflict) - { - *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity); - } - } - - // Calculate padding at the beginning based on current offset. - const VkDeviceSize paddingBegin = *pOffset - suballoc.offset; - - // Calculate required margin at the end if this is not last suballocation. - VmaSuballocationList::const_iterator next = freeSuballocItem; - ++next; - const VkDeviceSize requiredEndMargin = - (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0; - - // Fail if requested size plus margin before and after is bigger than size of this suballocation. - if(paddingBegin + allocSize + requiredEndMargin > suballoc.size) - { - return false; - } - - // Check next suballocations for BufferImageGranularity conflicts. - // If conflict exists, allocation cannot be made here. - if(bufferImageGranularity > 1) - { - VmaSuballocationList::const_iterator nextSuballocItem = freeSuballocItem; - ++nextSuballocItem; - while(nextSuballocItem != m_Suballocations.cend()) - { - const VmaSuballocation& nextSuballoc = *nextSuballocItem; - if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) { - if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type)) + return false; + } + } + + // Remaining size is too small for this request: Early return. + if(m_Size - suballocItem->offset < allocSize) + { + return false; + } + + // Start from offset equal to beginning of this suballocation. + *pOffset = suballocItem->offset; + + // Apply VMA_DEBUG_MARGIN at the beginning. + if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin()) + { + *pOffset += VMA_DEBUG_MARGIN; + } + + // Apply alignment. + const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast(VMA_DEBUG_ALIGNMENT)); + *pOffset = VmaAlignUp(*pOffset, alignment); + + // Check previous suballocations for BufferImageGranularity conflicts. + // Make bigger alignment if necessary. + if(bufferImageGranularity > 1) + { + bool bufferImageGranularityConflict = false; + VmaSuballocationList::const_iterator prevSuballocItem = suballocItem; + while(prevSuballocItem != m_Suballocations.cbegin()) + { + --prevSuballocItem; + const VmaSuballocation& prevSuballoc = *prevSuballocItem; + if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType)) + { + bufferImageGranularityConflict = true; + break; + } + } + else + // Already on previous page. + break; + } + if(bufferImageGranularityConflict) + { + *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity); + } + } + + // Now that we have final *pOffset, check if we are past suballocItem. + // If yes, return false - this function should be called for another suballocItem as starting point. + if(*pOffset >= suballocItem->offset + suballocItem->size) + { + return false; + } + + // Calculate padding at the beginning based on current offset. + const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset; + + // Calculate required margin at the end if this is not last suballocation. + VmaSuballocationList::const_iterator next = suballocItem; + ++next; + const VkDeviceSize requiredEndMargin = + (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0; + + const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin; + // Another early return check. + if(suballocItem->offset + totalSize > m_Size) + { + return false; + } + + // Advance lastSuballocItem until desired size is reached. + // Update itemsToMakeLostCount. + VmaSuballocationList::const_iterator lastSuballocItem = suballocItem; + if(totalSize > suballocItem->size) + { + VkDeviceSize remainingSize = totalSize - suballocItem->size; + while(remainingSize > 0) + { + ++lastSuballocItem; + if(lastSuballocItem == m_Suballocations.cend()) { return false; } + if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE) + { + *pSumFreeSize += lastSuballocItem->size; + } + else + { + VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE); + if(lastSuballocItem->hAllocation->CanBecomeLost() && + lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex) + { + ++*itemsToMakeLostCount; + *pSumItemSize += lastSuballocItem->size; + } + else + { + return false; + } + } + remainingSize = (lastSuballocItem->size < remainingSize) ? + remainingSize - lastSuballocItem->size : 0; } - else - { - // Already on next page. - break; - } + } + + // Check next suballocations for BufferImageGranularity conflicts. + // If conflict exists, we must mark more allocations lost or fail. + if(bufferImageGranularity > 1) + { + VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem; ++nextSuballocItem; + while(nextSuballocItem != m_Suballocations.cend()) + { + const VmaSuballocation& nextSuballoc = *nextSuballocItem; + if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type)) + { + VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE); + if(nextSuballoc.hAllocation->CanBecomeLost() && + nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex) + { + ++*itemsToMakeLostCount; + } + else + { + return false; + } + } + } + else + { + // Already on next page. + break; + } + ++nextSuballocItem; + } + } + } + else + { + const VmaSuballocation& suballoc = *suballocItem; + VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); + + *pSumFreeSize = suballoc.size; + + // Size of this suballocation is too small for this request: Early return. + if(suballoc.size < allocSize) + { + return false; + } + + // Start from offset equal to beginning of this suballocation. + *pOffset = suballoc.offset; + + // Apply VMA_DEBUG_MARGIN at the beginning. + if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin()) + { + *pOffset += VMA_DEBUG_MARGIN; + } + + // Apply alignment. + const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast(VMA_DEBUG_ALIGNMENT)); + *pOffset = VmaAlignUp(*pOffset, alignment); + + // Check previous suballocations for BufferImageGranularity conflicts. + // Make bigger alignment if necessary. + if(bufferImageGranularity > 1) + { + bool bufferImageGranularityConflict = false; + VmaSuballocationList::const_iterator prevSuballocItem = suballocItem; + while(prevSuballocItem != m_Suballocations.cbegin()) + { + --prevSuballocItem; + const VmaSuballocation& prevSuballoc = *prevSuballocItem; + if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType)) + { + bufferImageGranularityConflict = true; + break; + } + } + else + // Already on previous page. + break; + } + if(bufferImageGranularityConflict) + { + *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity); + } + } + + // Calculate padding at the beginning based on current offset. + const VkDeviceSize paddingBegin = *pOffset - suballoc.offset; + + // Calculate required margin at the end if this is not last suballocation. + VmaSuballocationList::const_iterator next = suballocItem; + ++next; + const VkDeviceSize requiredEndMargin = + (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0; + + // Fail if requested size plus margin before and after is bigger than size of this suballocation. + if(paddingBegin + allocSize + requiredEndMargin > suballoc.size) + { + return false; + } + + // Check next suballocations for BufferImageGranularity conflicts. + // If conflict exists, allocation cannot be made here. + if(bufferImageGranularity > 1) + { + VmaSuballocationList::const_iterator nextSuballocItem = suballocItem; + ++nextSuballocItem; + while(nextSuballocItem != m_Suballocations.cend()) + { + const VmaSuballocation& nextSuballoc = *nextSuballocItem; + if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity)) + { + if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type)) + { + return false; + } + } + else + { + // Already on next page. + break; + } + ++nextSuballocItem; + } } } @@ -3171,18 +4673,19 @@ bool VmaBlock::CheckAllocation( return true; } -bool VmaBlock::IsEmpty() const +bool VmaDeviceMemoryBlock::IsEmpty() const { return (m_Suballocations.size() == 1) && (m_FreeCount == 1); } -void VmaBlock::Alloc( +void VmaDeviceMemoryBlock::Alloc( const VmaAllocationRequest& request, VmaSuballocationType type, - VkDeviceSize allocSize) + VkDeviceSize allocSize, + VmaAllocation hAllocation) { - VMA_ASSERT(request.freeSuballocationItem != m_Suballocations.end()); - VmaSuballocation& suballoc = *request.freeSuballocationItem; + VMA_ASSERT(request.item != m_Suballocations.end()); + VmaSuballocation& suballoc = *request.item; // Given suballocation is a free block. VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE); // Given offset is inside this suballocation. @@ -3193,11 +4696,12 @@ void VmaBlock::Alloc( // Unregister this free suballocation from m_FreeSuballocationsBySize and update // it to become used. - UnregisterFreeSuballocation(request.freeSuballocationItem); + UnregisterFreeSuballocation(request.item); suballoc.offset = request.offset; suballoc.size = allocSize; suballoc.type = type; + suballoc.hAllocation = hAllocation; // If there are any free bytes remaining at the end, insert new free suballocation after current one. if(paddingEnd) @@ -3206,7 +4710,7 @@ void VmaBlock::Alloc( paddingSuballoc.offset = request.offset + allocSize; paddingSuballoc.size = paddingEnd; paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE; - VmaSuballocationList::iterator next = request.freeSuballocationItem; + VmaSuballocationList::iterator next = request.item; ++next; const VmaSuballocationList::iterator paddingEndItem = m_Suballocations.insert(next, paddingSuballoc); @@ -3221,7 +4725,7 @@ void VmaBlock::Alloc( paddingSuballoc.size = paddingBegin; paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE; const VmaSuballocationList::iterator paddingBeginItem = - m_Suballocations.insert(request.freeSuballocationItem, paddingSuballoc); + m_Suballocations.insert(request.item, paddingSuballoc); RegisterFreeSuballocation(paddingBeginItem); } @@ -3238,11 +4742,12 @@ void VmaBlock::Alloc( m_SumFreeSize -= allocSize; } -void VmaBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem) +VmaSuballocationList::iterator VmaDeviceMemoryBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem) { // Change this suballocation to be marked as free. VmaSuballocation& suballoc = *suballocItem; suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; + suballoc.hAllocation = VK_NULL_HANDLE; // Update totals. ++m_FreeCount; @@ -3280,20 +4785,23 @@ void VmaBlock::FreeSuballocation(VmaSuballocationList::iterator suballocItem) UnregisterFreeSuballocation(prevItem); MergeFreeWithNext(prevItem); RegisterFreeSuballocation(prevItem); + return prevItem; } else + { RegisterFreeSuballocation(suballocItem); + return suballocItem; + } } -void VmaBlock::Free(const VmaAllocation allocation) +void VmaDeviceMemoryBlock::Free(const VmaAllocation allocation) { - const VkDeviceSize allocationOffset = allocation->GetOffset(); for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin(); suballocItem != m_Suballocations.end(); ++suballocItem) { VmaSuballocation& suballoc = *suballocItem; - if(suballoc.offset == allocationOffset) + if(suballoc.hAllocation == allocation) { FreeSuballocation(suballocItem); VMA_HEAVY_ASSERT(Validate()); @@ -3305,45 +4813,50 @@ void VmaBlock::Free(const VmaAllocation allocation) #if VMA_STATS_STRING_ENABLED -void VmaBlock::PrintDetailedMap(class VmaStringBuilder& sb) const +void VmaDeviceMemoryBlock::PrintDetailedMap(class VmaJsonWriter& json) const { - sb.Add("{\n\t\t\t\"Bytes\": "); - sb.AddNumber(m_Size); - sb.Add(",\n\t\t\t\"FreeBytes\": "); - sb.AddNumber(m_SumFreeSize); - sb.Add(",\n\t\t\t\"Suballocations\": "); - sb.AddNumber(m_Suballocations.size()); - sb.Add(",\n\t\t\t\"FreeSuballocations\": "); - sb.AddNumber(m_FreeCount); - sb.Add(",\n\t\t\t\"SuballocationList\": ["); + json.BeginObject(); + json.WriteString("TotalBytes"); + json.WriteNumber(m_Size); + + json.WriteString("UnusedBytes"); + json.WriteNumber(m_SumFreeSize); + + json.WriteString("Allocations"); + json.WriteNumber(m_Suballocations.size() - m_FreeCount); + + json.WriteString("UnusedRanges"); + json.WriteNumber(m_FreeCount); + + json.WriteString("Suballocations"); + json.BeginArray(); size_t i = 0; for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin(); suballocItem != m_Suballocations.cend(); ++suballocItem, ++i) { - if(i > 0) - { - sb.Add(",\n\t\t\t\t{ \"Type\": "); - } - else - { - sb.Add("\n\t\t\t\t{ \"Type\": "); - } - sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]); - sb.Add(", \"Size\": "); - sb.AddNumber(suballocItem->size); - sb.Add(", \"Offset\": "); - sb.AddNumber(suballocItem->offset); - sb.Add(" }"); - } + json.BeginObject(true); + + json.WriteString("Type"); + json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]); - sb.Add("\n\t\t\t]\n\t\t}"); + json.WriteString("Size"); + json.WriteNumber(suballocItem->size); + + json.WriteString("Offset"); + json.WriteNumber(suballocItem->offset); + + json.EndObject(); + } + json.EndArray(); + + json.EndObject(); } #endif // #if VMA_STATS_STRING_ENABLED -void VmaBlock::MergeFreeWithNext(VmaSuballocationList::iterator item) +void VmaDeviceMemoryBlock::MergeFreeWithNext(VmaSuballocationList::iterator item) { VMA_ASSERT(item != m_Suballocations.end()); VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); @@ -3358,11 +4871,15 @@ void VmaBlock::MergeFreeWithNext(VmaSuballocationList::iterator item) m_Suballocations.erase(nextItem); } -void VmaBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item) +void VmaDeviceMemoryBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item) { VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(item->size > 0); + // You may want to enable this validation at the beginning or at the end of + // this function, depending on what do you want to check. + VMA_HEAVY_ASSERT(ValidateFreeSuballocationList()); + if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) { if(m_FreeSuballocationsBySize.empty()) @@ -3371,22 +4888,23 @@ void VmaBlock::RegisterFreeSuballocation(VmaSuballocationList::iterator item) } else { - VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess( - m_FreeSuballocationsBySize.data(), - m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(), - item, - VmaSuballocationItemSizeLess()); - size_t index = it - m_FreeSuballocationsBySize.data(); - VmaVectorInsert(m_FreeSuballocationsBySize, index, item); + VmaVectorInsertSorted(m_FreeSuballocationsBySize, item); } } + + //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList()); } -void VmaBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item) + +void VmaDeviceMemoryBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item) { VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(item->size > 0); + // You may want to enable this validation at the beginning or at the end of + // this function, depending on what do you want to check. + VMA_HEAVY_ASSERT(ValidateFreeSuballocationList()); + if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) { VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess( @@ -3407,28 +4925,58 @@ void VmaBlock::UnregisterFreeSuballocation(VmaSuballocationList::iterator item) } VMA_ASSERT(0 && "Not found."); } + + //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList()); +} + +bool VmaDeviceMemoryBlock::ValidateFreeSuballocationList() const +{ + VkDeviceSize lastSize = 0; + for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i) + { + const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i]; + + if(it->type != VMA_SUBALLOCATION_TYPE_FREE) + { + VMA_ASSERT(0); + return false; + } + if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER) + { + VMA_ASSERT(0); + return false; + } + if(it->size < lastSize) + { + VMA_ASSERT(0); + return false; + } + + lastSize = it->size; + } + return true; } static void InitStatInfo(VmaStatInfo& outInfo) { memset(&outInfo, 0, sizeof(outInfo)); - outInfo.SuballocationSizeMin = UINT64_MAX; + outInfo.AllocationSizeMin = UINT64_MAX; outInfo.UnusedRangeSizeMin = UINT64_MAX; } -static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc) +static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaDeviceMemoryBlock& alloc) { - outInfo.AllocationCount = 1; + outInfo.BlockCount = 1; const uint32_t rangeCount = (uint32_t)alloc.m_Suballocations.size(); - outInfo.SuballocationCount = rangeCount - alloc.m_FreeCount; + outInfo.AllocationCount = rangeCount - alloc.m_FreeCount; outInfo.UnusedRangeCount = alloc.m_FreeCount; outInfo.UnusedBytes = alloc.m_SumFreeSize; outInfo.UsedBytes = alloc.m_Size - outInfo.UnusedBytes; - outInfo.SuballocationSizeMin = UINT64_MAX; - outInfo.SuballocationSizeMax = 0; + outInfo.AllocationSizeMin = UINT64_MAX; + outInfo.AllocationSizeMax = 0; outInfo.UnusedRangeSizeMin = UINT64_MAX; outInfo.UnusedRangeSizeMax = 0; @@ -3439,8 +4987,8 @@ static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc) const VmaSuballocation& suballoc = *suballocItem; if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) { - outInfo.SuballocationSizeMin = VMA_MIN(outInfo.SuballocationSizeMin, suballoc.size); - outInfo.SuballocationSizeMax = VMA_MAX(outInfo.SuballocationSizeMax, suballoc.size); + outInfo.AllocationSizeMin = VMA_MIN(outInfo.AllocationSizeMin, suballoc.size); + outInfo.AllocationSizeMax = VMA_MAX(outInfo.AllocationSizeMax, suballoc.size); } else { @@ -3453,33 +5001,94 @@ static void CalcAllocationStatInfo(VmaStatInfo& outInfo, const VmaBlock& alloc) // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo. static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo) { + inoutInfo.BlockCount += srcInfo.BlockCount; inoutInfo.AllocationCount += srcInfo.AllocationCount; - inoutInfo.SuballocationCount += srcInfo.SuballocationCount; inoutInfo.UnusedRangeCount += srcInfo.UnusedRangeCount; inoutInfo.UsedBytes += srcInfo.UsedBytes; inoutInfo.UnusedBytes += srcInfo.UnusedBytes; - inoutInfo.SuballocationSizeMin = VMA_MIN(inoutInfo.SuballocationSizeMin, srcInfo.SuballocationSizeMin); - inoutInfo.SuballocationSizeMax = VMA_MAX(inoutInfo.SuballocationSizeMax, srcInfo.SuballocationSizeMax); + inoutInfo.AllocationSizeMin = VMA_MIN(inoutInfo.AllocationSizeMin, srcInfo.AllocationSizeMin); + inoutInfo.AllocationSizeMax = VMA_MAX(inoutInfo.AllocationSizeMax, srcInfo.AllocationSizeMax); inoutInfo.UnusedRangeSizeMin = VMA_MIN(inoutInfo.UnusedRangeSizeMin, srcInfo.UnusedRangeSizeMin); inoutInfo.UnusedRangeSizeMax = VMA_MAX(inoutInfo.UnusedRangeSizeMax, srcInfo.UnusedRangeSizeMax); } static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo) { - inoutInfo.SuballocationSizeAvg = (inoutInfo.SuballocationCount > 0) ? - VmaRoundDiv(inoutInfo.UsedBytes, inoutInfo.SuballocationCount) : 0; + inoutInfo.AllocationSizeAvg = (inoutInfo.AllocationCount > 0) ? + VmaRoundDiv(inoutInfo.UsedBytes, inoutInfo.AllocationCount) : 0; inoutInfo.UnusedRangeSizeAvg = (inoutInfo.UnusedRangeCount > 0) ? VmaRoundDiv(inoutInfo.UnusedBytes, inoutInfo.UnusedRangeCount) : 0; } -VmaBlockVector::VmaBlockVector(VmaAllocator hAllocator) : +VmaPool_T::VmaPool_T( + VmaAllocator hAllocator, + const VmaPoolCreateInfo& createInfo) : + m_BlockVector( + hAllocator, + createInfo.memoryTypeIndex, + (createInfo.flags & VMA_POOL_CREATE_PERSISTENT_MAP_BIT) != 0 ? + VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED, + createInfo.blockSize, + createInfo.minBlockCount, + createInfo.maxBlockCount, + (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(), + createInfo.frameInUseCount, + true) // isCustomPool +{ +} + +VmaPool_T::~VmaPool_T() +{ +} + +#if VMA_STATS_STRING_ENABLED + +/* +void VmaPool_T::PrintDetailedMap(class VmaStringBuilder& sb) +{ + sb.Add("\n{\n\"This\": \""); + sb.AddPointer(this); + sb.Add("\",\n\"Flags\": "); + sb.AddNumber(m_Flags); + sb.Add(",\n\"FrameInUseCount\": "); + sb.AddNumber(m_FrameInUseCount); + sb.Add(",\n\"DeviceMemoryBlock\": "); + m_pDeviceMemoryBlock->PrintDetailedMap(sb); + sb.Add("\n}"); +} +*/ + +#endif // #if VMA_STATS_STRING_ENABLED + +VmaBlockVector::VmaBlockVector( + VmaAllocator hAllocator, + uint32_t memoryTypeIndex, + VMA_BLOCK_VECTOR_TYPE blockVectorType, + VkDeviceSize preferredBlockSize, + size_t minBlockCount, + size_t maxBlockCount, + VkDeviceSize bufferImageGranularity, + uint32_t frameInUseCount, + bool isCustomPool) : + m_Blocks(VmaStlAllocator(hAllocator->GetAllocationCallbacks())), m_hAllocator(hAllocator), - m_Blocks(VmaStlAllocator(hAllocator->GetAllocationCallbacks())) + m_MemoryTypeIndex(memoryTypeIndex), + m_BlockVectorType(blockVectorType), + m_PreferredBlockSize(preferredBlockSize), + m_MinBlockCount(minBlockCount), + m_MaxBlockCount(maxBlockCount), + m_BufferImageGranularity(bufferImageGranularity), + m_FrameInUseCount(frameInUseCount), + m_IsCustomPool(isCustomPool), + m_HasEmptyBlock(false), + m_pDefragmentator(VMA_NULL) { } VmaBlockVector::~VmaBlockVector() { + VMA_ASSERT(m_pDefragmentator == VMA_NULL); + for(size_t i = m_Blocks.size(); i--; ) { m_Blocks[i]->Destroy(m_hAllocator); @@ -3487,7 +5096,303 @@ VmaBlockVector::~VmaBlockVector() } } -void VmaBlockVector::Remove(VmaBlock* pBlock) +VkResult VmaBlockVector::CreateMinBlocks() +{ + for(size_t i = 0; i < m_MinBlockCount; ++i) + { + VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL); + if(res != VK_SUCCESS) + { + return res; + } + } + return VK_SUCCESS; +} + +void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats) +{ + pStats->size = 0; + pStats->unusedSize = 0; + pStats->allocationCount = 0; + pStats->unusedRangeCount = 0; + + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + { + const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pBlock); + VMA_HEAVY_ASSERT(pBlock->Validate()); + + const uint32_t rangeCount = (uint32_t)pBlock->m_Suballocations.size(); + + pStats->size += pBlock->m_Size; + pStats->unusedSize += pBlock->m_SumFreeSize; + pStats->allocationCount += rangeCount - pBlock->m_FreeCount; + pStats->unusedRangeCount += pBlock->m_FreeCount; + } +} + +static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32; + +VkResult VmaBlockVector::Allocate( + VmaPool hCurrentPool, + uint32_t currentFrameIndex, + const VkMemoryRequirements& vkMemReq, + const VmaAllocationCreateInfo& createInfo, + VmaSuballocationType suballocType, + VmaAllocation* pAllocation) +{ + // Validate flags. + if(((createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) != + (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED)) + { + VMA_ASSERT(0 && "Usage of VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT must match VMA_POOL_CREATE_PERSISTENT_MAP_BIT."); + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + // 1. Search existing allocations. Try to allocate without making other allocations lost. + // Forward order in m_Blocks - prefer blocks with smallest amount of free space. + for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) + { + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VmaAllocationRequest currRequest = {}; + if(pCurrBlock->CreateAllocationRequest( + currentFrameIndex, + m_FrameInUseCount, + m_BufferImageGranularity, + vkMemReq.size, + vkMemReq.alignment, + suballocType, + false, // canMakeOtherLost + &currRequest)) + { + // Allocate from pCurrBlock. + VMA_ASSERT(currRequest.itemsToMakeLostCount == 0); + + // We no longer have an empty Allocation. + if(pCurrBlock->IsEmpty()) + { + m_HasEmptyBlock = false; + } + + *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex); + pCurrBlock->Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation); + (*pAllocation)->InitBlockAllocation( + hCurrentPool, + pCurrBlock, + currRequest.offset, + vkMemReq.alignment, + vkMemReq.size, + suballocType, + createInfo.pUserData, + (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); + VMA_HEAVY_ASSERT(pCurrBlock->Validate()); + VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex); + return VK_SUCCESS; + } + } + + const bool canCreateNewBlock = + ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) && + (m_Blocks.size() < m_MaxBlockCount); + + // 2. Try to create new block. + if(canCreateNewBlock) + { + // 2.1. Start with full preferredBlockSize. + VkDeviceSize blockSize = m_PreferredBlockSize; + size_t newBlockIndex = 0; + VkResult res = CreateBlock(blockSize, &newBlockIndex); + // Allocating blocks of other sizes is allowed only in default pools. + // In custom pools block size is fixed. + if(res < 0 && m_IsCustomPool == false) + { + // 2.2. Try half the size. + blockSize /= 2; + if(blockSize >= vkMemReq.size) + { + res = CreateBlock(blockSize, &newBlockIndex); + if(res < 0) + { + // 2.3. Try quarter the size. + blockSize /= 2; + if(blockSize >= vkMemReq.size) + { + res = CreateBlock(blockSize, &newBlockIndex); + } + } + } + } + if(res == VK_SUCCESS) + { + VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex]; + VMA_ASSERT(pBlock->m_Size >= vkMemReq.size); + + // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled. + VmaAllocationRequest allocRequest = {}; + allocRequest.item = pBlock->m_Suballocations.begin(); + allocRequest.offset = 0; + *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex); + pBlock->Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation); + (*pAllocation)->InitBlockAllocation( + hCurrentPool, + pBlock, + allocRequest.offset, + vkMemReq.alignment, + vkMemReq.size, + suballocType, + createInfo.pUserData, + (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); + VMA_HEAVY_ASSERT(pBlock->Validate()); + VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize); + + return VK_SUCCESS; + } + } + + const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0; + + // 3. Try to allocate from existing blocks with making other allocations lost. + if(canMakeOtherLost) + { + uint32_t tryIndex = 0; + for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex) + { + VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL; + VmaAllocationRequest bestRequest = {}; + VkDeviceSize bestRequestCost = VK_WHOLE_SIZE; + + // 1. Search existing allocations. + // Forward order in m_Blocks - prefer blocks with smallest amount of free space. + for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex ) + { + VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pCurrBlock); + VmaAllocationRequest currRequest = {}; + if(pCurrBlock->CreateAllocationRequest( + currentFrameIndex, + m_FrameInUseCount, + m_BufferImageGranularity, + vkMemReq.size, + vkMemReq.alignment, + suballocType, + canMakeOtherLost, + &currRequest)) + { + const VkDeviceSize currRequestCost = currRequest.CalcCost(); + if(pBestRequestBlock == VMA_NULL || + currRequestCost < bestRequestCost) + { + pBestRequestBlock = pCurrBlock; + bestRequest = currRequest; + bestRequestCost = currRequestCost; + + if(bestRequestCost == 0) + { + break; + } + } + } + } + + if(pBestRequestBlock != VMA_NULL) + { + if(pBestRequestBlock->MakeRequestedAllocationsLost( + currentFrameIndex, + m_FrameInUseCount, + &bestRequest)) + { + // We no longer have an empty Allocation. + if(pBestRequestBlock->IsEmpty()) + { + m_HasEmptyBlock = false; + } + // Allocate from this pBlock. + *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex); + pBestRequestBlock->Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation); + (*pAllocation)->InitBlockAllocation( + hCurrentPool, + pBestRequestBlock, + bestRequest.offset, + vkMemReq.alignment, + vkMemReq.size, + suballocType, + createInfo.pUserData, + (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0); + VMA_HEAVY_ASSERT(pBlock->Validate()); + VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex); + return VK_SUCCESS; + } + // else: Some allocations must have been touched while we are here. Next try. + } + else + { + // Could not find place in any of the blocks - break outer loop. + break; + } + } + /* Maximum number of tries exceeded - a very unlike event when many other + threads are simultaneously touching allocations making it impossible to make + lost at the same time as we try to allocate. */ + if(tryIndex == VMA_ALLOCATION_TRY_COUNT) + { + return VK_ERROR_TOO_MANY_OBJECTS; + } + } + + return VK_ERROR_OUT_OF_DEVICE_MEMORY; +} + +void VmaBlockVector::Free( + VmaAllocation hAllocation) +{ + VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL; + + // Scope for lock. + { + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock(); + + pBlock->Free(hAllocation); + VMA_HEAVY_ASSERT(pBlock->Validate()); + + VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex); + + // pBlock became empty after this deallocation. + if(pBlock->IsEmpty()) + { + // Already has empty Allocation. We don't want to have two, so delete this one. + if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount) + { + pBlockToDelete = pBlock; + Remove(pBlock); + } + // We now have first empty Allocation. + else + { + m_HasEmptyBlock = true; + } + } + // Must be called after srcBlockIndex is used, because later it may become invalid! + IncrementallySortBlocks(); + } + + // Destruction of a free Allocation. Deferred until this point, outside of mutex + // lock, for performance reason. + if(pBlockToDelete != VMA_NULL) + { + VMA_DEBUG_LOG(" Deleted empty allocation"); + pBlockToDelete->Destroy(m_hAllocator); + vma_delete(m_hAllocator, pBlockToDelete); + } +} + +void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock) { for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) { @@ -3513,31 +5418,82 @@ void VmaBlockVector::IncrementallySortBlocks() } } +VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex) +{ + VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; + allocInfo.memoryTypeIndex = m_MemoryTypeIndex; + allocInfo.allocationSize = blockSize; + const VkDevice hDevice = m_hAllocator->m_hDevice; + VkDeviceMemory mem = VK_NULL_HANDLE; + VkResult res = vkAllocateMemory(hDevice, &allocInfo, m_hAllocator->GetAllocationCallbacks(), &mem); + if(res < 0) + { + return res; + } + + // New VkDeviceMemory successfully created. + + // Map memory if needed. + void* pMappedData = VMA_NULL; + const bool persistentMap = (m_BlockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED); + if(persistentMap && m_hAllocator->m_UnmapPersistentlyMappedMemoryCounter == 0) + { + res = vkMapMemory(hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData); + if(res < 0) + { + VMA_DEBUG_LOG(" vkMapMemory FAILED"); + vkFreeMemory(hDevice, mem, m_hAllocator->GetAllocationCallbacks()); + return res; + } + } + + // Callback. + if(m_hAllocator->m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL) + { + (*m_hAllocator->m_DeviceMemoryCallbacks.pfnAllocate)(m_hAllocator, m_MemoryTypeIndex, mem, allocInfo.allocationSize); + } + + // Create new Allocation for it. + VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator); + pBlock->Init( + m_MemoryTypeIndex, + (VMA_BLOCK_VECTOR_TYPE)m_BlockVectorType, + mem, + allocInfo.allocationSize, + persistentMap, + pMappedData); + + m_Blocks.push_back(pBlock); + if(pNewBlockIndex != VMA_NULL) + { + *pNewBlockIndex = m_Blocks.size() - 1; + } + + return VK_SUCCESS; +} + #if VMA_STATS_STRING_ENABLED -void VmaBlockVector::PrintDetailedMap(class VmaStringBuilder& sb) const +void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) { + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + json.BeginArray(); for(size_t i = 0; i < m_Blocks.size(); ++i) { - if(i > 0) - { - sb.Add(",\n\t\t"); - } - else - { - sb.Add("\n\t\t"); - } - m_Blocks[i]->PrintDetailedMap(sb); + m_Blocks[i]->PrintDetailedMap(json); } + json.EndArray(); } #endif // #if VMA_STATS_STRING_ENABLED void VmaBlockVector::UnmapPersistentlyMappedMemory() { + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + for(size_t i = m_Blocks.size(); i--; ) { - VmaBlock* pBlock = m_Blocks[i]; + VmaDeviceMemoryBlock* pBlock = m_Blocks[i]; if(pBlock->m_pMappedData != VMA_NULL) { VMA_ASSERT(pBlock->m_PersistentMap != false); @@ -3549,10 +5505,12 @@ void VmaBlockVector::UnmapPersistentlyMappedMemory() VkResult VmaBlockVector::MapPersistentlyMappedMemory() { + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + VkResult finalResult = VK_SUCCESS; for(size_t i = 0, count = m_Blocks.size(); i < count; ++i) { - VmaBlock* pBlock = m_Blocks[i]; + VmaDeviceMemoryBlock* pBlock = m_Blocks[i]; if(pBlock->m_PersistentMap) { VMA_ASSERT(pBlock->m_pMappedData == nullptr); @@ -3566,11 +5524,110 @@ VkResult VmaBlockVector::MapPersistentlyMappedMemory() return finalResult; } -void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) const +VmaDefragmentator* VmaBlockVector::EnsureDefragmentator( + VkDevice hDevice, + const VkAllocationCallbacks* pAllocationCallbacks, + uint32_t currentFrameIndex) { - for(uint32_t allocIndex = 0; allocIndex < m_Blocks.size(); ++allocIndex) + if(m_pDefragmentator == VMA_NULL) { - const VmaBlock* const pBlock = m_Blocks[allocIndex]; + m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)( + hDevice, + pAllocationCallbacks, + this, + currentFrameIndex); + } + + return m_pDefragmentator; +} + +VkResult VmaBlockVector::Defragment( + VmaDefragmentationStats* pDefragmentationStats, + VkDeviceSize& maxBytesToMove, + uint32_t& maxAllocationsToMove) +{ + if(m_pDefragmentator == VMA_NULL) + { + return VK_SUCCESS; + } + + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + // Defragment. + VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove); + + // Accumulate statistics. + if(pDefragmentationStats != VMA_NULL) + { + const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved(); + const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved(); + pDefragmentationStats->bytesMoved += bytesMoved; + pDefragmentationStats->allocationsMoved += allocationsMoved; + VMA_ASSERT(bytesMoved <= maxBytesToMove); + VMA_ASSERT(allocationsMoved <= maxAllocationsToMove); + maxBytesToMove -= bytesMoved; + maxAllocationsToMove -= allocationsMoved; + } + + // Free empty blocks. + m_HasEmptyBlock = false; + for(size_t blockIndex = m_Blocks.size(); blockIndex--; ) + { + VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex]; + if(pBlock->IsEmpty()) + { + if(m_Blocks.size() > m_MinBlockCount) + { + if(pDefragmentationStats != VMA_NULL) + { + ++pDefragmentationStats->deviceMemoryBlocksFreed; + pDefragmentationStats->bytesFreed += pBlock->m_Size; + } + + VmaVectorRemove(m_Blocks, blockIndex); + pBlock->Destroy(m_hAllocator); + vma_delete(m_hAllocator, pBlock); + } + else + { + m_HasEmptyBlock = true; + } + } + } + + return result; +} + +void VmaBlockVector::DestroyDefragmentator() +{ + if(m_pDefragmentator != VMA_NULL) + { + vma_delete(m_hAllocator, m_pDefragmentator); + m_pDefragmentator = VMA_NULL; + } +} + +void VmaBlockVector::MakePoolAllocationsLost( + uint32_t currentFrameIndex, + size_t* pLostAllocationCount) +{ + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + { + VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; + VMA_ASSERT(pBlock); + pBlock->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount); + } +} + +void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t memHeapIndex) +{ + VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex); + + for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex) + { + const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex]; VMA_ASSERT(pBlock); VMA_HEAVY_ASSERT(pBlock->Validate()); VmaStatInfo allocationStatInfo; @@ -3582,181 +5639,17 @@ void VmaBlockVector::AddStats(VmaStats* pStats, uint32_t memTypeIndex, uint32_t } //////////////////////////////////////////////////////////////////////////////// -// VmaDefragmentator - -class VmaDefragmentator -{ - VkDevice m_hDevice; - const VkAllocationCallbacks* m_pAllocationCallbacks; - VkDeviceSize m_BufferImageGranularity; - uint32_t m_MemTypeIndex; - VMA_BLOCK_VECTOR_TYPE m_BlockVectorType; - VkDeviceSize m_BytesMoved; - uint32_t m_AllocationsMoved; - - struct AllocationInfo - { - VmaAllocation m_hAllocation; - VkBool32* m_pChanged; - - AllocationInfo() : - m_hAllocation(VK_NULL_HANDLE), - m_pChanged(VMA_NULL) - { - } - }; - - struct AllocationInfoSizeGreater - { - bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const - { - return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize(); - } - }; - - // Used between AddAllocation and Defragment. - VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; - - struct BlockInfo - { - VmaBlock* m_pBlock; - bool m_HasNonMovableAllocations; - VmaVector< AllocationInfo, VmaStlAllocator > m_Allocations; - - BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) : - m_pBlock(VMA_NULL), - m_HasNonMovableAllocations(true), - m_Allocations(pAllocationCallbacks), - m_pMappedDataForDefragmentation(VMA_NULL) - { - } - - void CalcHasNonMovableAllocations() - { - const size_t blockAllocCount = - m_pBlock->m_Suballocations.size() - m_pBlock->m_FreeCount; - const size_t defragmentAllocCount = m_Allocations.size(); - m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount; - } - - void SortAllocationsBySizeDescecnding() - { - VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater()); - } - - VkResult EnsureMapping(VkDevice hDevice, void** ppMappedData) - { - // It has already been mapped for defragmentation. - if(m_pMappedDataForDefragmentation) - { - *ppMappedData = m_pMappedDataForDefragmentation; - return VK_SUCCESS; - } - - // It is persistently mapped. - if(m_pBlock->m_PersistentMap) - { - VMA_ASSERT(m_pBlock->m_pMappedData != VMA_NULL); - *ppMappedData = m_pBlock->m_pMappedData; - return VK_SUCCESS; - } - - // Map on first usage. - VkResult res = vkMapMemory(hDevice, m_pBlock->m_hMemory, 0, VK_WHOLE_SIZE, 0, &m_pMappedDataForDefragmentation); - *ppMappedData = m_pMappedDataForDefragmentation; - return res; - } - - void Unmap(VkDevice hDevice) - { - if(m_pMappedDataForDefragmentation != VMA_NULL) - { - vkUnmapMemory(hDevice, m_pBlock->m_hMemory); - } - } - - private: - // Not null if mapped for defragmentation only, not persistently mapped. - void* m_pMappedDataForDefragmentation; - }; - - struct BlockPointerLess - { - bool operator()(const BlockInfo* pLhsBlockInfo, const VmaBlock* pRhsBlock) const - { - return pLhsBlockInfo->m_pBlock < pRhsBlock; - } - bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const - { - return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock; - } - }; - - // 1. Blocks with some non-movable allocations go first. - // 2. Blocks with smaller sumFreeSize go first. - struct BlockInfoCompareMoveDestination - { - bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const - { - if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations) - { - return true; - } - if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations) - { - return false; - } - if(pLhsBlockInfo->m_pBlock->m_SumFreeSize < pRhsBlockInfo->m_pBlock->m_SumFreeSize) - { - return true; - } - return false; - } - }; - - typedef VmaVector< BlockInfo*, VmaStlAllocator > BlockInfoVector; - BlockInfoVector m_Blocks; - - VkResult DefragmentRound( - VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); - - static bool MoveMakesSense( - size_t dstBlockIndex, VkDeviceSize dstOffset, - size_t srcBlockIndex, VkDeviceSize srcOffset); - -public: - VmaDefragmentator( - VkDevice hDevice, - const VkAllocationCallbacks* pAllocationCallbacks, - VkDeviceSize bufferImageGranularity, - uint32_t memTypeIndex, - VMA_BLOCK_VECTOR_TYPE blockVectorType); - - ~VmaDefragmentator(); - - VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } - uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } - - void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged); - - VkResult Defragment( - VmaBlockVector* pBlockVector, - VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); -}; +// VmaDefragmentator members definition VmaDefragmentator::VmaDefragmentator( VkDevice hDevice, const VkAllocationCallbacks* pAllocationCallbacks, - VkDeviceSize bufferImageGranularity, - uint32_t memTypeIndex, - VMA_BLOCK_VECTOR_TYPE blockVectorType) : + VmaBlockVector* pBlockVector, + uint32_t currentFrameIndex) : m_hDevice(hDevice), m_pAllocationCallbacks(pAllocationCallbacks), - m_BufferImageGranularity(bufferImageGranularity), - m_MemTypeIndex(memTypeIndex), - m_BlockVectorType(blockVectorType), + m_pBlockVector(pBlockVector), + m_CurrentFrameIndex(currentFrameIndex), m_BytesMoved(0), m_AllocationsMoved(0), m_Allocations(VmaStlAllocator(pAllocationCallbacks)), @@ -3831,14 +5724,19 @@ VkResult VmaDefragmentator::DefragmentRound( BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex]; VmaAllocationRequest dstAllocRequest; if(pDstBlockInfo->m_pBlock->CreateAllocationRequest( - m_BufferImageGranularity, + m_CurrentFrameIndex, + m_pBlockVector->GetFrameInUseCount(), + m_pBlockVector->GetBufferImageGranularity(), size, alignment, suballocType, + false, // canMakeOtherLost &dstAllocRequest) && MoveMakesSense( dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset)) { + VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0); + // Reached limit on number of allocations or bytes to move. if((m_AllocationsMoved + 1 > maxAllocationsToMove) || (m_BytesMoved + size > maxBytesToMove)) @@ -3866,7 +5764,7 @@ VkResult VmaDefragmentator::DefragmentRound( reinterpret_cast(pSrcMappedData) + srcOffset, size); - pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size); + pDstBlockInfo->m_pBlock->Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation); pSrcBlockInfo->m_pBlock->Free(allocInfo.m_hAllocation); allocInfo.m_hAllocation->ChangeBlockAllocation(pDstBlockInfo->m_pBlock, dstAllocRequest.offset); @@ -3907,7 +5805,6 @@ VkResult VmaDefragmentator::DefragmentRound( } VkResult VmaDefragmentator::Defragment( - VmaBlockVector* pBlockVector, VkDeviceSize maxBytesToMove, uint32_t maxAllocationsToMove) { @@ -3917,30 +5814,34 @@ VkResult VmaDefragmentator::Defragment( } // Create block info for each block. - const size_t blockCount = pBlockVector->m_Blocks.size(); + const size_t blockCount = m_pBlockVector->m_Blocks.size(); for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex) { BlockInfo* pBlockInfo = vma_new(m_pAllocationCallbacks, BlockInfo)(m_pAllocationCallbacks); - pBlockInfo->m_pBlock = pBlockVector->m_Blocks[blockIndex]; + pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex]; m_Blocks.push_back(pBlockInfo); } // Sort them by m_pBlock pointer value. VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess()); - // Move allocation infos from m_Allocations to appropriate m_Blocks[i].m_Allocations. - for(size_t allocIndex = 0, allocCount = m_Allocations.size(); allocIndex < allocCount; ++allocIndex) + // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations. + for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex) { - AllocationInfo& allocInfo = m_Allocations[allocIndex]; - VmaBlock* pBlock = allocInfo.m_hAllocation->GetBlock(); - BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess()); - if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock) + AllocationInfo& allocInfo = m_Allocations[blockIndex]; + // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost. + if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST) { - (*it)->m_Allocations.push_back(allocInfo); - } - else - { - VMA_ASSERT(0); + VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock(); + BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess()); + if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock) + { + (*it)->m_Allocations.push_back(allocInfo); + } + else + { + VMA_ASSERT(0); + } } } m_Allocations.clear(); @@ -3955,7 +5856,7 @@ VkResult VmaDefragmentator::Defragment( // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks. VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination()); - // Execute defragmentation round (the main part). + // Execute defragmentation rounds (the main part). VkResult result = VK_SUCCESS; for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round) { @@ -4000,9 +5901,11 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL), m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ? *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks), + m_UnmapPersistentlyMappedMemoryCounter(0), m_PreferredLargeHeapBlockSize(0), m_PreferredSmallHeapBlockSize(0), - m_UnmapPersistentlyMappedMemoryCounter(0) + m_CurrentFrameIndex(0), + m_Pools(VmaStlAllocator(GetAllocationCallbacks())) { VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device); @@ -4011,7 +5914,6 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties)); memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors)); - memset(&m_HasEmptyBlock, 0, sizeof(m_HasEmptyBlock)); memset(&m_pOwnAllocations, 0, sizeof(m_pOwnAllocations)); if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL) @@ -4020,26 +5922,41 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree; } + vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties); + vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps); + m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ? pCreateInfo->preferredLargeHeapBlockSize : static_cast(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE); m_PreferredSmallHeapBlockSize = (pCreateInfo->preferredSmallHeapBlockSize != 0) ? pCreateInfo->preferredSmallHeapBlockSize : static_cast(VMA_DEFAULT_SMALL_HEAP_BLOCK_SIZE); - vkGetPhysicalDeviceProperties(m_PhysicalDevice, &m_PhysicalDeviceProperties); - vkGetPhysicalDeviceMemoryProperties(m_PhysicalDevice, &m_MemProps); - - for(size_t i = 0; i < GetMemoryTypeCount(); ++i) + for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { - for(size_t j = 0; j < VMA_BLOCK_VECTOR_TYPE_COUNT; ++j) + const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex); + + for(size_t blockVectorTypeIndex = 0; blockVectorTypeIndex < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorTypeIndex) { - m_pBlockVectors[i][j] = vma_new(this, VmaBlockVector)(this); - m_pOwnAllocations[i][j] = vma_new(this, AllocationVectorType)(VmaStlAllocator(GetAllocationCallbacks())); + m_pBlockVectors[memTypeIndex][blockVectorTypeIndex] = vma_new(this, VmaBlockVector)( + this, + memTypeIndex, + static_cast(blockVectorTypeIndex), + preferredBlockSize, + 0, + SIZE_MAX, + GetBufferImageGranularity(), + pCreateInfo->frameInUseCount, + false); // isCustomPool + // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here, + // becase minBlockCount is 0. + m_pOwnAllocations[memTypeIndex][blockVectorTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator(GetAllocationCallbacks())); } } } VmaAllocator_T::~VmaAllocator_T() { + VMA_ASSERT(m_Pools.empty()); + for(size_t i = GetMemoryTypeCount(); i--; ) { for(size_t j = VMA_BLOCK_VECTOR_TYPE_COUNT; j--; ) @@ -4050,16 +5967,16 @@ VmaAllocator_T::~VmaAllocator_T() } } -VkDeviceSize VmaAllocator_T::GetPreferredBlockSize(uint32_t memTypeIndex) const +VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex) { - VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size; + const VkDeviceSize heapSize = m_MemProps.memoryHeaps[m_MemProps.memoryTypes[memTypeIndex].heapIndex].size; return (heapSize <= VMA_SMALL_HEAP_MAX_SIZE) ? m_PreferredSmallHeapBlockSize : m_PreferredLargeHeapBlockSize; } VkResult VmaAllocator_T::AllocateMemoryOfType( const VkMemoryRequirements& vkMemReq, - const VmaMemoryRequirements& vmaMemReq, + const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, VmaSuballocationType suballocType, VmaAllocation* pAllocation) @@ -4067,17 +5984,21 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VMA_ASSERT(pAllocation != VMA_NULL); VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size); - const VkDeviceSize preferredBlockSize = GetPreferredBlockSize(memTypeIndex); + uint32_t blockVectorType = VmaAllocationCreateFlagsToBlockVectorType(createInfo.flags); + VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; + VMA_ASSERT(blockVector); + + const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize(); // Heuristics: Allocate own memory if requested size if greater than half of preferred block size. const bool ownMemory = - (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 || + (createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 || VMA_DEBUG_ALWAYS_OWN_MEMORY || - ((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) == 0 && + ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 && vkMemReq.size > preferredBlockSize / 2); if(ownMemory) { - if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0) + if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) { return VK_ERROR_OUT_OF_DEVICE_MEMORY; } @@ -4087,161 +6008,44 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( vkMemReq.size, suballocType, memTypeIndex, - (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0, - vmaMemReq.pUserData, + (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, + createInfo.pUserData, pAllocation); } } else { - uint32_t blockVectorType = VmaMemoryRequirementFlagsToBlockVectorType(vmaMemReq.flags); - - VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex); - VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; - VMA_ASSERT(blockVector); - - // 1. Search existing allocations. - // Forward order - prefer blocks with smallest amount of free space. - for(size_t allocIndex = 0; allocIndex < blockVector->m_Blocks.size(); ++allocIndex ) + VkResult res = blockVector->Allocate( + VK_NULL_HANDLE, // hCurrentPool + m_CurrentFrameIndex.load(), + vkMemReq, + createInfo, + suballocType, + pAllocation); + if(res == VK_SUCCESS) { - VmaBlock* const pBlock = blockVector->m_Blocks[allocIndex]; - VMA_ASSERT(pBlock); - VmaAllocationRequest allocRequest = {}; - // Check if can allocate from pBlock. - if(pBlock->CreateAllocationRequest( - GetBufferImageGranularity(), - vkMemReq.size, - vkMemReq.alignment, - suballocType, - &allocRequest)) - { - // We no longer have an empty Allocation. - if(pBlock->IsEmpty()) - { - m_HasEmptyBlock[memTypeIndex] = false; - } - // Allocate from this pBlock. - pBlock->Alloc(allocRequest, suballocType, vkMemReq.size); - *pAllocation = vma_new(this, VmaAllocation_T)(); - (*pAllocation)->InitBlockAllocation( - pBlock, - allocRequest.offset, - vkMemReq.alignment, - vkMemReq.size, - suballocType, - vmaMemReq.pUserData); - VMA_HEAVY_ASSERT(pBlock->Validate()); - VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)allocIndex); - return VK_SUCCESS; - } + return res; } - // 2. Create new Allocation. - if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0) + // 5. Try own memory. + res = AllocateOwnMemory( + vkMemReq.size, + suballocType, + memTypeIndex, + (createInfo.flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0, + createInfo.pUserData, + pAllocation); + if(res == VK_SUCCESS) { - VMA_DEBUG_LOG(" FAILED due to VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT"); - return VK_ERROR_OUT_OF_DEVICE_MEMORY; + // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here. + VMA_DEBUG_LOG(" Allocated as OwnMemory"); + return VK_SUCCESS; } else { - // Start with full preferredBlockSize. - VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; - allocInfo.memoryTypeIndex = memTypeIndex; - allocInfo.allocationSize = preferredBlockSize; - VkDeviceMemory mem = VK_NULL_HANDLE; - VkResult res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem); - if(res < 0) - { - // 3. Try half the size. - allocInfo.allocationSize /= 2; - if(allocInfo.allocationSize >= vkMemReq.size) - { - res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem); - if(res < 0) - { - // 4. Try quarter the size. - allocInfo.allocationSize /= 2; - if(allocInfo.allocationSize >= vkMemReq.size) - { - res = vkAllocateMemory(m_hDevice, &allocInfo, GetAllocationCallbacks(), &mem); - } - } - } - } - if(res < 0) - { - // 5. Try OwnAlloc. - res = AllocateOwnMemory( - vkMemReq.size, - suballocType, - memTypeIndex, - (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0, - vmaMemReq.pUserData, - pAllocation); - if(res == VK_SUCCESS) - { - // Succeeded: AllocateOwnMemory function already filld pMemory, nothing more to do here. - VMA_DEBUG_LOG(" Allocated as OwnMemory"); - return VK_SUCCESS; - } - else - { - // Everything failed: Return error code. - VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); - return res; - } - } - - // New VkDeviceMemory successfully created. - - // Map memory if needed. - void* pMappedData = VMA_NULL; - const bool persistentMap = (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0; - if(persistentMap && m_UnmapPersistentlyMappedMemoryCounter == 0) - { - res = vkMapMemory(m_hDevice, mem, 0, VK_WHOLE_SIZE, 0, &pMappedData); - if(res < 0) - { - VMA_DEBUG_LOG(" vkMapMemory FAILED"); - vkFreeMemory(m_hDevice, mem, GetAllocationCallbacks()); - return res; - } - } - - // Callback. - if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL) - { - (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, mem, allocInfo.allocationSize); - } - - // Create new Allocation for it. - VmaBlock* const pBlock = vma_new(this, VmaBlock)(this); - pBlock->Init( - memTypeIndex, - (VMA_BLOCK_VECTOR_TYPE)blockVectorType, - mem, - allocInfo.allocationSize, - persistentMap, - pMappedData); - - blockVector->m_Blocks.push_back(pBlock); - - // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled. - VmaAllocationRequest allocRequest = {}; - allocRequest.freeSuballocationItem = pBlock->m_Suballocations.begin(); - allocRequest.offset = 0; - pBlock->Alloc(allocRequest, suballocType, vkMemReq.size); - *pAllocation = vma_new(this, VmaAllocation_T)(); - (*pAllocation)->InitBlockAllocation( - pBlock, - allocRequest.offset, - vkMemReq.alignment, - vkMemReq.size, - suballocType, - vmaMemReq.pUserData); - VMA_HEAVY_ASSERT(pBlock->Validate()); - VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize); - return VK_SUCCESS; + // Everything failed: Return error code. + VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); + return res; } } } @@ -4290,7 +6094,7 @@ VkResult VmaAllocator_T::AllocateOwnMemory( (*m_DeviceMemoryCallbacks.pfnAllocate)(this, memTypeIndex, hMemory, size); } - *pAllocation = vma_new(this, VmaAllocation_T)(); + *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load()); (*pAllocation)->InitOwnAllocation(memTypeIndex, hMemory, suballocType, map, pMappedData, size, pUserData); // Register it in m_pOwnAllocations. @@ -4298,14 +6102,7 @@ VkResult VmaAllocator_T::AllocateOwnMemory( VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex); AllocationVectorType* pOwnAllocations = m_pOwnAllocations[memTypeIndex][map ? VMA_BLOCK_VECTOR_TYPE_MAPPED : VMA_BLOCK_VECTOR_TYPE_UNMAPPED]; VMA_ASSERT(pOwnAllocations); - VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data(); - VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size(); - const size_t indexToInsert = VmaBinaryFindFirstNotLess( - pOwnAllocationsBeg, - pOwnAllocationsEnd, - *pAllocation, - VmaPointerLess()) - pOwnAllocationsBeg; - VmaVectorInsert(*pOwnAllocations, indexToInsert, *pAllocation); + VmaVectorInsertSorted(*pOwnAllocations, *pAllocation); } VMA_DEBUG_LOG(" Allocated OwnMemory MemoryTypeIndex=#%u", memTypeIndex); @@ -4315,116 +6112,116 @@ VkResult VmaAllocator_T::AllocateOwnMemory( VkResult VmaAllocator_T::AllocateMemory( const VkMemoryRequirements& vkMemReq, - const VmaMemoryRequirements& vmaMemReq, + const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, VmaAllocation* pAllocation) { - if((vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT) != 0 && - (vmaMemReq.flags & VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT) != 0) + if((createInfo.flags & VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT) != 0 && + (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0) { - VMA_ASSERT(0 && "Specifying VMA_MEMORY_REQUIREMENT_OWN_MEMORY_BIT together with VMA_MEMORY_REQUIREMENT_NEVER_ALLOCATE_BIT makes no sense."); + VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense."); + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + if((createInfo.pool != VK_NULL_HANDLE) && + ((createInfo.flags & (VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT)) != 0)) + { + VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_OWN_MEMORY_BIT when pool != null is invalid."); return VK_ERROR_OUT_OF_DEVICE_MEMORY; } - // Bit mask of memory Vulkan types acceptable for this allocation. - uint32_t memoryTypeBits = vkMemReq.memoryTypeBits; - uint32_t memTypeIndex = UINT32_MAX; - VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex); - if(res == VK_SUCCESS) + if(createInfo.pool != VK_NULL_HANDLE) { - res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation); - // Succeeded on first try. + return createInfo.pool->m_BlockVector.Allocate( + createInfo.pool, + m_CurrentFrameIndex.load(), + vkMemReq, + createInfo, + suballocType, + pAllocation); + } + else + { + // Bit mask of memory Vulkan types acceptable for this allocation. + uint32_t memoryTypeBits = vkMemReq.memoryTypeBits; + uint32_t memTypeIndex = UINT32_MAX; + VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex); if(res == VK_SUCCESS) { - return res; - } - // Allocation from this memory type failed. Try other compatible memory types. - else - { - for(;;) + res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation); + // Succeeded on first try. + if(res == VK_SUCCESS) { - // Remove old memTypeIndex from list of possibilities. - memoryTypeBits &= ~(1u << memTypeIndex); - // Find alternative memTypeIndex. - res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &vmaMemReq, &memTypeIndex); - if(res == VK_SUCCESS) + return res; + } + // Allocation from this memory type failed. Try other compatible memory types. + else + { + for(;;) { - res = AllocateMemoryOfType(vkMemReq, vmaMemReq, memTypeIndex, suballocType, pAllocation); - // Allocation from this alternative memory type succeeded. + // Remove old memTypeIndex from list of possibilities. + memoryTypeBits &= ~(1u << memTypeIndex); + // Find alternative memTypeIndex. + res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex); if(res == VK_SUCCESS) { - return res; + res = AllocateMemoryOfType(vkMemReq, createInfo, memTypeIndex, suballocType, pAllocation); + // Allocation from this alternative memory type succeeded. + if(res == VK_SUCCESS) + { + return res; + } + // else: Allocation from this memory type failed. Try next one - next loop iteration. + } + // No other matching memory type index could be found. + else + { + // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once. + return VK_ERROR_OUT_OF_DEVICE_MEMORY; } - // else: Allocation from this memory type failed. Try next one - next loop iteration. - } - // No other matching memory type index could be found. - else - { - // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once. - return VK_ERROR_OUT_OF_DEVICE_MEMORY; } } } + // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT. + else + return res; } - // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT. - else - return res; } void VmaAllocator_T::FreeMemory(const VmaAllocation allocation) { VMA_ASSERT(allocation); - if(allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) + if(allocation->CanBecomeLost() == false || + allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST) { - VmaBlock* pBlockToDelete = VMA_NULL; - - const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); - const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType(); + switch(allocation->GetType()) { - VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex); - - VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; - VmaBlock* pBlock = allocation->GetBlock(); - - pBlock->Free(allocation); - VMA_HEAVY_ASSERT(pBlock->Validate()); - - VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex); - - // pBlock became empty after this deallocation. - if(pBlock->IsEmpty()) + case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: { - // Already has empty Allocation. We don't want to have two, so delete this one. - if(m_HasEmptyBlock[memTypeIndex]) + VmaBlockVector* pBlockVector = VMA_NULL; + VmaPool hPool = allocation->GetPool(); + if(hPool != VK_NULL_HANDLE) { - pBlockToDelete = pBlock; - pBlockVector->Remove(pBlock); + pBlockVector = &hPool->m_BlockVector; } - // We now have first empty Allocation. else { - m_HasEmptyBlock[memTypeIndex] = true; + const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); + const VMA_BLOCK_VECTOR_TYPE blockVectorType = allocation->GetBlockVectorType(); + pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; } + pBlockVector->Free(allocation); } - // Must be called after srcBlockIndex is used, because later it may become invalid! - pBlockVector->IncrementallySortBlocks(); - } - // Destruction of a free Allocation. Deferred until this point, outside of mutex - // lock, for performance reason. - if(pBlockToDelete != VMA_NULL) - { - VMA_DEBUG_LOG(" Deleted empty allocation"); - pBlockToDelete->Destroy(this); - vma_delete(this, pBlockToDelete); + break; + case VmaAllocation_T::ALLOCATION_TYPE_OWN: + FreeOwnMemory(allocation); + break; + default: + VMA_ASSERT(0); } + } - vma_delete(this, allocation); - } - else // VmaAllocation_T::ALLOCATION_TYPE_OWN - { - FreeOwnMemory(allocation); - } + vma_delete(this, allocation); } void VmaAllocator_T::CalculateStats(VmaStats* pStats) @@ -4437,11 +6234,10 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { - VmaMutexLock allocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex); const uint32_t heapIndex = m_MemProps.memoryTypes[memTypeIndex].heapIndex; for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType) { - const VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; + VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; VMA_ASSERT(pBlockVector); pBlockVector->AddStats(pStats, memTypeIndex, heapIndex); } @@ -4481,12 +6277,20 @@ void VmaAllocator_T::UnmapPersistentlyMappedMemory() // Process normal Allocations. { - VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex); VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED]; pBlockVector->UnmapPersistentlyMappedMemory(); } } } + + // Process custom pools. + { + VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) + { + m_Pools[poolIndex]->GetBlockVector().UnmapPersistentlyMappedMemory(); + } + } } } } @@ -4499,6 +6303,15 @@ VkResult VmaAllocator_T::MapPersistentlyMappedMemory() VkResult finalResult = VK_SUCCESS; if(m_PhysicalDeviceProperties.vendorID == VMA_VENDOR_ID_AMD) { + // Process custom pools. + { + VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) + { + m_Pools[poolIndex]->GetBlockVector().MapPersistentlyMappedMemory(); + } + } + for(size_t memTypeIndex = 0; memTypeIndex < m_MemProps.memoryTypeCount; ++memTypeIndex) { const VkMemoryPropertyFlags memFlags = m_MemProps.memoryTypes[memTypeIndex].propertyFlags; @@ -4518,7 +6331,6 @@ VkResult VmaAllocator_T::MapPersistentlyMappedMemory() // Process normal Allocations. { - VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex); VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][VMA_BLOCK_VECTOR_TYPE_MAPPED]; VkResult localResult = pBlockVector->MapPersistentlyMappedMemory(); if(localResult != VK_SUCCESS) @@ -4557,51 +6369,54 @@ VkResult VmaAllocator_T::Defragment( return VK_ERROR_MEMORY_MAP_FAILED; } - // Initialize defragmentators per memory type. - const VkDeviceSize bufferImageGranularity = GetBufferImageGranularity(); - VmaDefragmentator* pDefragmentators[VK_MAX_MEMORY_TYPES][VMA_BLOCK_VECTOR_TYPE_COUNT]; - memset(pDefragmentators, 0, sizeof(pDefragmentators)); - for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) - { - // Only HOST_VISIBLE memory types can be defragmented. - if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) - { - for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType) - { - pDefragmentators[memTypeIndex][blockVectorType] = vma_new(this, VmaDefragmentator)( - m_hDevice, - GetAllocationCallbacks(), - bufferImageGranularity, - memTypeIndex, - (VMA_BLOCK_VECTOR_TYPE)blockVectorType); - } - } - } + const uint32_t currentFrameIndex = m_CurrentFrameIndex.load(); - // Dispatch pAllocations among defragmentators. + VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex); + + const size_t poolCount = m_Pools.size(); + + // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary. for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex) { VmaAllocation hAlloc = pAllocations[allocIndex]; VMA_ASSERT(hAlloc); - if(hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) - { - const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex(); + const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex(); + // OwnAlloc cannot be defragmented. + if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) && // Only HOST_VISIBLE memory types can be defragmented. - if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) + ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) && + // Lost allocation cannot be defragmented. + (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)) + { + VmaBlockVector* pAllocBlockVector = nullptr; + + const VmaPool hAllocPool = hAlloc->GetPool(); + // This allocation belongs to custom pool. + if(hAllocPool != VK_NULL_HANDLE) { - const VMA_BLOCK_VECTOR_TYPE blockVectorType = hAlloc->GetBlockVectorType(); - VkBool32* pChanged = (pAllocationsChanged != VMA_NULL) ? - &pAllocationsChanged[allocIndex] : VMA_NULL; - pDefragmentators[memTypeIndex][blockVectorType]->AddAllocation(hAlloc, pChanged); + pAllocBlockVector = &hAllocPool->GetBlockVector(); } - // else: skip this allocation, cannot move it. + // This allocation belongs to general pool. + else + { + pAllocBlockVector = m_pBlockVectors[memTypeIndex][hAlloc->GetBlockVectorType()]; + } + + VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator( + m_hDevice, + GetAllocationCallbacks(), + currentFrameIndex); + + VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ? + &pAllocationsChanged[allocIndex] : VMA_NULL; + pDefragmentator->AddAllocation(hAlloc, pChanged); } - // else ALLOCATION_TYPE_OWN: skip this allocation, nothing to defragment. } VkResult result = VK_SUCCESS; - // Main processing. + // ======== Main processing. + VkDeviceSize maxBytesToMove = SIZE_MAX; uint32_t maxAllocationsToMove = UINT32_MAX; if(pDefragmentationInfo != VMA_NULL) @@ -4609,6 +6424,8 @@ VkResult VmaAllocator_T::Defragment( maxBytesToMove = pDefragmentationInfo->maxBytesToMove; maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove; } + + // Process standard memory. for(uint32_t memTypeIndex = 0; (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS); ++memTypeIndex) @@ -4616,63 +6433,44 @@ VkResult VmaAllocator_T::Defragment( // Only HOST_VISIBLE memory types can be defragmented. if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { - VmaMutexLock lock(m_BlocksMutex[memTypeIndex], m_UseMutex); - for(uint32_t blockVectorType = 0; (blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT) && (result == VK_SUCCESS); ++blockVectorType) { - VmaBlockVector* pBlockVector = m_pBlockVectors[memTypeIndex][blockVectorType]; - - // Defragment. - result = pDefragmentators[memTypeIndex][blockVectorType]->Defragment(pBlockVector, maxBytesToMove, maxAllocationsToMove); - - // Accumulate statistics. - if(pDefragmentationStats != VMA_NULL) - { - const VkDeviceSize bytesMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetBytesMoved(); - const uint32_t allocationsMoved = pDefragmentators[memTypeIndex][blockVectorType]->GetAllocationsMoved(); - pDefragmentationStats->bytesMoved += bytesMoved; - pDefragmentationStats->allocationsMoved += allocationsMoved; - VMA_ASSERT(bytesMoved <= maxBytesToMove); - VMA_ASSERT(allocationsMoved <= maxAllocationsToMove); - maxBytesToMove -= bytesMoved; - maxAllocationsToMove -= allocationsMoved; - } - - // Free empty blocks. - for(size_t blockIndex = pBlockVector->m_Blocks.size(); blockIndex--; ) - { - VmaBlock* pBlock = pBlockVector->m_Blocks[blockIndex]; - if(pBlock->IsEmpty()) - { - if(pDefragmentationStats != VMA_NULL) - { - ++pDefragmentationStats->deviceMemoryBlocksFreed; - pDefragmentationStats->bytesFreed += pBlock->m_Size; - } - - VmaVectorRemove(pBlockVector->m_Blocks, blockIndex); - pBlock->Destroy(this); - vma_delete(this, pBlock); - } - } - - // All block vector types processed: we can be sure that all empty allocations have been freed. - if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_COUNT - 1) - { - m_HasEmptyBlock[memTypeIndex] = false; - } + result = m_pBlockVectors[memTypeIndex][blockVectorType]->Defragment( + pDefragmentationStats, + maxBytesToMove, + maxAllocationsToMove); } } } - // Destroy defragmentators. + // Process custom pools. + for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex) + { + result = m_Pools[poolIndex]->GetBlockVector().Defragment( + pDefragmentationStats, + maxBytesToMove, + maxAllocationsToMove); + } + + // ======== Destroy defragmentators. + + // Process custom pools. + for(size_t poolIndex = poolCount; poolIndex--; ) + { + m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator(); + } + + // Process standard memory. for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; ) { - for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; ) + if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { - vma_delete(this, pDefragmentators[memTypeIndex][blockVectorType]); + for(size_t blockVectorType = VMA_BLOCK_VECTOR_TYPE_COUNT; blockVectorType--; ) + { + m_pBlockVectors[memTypeIndex][blockVectorType]->DestroyDefragmentator(); + } } } @@ -4681,12 +6479,126 @@ VkResult VmaAllocator_T::Defragment( void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo) { - pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex(); - pAllocationInfo->deviceMemory = hAllocation->GetMemory(); - pAllocationInfo->offset = hAllocation->GetOffset(); - pAllocationInfo->size = hAllocation->GetSize(); - pAllocationInfo->pMappedData = hAllocation->GetMappedData(); - pAllocationInfo->pUserData = hAllocation->GetUserData(); + if(hAllocation->CanBecomeLost()) + { + /* + Warning: This is a carefully designed algorithm. + Do not modify unless you really know what you're doing :) + */ + uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load(); + uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex(); + for(;;) + { + if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST) + { + pAllocationInfo->memoryType = UINT32_MAX; + pAllocationInfo->deviceMemory = VK_NULL_HANDLE; + pAllocationInfo->offset = 0; + pAllocationInfo->size = hAllocation->GetSize(); + pAllocationInfo->pMappedData = VMA_NULL; + pAllocationInfo->pUserData = hAllocation->GetUserData(); + return; + } + else if(localLastUseFrameIndex == localCurrFrameIndex) + { + pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex(); + pAllocationInfo->deviceMemory = hAllocation->GetMemory(); + pAllocationInfo->offset = hAllocation->GetOffset(); + pAllocationInfo->size = hAllocation->GetSize(); + pAllocationInfo->pMappedData = hAllocation->GetMappedData(); + pAllocationInfo->pUserData = hAllocation->GetUserData(); + return; + } + else // Last use time earlier than current time. + { + if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex)) + { + localLastUseFrameIndex = localCurrFrameIndex; + } + } + } + } + // We could use the same code here, but for performance reasons we don't need to use the hAllocation.LastUseFrameIndex atomic. + else + { + pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex(); + pAllocationInfo->deviceMemory = hAllocation->GetMemory(); + pAllocationInfo->offset = hAllocation->GetOffset(); + pAllocationInfo->size = hAllocation->GetSize(); + pAllocationInfo->pMappedData = hAllocation->GetMappedData(); + pAllocationInfo->pUserData = hAllocation->GetUserData(); + } +} + +VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool) +{ + VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex); + + VmaPoolCreateInfo newCreateInfo = *pCreateInfo; + + if(newCreateInfo.maxBlockCount == 0) + { + newCreateInfo.maxBlockCount = SIZE_MAX; + } + if(newCreateInfo.blockSize == 0) + { + newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex); + } + + *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo); + + VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks(); + if(res != VK_SUCCESS) + { + vma_delete(this, *pPool); + *pPool = VMA_NULL; + return res; + } + + // Add to m_Pools. + { + VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + VmaVectorInsertSorted(m_Pools, *pPool); + } + + return VK_SUCCESS; +} + +void VmaAllocator_T::DestroyPool(VmaPool pool) +{ + // Remove from m_Pools. + { + VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + bool success = VmaVectorRemoveSorted(m_Pools, pool); + VMA_ASSERT(success && "Pool not found in Allocator."); + } + + vma_delete(this, pool); +} + +void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats) +{ + pool->m_BlockVector.GetPoolStats(pPoolStats); +} + +void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex) +{ + m_CurrentFrameIndex.store(frameIndex); +} + +void VmaAllocator_T::MakePoolAllocationsLost( + VmaPool hPool, + size_t* pLostAllocationCount) +{ + hPool->m_BlockVector.MakePoolAllocationsLost( + m_CurrentFrameIndex.load(), + pLostAllocationCount); +} + +void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation) +{ + *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST); + (*pAllocation)->InitLost(); } void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation) @@ -4698,22 +6610,8 @@ void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation) VmaMutexLock lock(m_OwnAllocationsMutex[memTypeIndex], m_UseMutex); AllocationVectorType* const pOwnAllocations = m_pOwnAllocations[memTypeIndex][allocation->GetBlockVectorType()]; VMA_ASSERT(pOwnAllocations); - VmaAllocation* const pOwnAllocationsBeg = pOwnAllocations->data(); - VmaAllocation* const pOwnAllocationsEnd = pOwnAllocationsBeg + pOwnAllocations->size(); - VmaAllocation* const pOwnAllocationIt = VmaBinaryFindFirstNotLess( - pOwnAllocationsBeg, - pOwnAllocationsEnd, - allocation, - VmaPointerLess()); - if(pOwnAllocationIt != pOwnAllocationsEnd) - { - const size_t ownAllocationIndex = pOwnAllocationIt - pOwnAllocationsBeg; - VmaVectorRemove(*pOwnAllocations, ownAllocationIndex); - } - else - { - VMA_ASSERT(0); - } + bool success = VmaVectorRemoveSorted(*pOwnAllocations, allocation); + VMA_ASSERT(success); } VkDeviceMemory hMemory = allocation->GetMemory(); @@ -4732,13 +6630,11 @@ void VmaAllocator_T::FreeOwnMemory(VmaAllocation allocation) vkFreeMemory(m_hDevice, hMemory, GetAllocationCallbacks()); VMA_DEBUG_LOG(" Freed OwnMemory MemoryTypeIndex=%u", memTypeIndex); - - vma_delete(this, allocation); } #if VMA_STATS_STRING_ENABLED -void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb) +void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) { bool ownAllocationsStarted = false; for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) @@ -4750,82 +6646,91 @@ void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb) VMA_ASSERT(pOwnAllocVector); if(pOwnAllocVector->empty() == false) { - if(ownAllocationsStarted) + if(ownAllocationsStarted == false) { - sb.Add(",\n\t\"Type "); - } - else - { - sb.Add(",\n\"OwnAllocations\": {\n\t\"Type "); ownAllocationsStarted = true; + json.WriteString("OwnAllocations"); + json.BeginObject(); } - sb.AddNumber(memTypeIndex); + + json.BeginString("Type "); + json.ContinueString(memTypeIndex); if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED) { - sb.Add(" Mapped"); + json.ContinueString(" Mapped"); } - sb.Add("\": ["); + json.EndString(); + + json.BeginArray(); for(size_t i = 0; i < pOwnAllocVector->size(); ++i) { const VmaAllocation hAlloc = (*pOwnAllocVector)[i]; - if(i > 0) - { - sb.Add(",\n\t\t{ \"Size\": "); - } - else - { - sb.Add("\n\t\t{ \"Size\": "); - } - sb.AddNumber(hAlloc->GetSize()); - sb.Add(", \"Type\": "); - sb.AddString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]); - sb.Add(" }"); + json.BeginObject(true); + + json.WriteString("Size"); + json.WriteNumber(hAlloc->GetSize()); + + json.WriteString("Type"); + json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]); + + json.EndObject(); } - sb.Add("\n\t]"); + json.EndArray(); } } } if(ownAllocationsStarted) { - sb.Add("\n}"); + json.EndObject(); } { bool allocationsStarted = false; for(size_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { - VmaMutexLock globalAllocationsLock(m_BlocksMutex[memTypeIndex], m_UseMutex); for(uint32_t blockVectorType = 0; blockVectorType < VMA_BLOCK_VECTOR_TYPE_COUNT; ++blockVectorType) { if(m_pBlockVectors[memTypeIndex][blockVectorType]->IsEmpty() == false) { - if(allocationsStarted) + if(allocationsStarted == false) { - sb.Add(",\n\t\"Type "); - } - else - { - sb.Add(",\n\"Allocations\": {\n\t\"Type "); allocationsStarted = true; + json.WriteString("Allocations"); + json.BeginObject(); } - sb.AddNumber(memTypeIndex); + + json.BeginString("Type "); + json.ContinueString(memTypeIndex); if(blockVectorType == VMA_BLOCK_VECTOR_TYPE_MAPPED) { - sb.Add(" Mapped"); + json.ContinueString(" Mapped"); } - sb.Add("\": ["); + json.EndString(); - m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(sb); - - sb.Add("\n\t]"); + m_pBlockVectors[memTypeIndex][blockVectorType]->PrintDetailedMap(json); } } } if(allocationsStarted) { - sb.Add("\n}"); + json.EndObject(); + } + } + + { + VmaMutexLock lock(m_PoolsMutex, m_UseMutex); + const size_t poolCount = m_Pools.size(); + if(poolCount > 0) + { + json.WriteString("Pools"); + json.BeginArray(); + for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex) + { + m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json); + } + json.EndArray(); } } } @@ -4835,18 +6740,18 @@ void VmaAllocator_T::PrintDetailedMap(VmaStringBuilder& sb) static VkResult AllocateMemoryForImage( VmaAllocator allocator, VkImage image, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pAllocationCreateInfo, VmaSuballocationType suballocType, VmaAllocation* pAllocation) { - VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pMemoryRequirements && pAllocation); + VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation); VkMemoryRequirements vkMemReq = {}; vkGetImageMemoryRequirements(allocator->m_hDevice, image, &vkMemReq); return allocator->AllocateMemory( vkMemReq, - *pMemoryRequirements, + *pAllocationCreateInfo, suballocType, pAllocation); } @@ -4901,6 +6806,18 @@ void vmaGetMemoryTypeProperties( *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags; } +void vmaSetCurrentFrameIndex( + VmaAllocator allocator, + uint32_t frameIndex) +{ + VMA_ASSERT(allocator); + VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + allocator->SetCurrentFrameIndex(frameIndex); +} + void vmaCalculateStats( VmaAllocator allocator, VmaStats* pStats) @@ -4922,77 +6839,92 @@ void vmaBuildStatsString( VmaStringBuilder sb(allocator); { + VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb); + json.BeginObject(); + VmaStats stats; allocator->CalculateStats(&stats); - sb.Add("{\n\"Total\": "); - VmaPrintStatInfo(sb, stats.total); + json.WriteString("Total"); + VmaPrintStatInfo(json, stats.total); for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex) { - sb.Add(",\n\"Heap "); - sb.AddNumber(heapIndex); - sb.Add("\": {\n\t\"Size\": "); - sb.AddNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size); - sb.Add(",\n\t\"Flags\": "); + json.BeginString("Heap "); + json.ContinueString(heapIndex); + json.EndString(); + json.BeginObject(); + + json.WriteString("Size"); + json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size); + + json.WriteString("Flags"); + json.BeginArray(true); if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) { - sb.AddString("DEVICE_LOCAL"); + json.WriteString("DEVICE_LOCAL"); } - else + json.EndArray(); + + if(stats.memoryHeap[heapIndex].BlockCount > 0) { - sb.AddString(""); - } - if(stats.memoryHeap[heapIndex].AllocationCount > 0) - { - sb.Add(",\n\t\"Stats:\": "); - VmaPrintStatInfo(sb, stats.memoryHeap[heapIndex]); + json.WriteString("Stats"); + VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]); } for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex) { if(allocator->m_MemProps.memoryTypes[typeIndex].heapIndex == heapIndex) { - sb.Add(",\n\t\"Type "); - sb.AddNumber(typeIndex); - sb.Add("\": {\n\t\t\"Flags\": \""); + json.BeginString("Type "); + json.ContinueString(typeIndex); + json.EndString(); + + json.BeginObject(); + + json.WriteString("Flags"); + json.BeginArray(true); VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags; if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) { - sb.Add(" DEVICE_LOCAL"); + json.WriteString("DEVICE_LOCAL"); } if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { - sb.Add(" HOST_VISIBLE"); + json.WriteString("HOST_VISIBLE"); } if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0) { - sb.Add(" HOST_COHERENT"); + json.WriteString("HOST_COHERENT"); } if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0) { - sb.Add(" HOST_CACHED"); + json.WriteString("HOST_CACHED"); } if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0) { - sb.Add(" LAZILY_ALLOCATED"); + json.WriteString("LAZILY_ALLOCATED"); } - sb.Add("\""); - if(stats.memoryType[typeIndex].AllocationCount > 0) + json.EndArray(); + + if(stats.memoryType[typeIndex].BlockCount > 0) { - sb.Add(",\n\t\t\"Stats\": "); - VmaPrintStatInfo(sb, stats.memoryType[typeIndex]); + json.WriteString("Stats"); + VmaPrintStatInfo(json, stats.memoryType[typeIndex]); } - sb.Add("\n\t}"); + + json.EndObject(); } } - sb.Add("\n}"); + + json.EndObject(); } if(detailedMap == VK_TRUE) { - allocator->PrintDetailedMap(sb); + allocator->PrintDetailedMap(json); } - sb.Add("\n}\n"); + + json.EndObject(); } const size_t len = sb.GetLength(); @@ -5024,15 +6956,15 @@ void vmaFreeStatsString( VkResult vmaFindMemoryTypeIndex( VmaAllocator allocator, uint32_t memoryTypeBits, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pAllocationCreateInfo, uint32_t* pMemoryTypeIndex) { VMA_ASSERT(allocator != VK_NULL_HANDLE); - VMA_ASSERT(pMemoryRequirements != VMA_NULL); + VMA_ASSERT(pAllocationCreateInfo != VMA_NULL); VMA_ASSERT(pMemoryTypeIndex != VMA_NULL); - uint32_t requiredFlags = pMemoryRequirements->requiredFlags; - uint32_t preferredFlags = pMemoryRequirements->preferredFlags; + uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags; + uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags; if(preferredFlags == 0) { preferredFlags = requiredFlags; @@ -5041,7 +6973,7 @@ VkResult vmaFindMemoryTypeIndex( VMA_ASSERT((requiredFlags & ~preferredFlags) == 0); // Convert usage to requiredFlags and preferredFlags. - switch(pMemoryRequirements->usage) + switch(pAllocationCreateInfo->usage) { case VMA_MEMORY_USAGE_UNKNOWN: break; @@ -5063,7 +6995,7 @@ VkResult vmaFindMemoryTypeIndex( break; } - if((pMemoryRequirements->flags & VMA_MEMORY_REQUIREMENT_PERSISTENT_MAP_BIT) != 0) + if((pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_PERSISTENT_MAP_BIT) != 0) { requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; } @@ -5100,14 +7032,65 @@ VkResult vmaFindMemoryTypeIndex( return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT; } +VkResult vmaCreatePool( + VmaAllocator allocator, + const VmaPoolCreateInfo* pCreateInfo, + VmaPool* pPool) +{ + VMA_ASSERT(allocator && pCreateInfo && pPool); + + VMA_DEBUG_LOG("vmaCreatePool"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + return allocator->CreatePool(pCreateInfo, pPool); +} + +void vmaDestroyPool( + VmaAllocator allocator, + VmaPool pool) +{ + VMA_ASSERT(allocator && pool); + + VMA_DEBUG_LOG("vmaDestroyPool"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + allocator->DestroyPool(pool); +} + +void vmaGetPoolStats( + VmaAllocator allocator, + VmaPool pool, + VmaPoolStats* pPoolStats) +{ + VMA_ASSERT(allocator && pool && pPoolStats); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + allocator->GetPoolStats(pool, pPoolStats); +} + +void vmaMakePoolAllocationsLost( + VmaAllocator allocator, + VmaPool pool, + size_t* pLostAllocationCount) +{ + VMA_ASSERT(allocator && pool); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + allocator->MakePoolAllocationsLost(pool, pLostAllocationCount); +} + VkResult vmaAllocateMemory( VmaAllocator allocator, const VkMemoryRequirements* pVkMemoryRequirements, - const VmaMemoryRequirements* pVmaMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - VMA_ASSERT(allocator && pVkMemoryRequirements && pVmaMemoryRequirements && pAllocation); + VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemory"); @@ -5115,7 +7098,7 @@ VkResult vmaAllocateMemory( VkResult result = allocator->AllocateMemory( *pVkMemoryRequirements, - *pVmaMemoryRequirements, + *pCreateInfo, VMA_SUBALLOCATION_TYPE_UNKNOWN, pAllocation); @@ -5130,11 +7113,11 @@ VkResult vmaAllocateMemory( VkResult vmaAllocateMemoryForBuffer( VmaAllocator allocator, VkBuffer buffer, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pMemoryRequirements && pAllocation); + VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer"); @@ -5145,7 +7128,7 @@ VkResult vmaAllocateMemoryForBuffer( VkResult result = allocator->AllocateMemory( vkMemReq, - *pMemoryRequirements, + *pCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); @@ -5160,11 +7143,11 @@ VkResult vmaAllocateMemoryForBuffer( VkResult vmaAllocateMemoryForImage( VmaAllocator allocator, VkImage image, - const VmaMemoryRequirements* pMemoryRequirements, + const VmaAllocationCreateInfo* pCreateInfo, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pMemoryRequirements && pAllocation); + VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation); VMA_DEBUG_LOG("vmaAllocateMemoryForImage"); @@ -5173,7 +7156,7 @@ VkResult vmaAllocateMemoryForImage( VkResult result = AllocateMemoryForImage( allocator, image, - pMemoryRequirements, + pCreateInfo, VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, pAllocation); @@ -5222,6 +7205,17 @@ void vmaSetAllocationUserData( allocation->SetUserData(pUserData); } +void vmaCreateLostAllocation( + VmaAllocator allocator, + VmaAllocation* pAllocation) +{ + VMA_ASSERT(allocator && pAllocation); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK; + + allocator->CreateLostAllocation(pAllocation); +} + VkResult vmaMapMemory( VmaAllocator allocator, VmaAllocation allocation, @@ -5283,13 +7277,13 @@ VkResult vmaDefragment( VkResult vmaCreateBuffer( VmaAllocator allocator, - const VkBufferCreateInfo* pCreateInfo, - const VmaMemoryRequirements* pMemoryRequirements, + const VkBufferCreateInfo* pBufferCreateInfo, + const VmaAllocationCreateInfo* pAllocationCreateInfo, VkBuffer* pBuffer, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pBuffer && pAllocation); + VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation); VMA_DEBUG_LOG("vmaCreateBuffer"); @@ -5299,7 +7293,7 @@ VkResult vmaCreateBuffer( *pAllocation = VK_NULL_HANDLE; // 1. Create VkBuffer. - VkResult res = vkCreateBuffer(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pBuffer); + VkResult res = vkCreateBuffer(allocator->m_hDevice, pBufferCreateInfo, allocator->GetAllocationCallbacks(), pBuffer); if(res >= 0) { // 2. vkGetBufferMemoryRequirements. @@ -5309,7 +7303,7 @@ VkResult vmaCreateBuffer( // 3. Allocate memory using allocator. res = allocator->AllocateMemory( vkMemReq, - *pMemoryRequirements, + *pAllocationCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, pAllocation); if(res >= 0) @@ -5357,13 +7351,13 @@ void vmaDestroyBuffer( VkResult vmaCreateImage( VmaAllocator allocator, - const VkImageCreateInfo* pCreateInfo, - const VmaMemoryRequirements* pMemoryRequirements, + const VkImageCreateInfo* pImageCreateInfo, + const VmaAllocationCreateInfo* pAllocationCreateInfo, VkImage* pImage, VmaAllocation* pAllocation, VmaAllocationInfo* pAllocationInfo) { - VMA_ASSERT(allocator && pCreateInfo && pMemoryRequirements && pImage && pAllocation); + VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation); VMA_DEBUG_LOG("vmaCreateImage"); @@ -5373,16 +7367,15 @@ VkResult vmaCreateImage( *pAllocation = VK_NULL_HANDLE; // 1. Create VkImage. - VkResult res = vkCreateImage(allocator->m_hDevice, pCreateInfo, allocator->GetAllocationCallbacks(), pImage); + VkResult res = vkCreateImage(allocator->m_hDevice, pImageCreateInfo, allocator->GetAllocationCallbacks(), pImage); if(res >= 0) { - VkMappedMemoryRange mem = {}; - VmaSuballocationType suballocType = pCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ? + VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ? VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL : VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR; // 2. Allocate memory using allocator. - res = AllocateMemoryForImage(allocator, *pImage, pMemoryRequirements, suballocType, pAllocation); + res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation); if(res >= 0) { // 3. Bind image with memory.